@tangle-network/agent-eval 0.144.6 → 0.144.7

package/dist/{campaign-CXsdyym7.js → campaign-Tdy3h62h.js}

@@ -1,16 +1,19 @@
 import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
-import { p as canonicalize, s as buildAgentProfileCell } from "./agent-profile-cell-CbfBm2g6.js";
+import { t as canonicalize } from "./pre-registration-DakwTRXk.js";
+import { buildAgentProfileCell } from "./profile-cell.js";
 import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-Dv5BiKLE.js";
+import { f as maximumChargeForLlmRequest, h as ModelSubstitutionError, l as costReceiptFromLlm, u as costReceiptFromLlmError, y as assertServedModel } from "./llm-client-DzvMUsS_.js";
 import { r as canonicalString } from "./canonical-D011XM8r.js";
-import { a as SEARCH_LEDGER_FILE_CONTEXT, c as SearchLedgerIntegrityError, o as SearchLedgerConflictError, s as SearchLedgerError, y as contentHash } from "./single-run-lock-D5iN0Xzb.js";
+import { a as SEARCH_LEDGER_FILE_CONTEXT, c as SearchLedgerIntegrityError, o as SearchLedgerConflictError, s as SearchLedgerError } from "./single-run-lock-BMQEv1wG.js";
+import { r as contentHash } from "./verdict-cache-BCcOh0kF.js";
 import { p as mapConcurrent, t as FileLedgerJournal } from "./ledger-core-DXZIqu17.js";
-import { B as surfaceContentHash, E as pairHoldout, F as assertCodeSurfaceIdentity, J as planCampaignRun, Mt as JudgeParseError, St as recoverTruncatedJson, Y as runCampaign, h as labelTrustRank, jt as summarizeBackendIntegrity, kt as assertRealBackend } from "./skillopt-optimization-method-Bfb-vBKe.js";
-import { C as mulberry32, E as pairedBootstrap, m as eProcess } from "./statistics-ByxzSiOM.js";
+import { B as planCampaignRun, D as assertCodeSurfaceIdentity, M as surfaceContentHash, V as runCampaign, _t as assertRealBackend, bt as JudgeParseError, h as labelTrustRank, mt as recoverTruncatedJson, yt as summarizeBackendIntegrity } from "./skillopt-optimization-method-CQwZ-ZX8.js";
+import { E as pairedBootstrap } from "./statistics-ByxzSiOM.js";
 import { t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
-import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-CWN8-VsV.js";
+import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-DqOw5X6_.js";
+import { l as pairHoldout } from "./promotion-policy-CrLrmys8.js";
 import { a as scoreAnalystFindings } from "./benchmark-B181aMF9.js";
-import { l as campaignCellToRunRecord } from "./reward-hacking-CyuzxKly.js";
+import { l as campaignCellToRunRecord } from "./reward-hacking-B2tPL9a3.js";
 import { z } from "zod";
 import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -336,6 +339,10 @@ function createLlmCorrectnessChecker(chat, opts = {}) {
 				});
 				if (!paid.succeeded) throw paid.error;
 				const resp = paid.value;
+				assertServedModel(model, resp.servedModel, {
+					allowUnreported: true,
+					context: `correctness checker for requirement ${requirement.reqId}`
+				});
 				const raw = resp.content;
 				await record({
 					eventId: randomUUID(),
@@ -366,6 +373,7 @@ function createLlmCorrectnessChecker(chat, opts = {}) {
 					errorMessage: err instanceof Error ? err.message : String(err),
 					redactedFields: []
 				});
+				if (err instanceof ModelSubstitutionError) throw err;
 			}
 		}
 		throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
@@ -1647,298 +1655,6 @@ function neutralizationGate(options) {
 	};
 }
 //#endregion
-//#region src/campaign/gates/sequential.ts
-/**
-* Anytime-valid sequential promotion gate — an e-process (betting
-* test-martingale, see `eProcess` in `statistics.ts`) over paired
-* per-scenario deltas, so a campaign stops the MOMENT evidence decides
-* instead of burning a fixed-n budget. Decisions remain valid at any
-* data-dependent stopping time (Ville's inequality), which is exactly what
-* fixed-n machinery cannot offer: peeking at a bootstrap CI after every
-* observation and stopping on the first significant peek inflates type-I
-* error far beyond alpha.
-*
-* REPLACES, never layers on, a fixed-n gate. Running `heldoutSignificance`
-* or `paretoSignificanceGate` repeatedly on a growing sample and stopping
-* early is optional stopping no matter how it is dressed up; this gate is
-* the valid way to stop early. Use one or the other per evidence stream.
-*
-* Pre-registration binding: anytime validity holds only for the
-* PRE-REGISTERED statistic. When a `SignedManifest` is bound, the gate takes
-* alpha from `manifest.alpha`, the observation budget from
-* `manifest.preRegisteredN`, orients deltas by `manifest.direction`, and
-* shifts the null boundary by `manifest.minEffect` — re-deciding the same
-* stream under different parameters after seeing data would reopen optional
-* stopping under a fancier name. The manifest's content hash is verified at
-* construction (sync, same `sha256-content` scheme as `signManifest`).
-*
-* Non-iid caveat (stated honestly): the supermartingale guarantee needs each
-* delta's conditional mean under H0 to stay ≤ the null boundary given the
-* past — exchangeable scenario deltas suffice. Scenario streams ordered by
-* difficulty or by scenario family violate this; `decide(ctx)` therefore
-* shuffles the paired deltas with a SEEDED permutation by default (the
-* permutation is data-independent, so bet predictability is preserved).
-* Stratified betting (per-stratum λ) is future work, not implemented here.
-*/
-/** Sync twin of `verifyManifest` — same `sha256-content` scheme
-*  (sha256 over the canonicalized manifest minus contentHash/algo), via
-*  node:crypto so gate construction can stay synchronous and fail loud
-*  before any observation is consumed. */
-function verifyManifestSync(m) {
-	if (m.algo !== void 0 && m.algo !== "sha256-content") throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
-	const { contentHash, algo: _algo, ...rest } = m;
-	const bytes = JSON.stringify(canonicalize(rest));
-	return createHash("sha256").update(bytes, "utf8").digest("hex") === contentHash;
-}
-function resolveConfig(opts) {
-	const m = opts.preRegistration;
-	let alpha;
-	let maxN;
-	let direction;
-	let minEffect;
-	if (m) {
-		if (!verifyManifestSync(m)) throw new Error(`sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`);
-		if (opts.alpha !== void 0 && opts.alpha !== m.alpha) throw new Error(`sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} — the registered statistic is the only one anytime validity covers`);
-		if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) throw new Error(`sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`);
-		alpha = m.alpha;
-		maxN = m.preRegisteredN;
-		direction = m.direction;
-		minEffect = m.minEffect;
-	} else {
-		alpha = opts.alpha ?? .05;
-		if (opts.maxN === void 0) throw new Error("sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) — an unbounded stream has no pre-registered budget");
-		maxN = opts.maxN;
-		direction = "increase";
-		minEffect = 0;
-	}
-	const scale = opts.scale ?? 1;
-	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
-	if (!Number.isInteger(maxN) || maxN < 1) throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
-	const minN = opts.minN ?? 5;
-	if (!Number.isInteger(minN) || minN < 1 || minN > maxN) throw new Error(`sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`);
-	if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) throw new Error(`sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`);
-	const nullMean = .5 + minEffect / (2 * scale);
-	return {
-		alpha,
-		minN,
-		maxN,
-		maxBet: opts.maxBet ?? .5,
-		scale,
-		shuffleSeed: opts.shuffleSeed ?? 1337,
-		direction,
-		nullMean,
-		minEffect
-	};
-}
-function makeStream(cfg) {
-	const proc = eProcess({
-		alpha: cfg.alpha,
-		maxBet: cfg.maxBet,
-		nullMean: cfg.nullMean
-	});
-	const threshold = 1 / cfg.alpha;
-	let terminal;
-	return {
-		observe(delta) {
-			if (terminal === "undecided-at-maxN") throw new Error(`sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted — extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`);
-			if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) throw new Error(`sequentialPairedGate: delta ${delta} outside ±scale=${cfg.scale} — pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`);
-			const d = cfg.direction === "decrease" ? -delta : delta;
-			const step = proc.update((d / cfg.scale + 1) / 2);
-			if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) terminal = "promote";
-			if (terminal === "promote") return {
-				decision: "promote",
-				eValue: step.wealth,
-				n: step.n,
-				reason: `e-value ${step.wealth.toFixed(2)} ≥ 1/α=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level α=${cfg.alpha}`
-			};
-			if (step.n >= cfg.maxN) {
-				terminal = "undecided-at-maxN";
-				return {
-					decision: "undecided-at-maxN",
-					eValue: step.wealth,
-					n: step.n,
-					reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/α=${threshold.toFixed(2)}). This is NOT evidence of no effect — the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
-				};
-			}
-			return {
-				decision: "continue",
-				eValue: step.wealth,
-				n: step.n,
-				reason: `e-value ${step.wealth.toFixed(2)} < 1/α=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} — keep observing`
-			};
-		},
-		state() {
-			return {
-				...proc.state(),
-				decision: terminal ?? "continue"
-			};
-		}
-	};
-}
-/** Data-independent in-place Fisher–Yates with a seeded PRNG — the permutation
-*  depends only on the seed, never the values, so bet predictability survives. */
-function seededShuffle(items, seed) {
-	const rng = mulberry32(seed);
-	for (let i = items.length - 1; i > 0; i--) {
-		const j = Math.floor(rng() * (i + 1));
-		const tmp = items[i];
-		items[i] = items[j];
-		items[j] = tmp;
-	}
-	return items;
-}
-/**
-* Anytime-valid sequential paired gate. Conforms to the existing `Gate`
-* contract (`decide(ctx)` consumes candidate vs baseline judge scores via
-* `pairHoldout` — same pairing granularity as the fixed-n gates: full cellId,
-* never scenarioId) and adds a streaming `observe(delta)` entry for campaigns
-* that score cells incrementally and want to stop mid-stream.
-*
-* Decision mapping onto the substrate's five-valued `GateDecision`:
-*   - 'promote'            → 'ship'
-*   - 'continue'           → 'need_more_work' (stream ended before maxN with
-*                            the e-value undecided — more reps could decide)
-*   - 'undecided-at-maxN'  → 'hold', with the reason stating it is NOT
-*                            evidence of no effect (never a silent default)
-*/
-function sequentialPairedGate(options) {
-	const cfg = resolveConfig(options);
-	const name = options.name ?? "sequentialPairedGate";
-	const manifest = options.preRegistration;
-	const observeStream = makeStream(cfg);
-	return {
-		name,
-		observe(delta) {
-			return observeStream.observe(delta);
-		},
-		state() {
-			return observeStream.state();
-		},
-		async decide(ctx) {
-			if (!ctx.baselineJudgeScores) throw new Error(`${name}: ctx.baselineJudgeScores is required — falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`);
-			const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
-			const paired = pairHoldout(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, (s) => s.composite);
-			const deltas = paired.after.map((a, i) => a - paired.before[i]);
-			seededShuffle(deltas, cfg.shuffleSeed);
-			const stream = makeStream(cfg);
-			let last;
-			for (const d of deltas) {
-				last = stream.observe(d);
-				if (last.decision !== "continue") break;
-			}
-			const detail = {
-				...stream.state(),
-				minN: cfg.minN,
-				maxN: cfg.maxN,
-				scale: cfg.scale,
-				shuffleSeed: cfg.shuffleSeed,
-				direction: cfg.direction,
-				minEffect: cfg.minEffect,
-				pairedN: deltas.length,
-				...manifest ? {
-					preRegisteredId: manifest.id,
-					metric: manifest.metric
-				} : {}
-			};
-			const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
-			if (last === void 0) return {
-				decision: "need_more_work",
-				reasons: [`${name}: no paired holdout observations — nothing to test`],
-				contributingGates: [{
-					name,
-					status: "not_evaluated",
-					detail
-				}]
-			};
-			const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
-			return {
-				decision,
-				reasons: [`${name}: ${last.reason}`],
-				contributingGates: [{
-					name,
-					status: decision === "ship" ? "pass" : decision === "need_more_work" ? "not_evaluated" : "fail",
-					detail
-				}],
-				delta: meanDelta
-			};
-		}
-	};
-}
-/**
-* `SurfaceProposer.decide` adapter — stops the optimization loop the moment
-* the e-process decides the loop has produced a real improvement, instead of
-* always running `maxGenerations`.
-*
-* Stream: for each generation g ≥ 1, the per-scenario composite deltas of
-* generation g's top candidate vs the generation-0 top candidate (the
-* incumbent the loop set out to beat), paired by scenarioId. H0: no proposed
-* surface improves any scenario's expected composite over the incumbent —
-* under it every delta has conditional mean ≤ 0 and the e-process is valid.
-* Once wealth ≥ 1/alpha the loop stops and hands the winner to the promotion
-* gate (which re-scores on HELD-OUT data — this adapter only spends the
-* exploration budget, it never promotes).
-*
-* Honesty caveats: (1) the incumbent's scores are measured once and shared
-* across all generations' deltas, so type-I control is exact only insofar as
-* those scores approximate the incumbent's true per-scenario means (more reps
-* → tighter); (2) an UNDECIDED process never stops the loop — absence of a
-* crossing is NOT evidence of no effect, so the loop simply runs its normal
-* course. Calling the adapter repeatedly with a growing history consumes each
-* generation exactly once (re-feeding an already-seen record would double-count
-* evidence).
-*/
-function sequentialDecide(options = {}) {
-	const alpha = options.alpha ?? .05;
-	const minN = options.minN ?? 5;
-	const scale = options.scale ?? 1;
-	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
-	const proc = eProcess({
-		alpha,
-		maxBet: options.maxBet ?? .5,
-		nullMean: .5
-	});
-	const threshold = 1 / alpha;
-	let processedGenerations = 0;
-	let reference;
-	let stopped;
-	const topCandidate = (record) => {
-		if (record.candidates.length === 0) throw new Error(`sequentialDecide: generation ${record.generationIndex} has no candidates — cannot extract a top candidate`);
-		return record.candidates.reduce((best, candidate) => {
-			if (best.composite === null) return candidate;
-			if (candidate.composite === null) return best;
-			return candidate.composite > best.composite ? candidate : best;
-		});
-	};
-	const decide = ({ history }) => {
-		if (stopped) return stopped;
-		if (history.length === 0) return { stop: false };
-		if (reference === void 0) reference = new Map(topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite]));
-		for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
-			const top = topCandidate(history[g]);
-			const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
-			for (const [scenarioId, refComposite] of reference) {
-				const candComposite = byScenario.get(scenarioId);
-				if (candComposite === void 0) throw new Error(`sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' — generations must score the same scenario set to pair`);
-				const delta = candComposite - refComposite;
-				if (!Number.isFinite(delta) || Math.abs(delta) > scale) throw new Error(`sequentialDecide: paired delta ${delta} outside ±scale=${scale} on scenario '${scenarioId}' — pass the composite scale explicitly`);
-				const step = proc.update((delta / scale + 1) / 2);
-				if (step.n >= minN && step.wealth >= threshold) {
-					stopped = {
-						stop: true,
-						reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} ≥ 1/α=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent — the improvement is real at α=${alpha}; stop exploring and promote via the gate`
-					};
-					processedGenerations = history.length;
-					return stopped;
-				}
-			}
-		}
-		processedGenerations = history.length;
-		return { stop: false };
-	};
-	decide.state = () => proc.state();
-	return decide;
-}
-//#endregion
 //#region src/campaign/grounded-reflection.ts
 /**
 * Deterministic per-field diff of call arguments between passing and failing
@@ -4074,6 +3790,6 @@ function resolveWorktreePath(surface, worktreeDir) {
 	return verifyCodeSurface(surface, worktreeDir).path;
 }
 //#endregion
-export { loadEvalFixture as A, agentProfileModelId as B, LabeledScenarioStoreError as C, sequentialPairedGate as D, sequentialDecide as E, traceAnalystQualityJudge as F, createLlmCorrectnessChecker as G, harnessAxisOf as H, CODING_HARNESSES as I, verifyCompletion as J, createTokenRecallChecker as K, HARNESS_NATIVE_MODEL as L, planEvalFixtureRun as M, analyzeCrossSurfaceInteractions as N, neutralizationGate as O, buildTraceAnalystSurfaceDispatch as P, agentProfileHash as R, FsLabeledScenarioStore as S, rolloutArgumentDiff as T, extractProducedState as U, expandProfileAxes as V, completionVerdict as W, renderScoreboardMarkdown as _, autoevalsScorerJudge as a, userStoryScoreboard as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, makePlaybackDispatch as g, runProfileMatrix as h, verifyCodeSurface as i, loadEvalFixtureScenarios as j, discoverEvalFixtures as k, SEARCH_LEDGER_SCHEMA as l, ProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, parseCorrectnessResponse as q, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, scoreUserStory as v, classifyUngroundedLiterals as w, neutralizeText as x, scoreboardSummary as y, agentProfileId as z };
+export { planEvalFixtureRun as A, harnessAxisOf as B, LabeledScenarioStoreError as C, discoverEvalFixtures as D, neutralizationGate as E, HARNESS_NATIVE_MODEL as F, parseCorrectnessResponse as G, completionVerdict as H, agentProfileHash as I, verifyCompletion as K, agentProfileId as L, buildTraceAnalystSurfaceDispatch as M, traceAnalystQualityJudge as N, loadEvalFixture as O, CODING_HARNESSES as P, agentProfileModelId as R, FsLabeledScenarioStore as S, rolloutArgumentDiff as T, createLlmCorrectnessChecker as U, extractProducedState as V, createTokenRecallChecker as W, renderScoreboardMarkdown as _, autoevalsScorerJudge as a, userStoryScoreboard as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, makePlaybackDispatch as g, runProfileMatrix as h, verifyCodeSurface as i, analyzeCrossSurfaceInteractions as j, loadEvalFixtureScenarios as k, SEARCH_LEDGER_SCHEMA as l, ProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, scoreUserStory as v, classifyUngroundedLiterals as w, neutralizeText as x, scoreboardSummary as y, expandProfileAxes as z };
 
-//# sourceMappingURL=campaign-CXsdyym7.js.map
+//# sourceMappingURL=campaign-Tdy3h62h.js.map