@rulvar/evals 1.81.2 → 1.83.0

package/dist/index.d.ts

@@ -421,7 +421,8 @@ interface RunBenchmarkOptions {
   /**
   * Aggregate debit-only envelope: every target and judge run
   * authorizes its ceiling here BEFORE starting, exactly like the eval
-  * runners. A target-run refusal throws SweepBudgetError; a judge-run
+  * runners. A target-run refusal ends the series monotonically
+  * (report.refusal) with every completed repeat preserved; a judge-run
   * refusal rejects that run from scoring as 'judge:refused'.
   */
   envelope?: SpendEnvelope;
@@ -520,13 +521,26 @@ interface BenchmarkReport {
   /** Every target and judge run, scored or rejected (honest spend). */
   totalCostUsd: number;
   judgeCostUsd: number;
+  /**
+  * Present when the aggregate envelope refused a TARGET run before it
+  * started (cycle 81): the series ends there and every completed
+  * repeat stays on the report, mirroring the eval suite's monotone
+  * refusal instead of a throw destroying the paid evidence. Judge
+  * refusals never appear here; they reject their own run as
+  * 'judge:refused'.
+  */
+  refusal?: {
+    atOrdinal: number;
+    detail: string;
+  };
   fingerprint: BenchmarkFingerprint;
 }
 /**
 * Runs the spec's repeats sequentially and reports the verified series.
 * Throws only for spec defects (invalid repeats, a throwing grader or
-* extractor) and for a target-run envelope refusal; everything a run
-* does wrong lands in its record instead.
+* extractor); everything a run does wrong lands in its record, and a
+* target-run envelope refusal ends the series monotonically with the
+* completed repeats preserved (report.refusal).
 */
 declare function runBenchmark(engine: Engine, spec: BenchmarkSpec, options?: RunBenchmarkOptions): Promise<BenchmarkReport>;
 //#endregion
@@ -622,6 +636,15 @@ interface SweepCellReport {
   */
   exhaustedRuns?: number;
   /**
+  * Count of case results whose TARGET run settled neither ok nor
+  * exhausted ('error', 'cancelled', 'suspended'): measurement
+  * artifacts, not model quality. Exactly like exhaustedRuns, any such
+  * run suppresses the cell's claim: a passRate deflated by a provider
+  * failure or a host cancellation must not become a committed model
+  * weakness (cycle 81).
+  */
+  nonOkRuns?: number;
+  /**
   * Count of result rows whose grading stopped on a judge budget event
   * (per-run judge ceiling or envelope refusal of a judge run). The
   * paid target evidence stays on those rows; the cell emits no claim.
@@ -707,6 +730,15 @@ interface CheckpointCell {
   recommended: boolean;
   baseline: CheckpointArm;
   treatment: CheckpointArm;
+  /**
+  * Either arm carried a measurement artifact (an envelope refusal, an
+  * incomplete row, or a target that settled non-ok): the arms are not
+  * comparable, the cell can never pass, and criterion 1 fails
+  * (cycle 81). Without this, an envelope drained by the baseline left
+  * an EMPTY refused treatment arm (n 0, cost 0) that mechanically beat
+  * any baseline under the cheaper-at-equal-quality branch.
+  */
+  contaminated?: true;
   passed: boolean;
 }
 interface CriterionOneReport {
@@ -716,11 +748,15 @@ interface CriterionOneReport {
   pooledBaseline: CheckpointArm;
   pooledTreatment: CheckpointArm;
   pooledHolds: boolean;
+  /** Cells with a contaminated arm; present when nonzero (cycle 81). */
+  contaminatedCells?: number;
   passed: boolean;
 }
 interface CriterionTwoReport {
   baseline: CheckpointArm;
   informed: CheckpointArm;
+  /** Either arm carried a measurement artifact; the criterion cannot pass. */
+  contaminated?: true;
   passed: boolean;
 }
 interface CheckpointReport {

package/dist/index.js

@@ -760,8 +760,9 @@ function workflowWarningsOf(events) {
 /**
 * Runs the spec's repeats sequentially and reports the verified series.
 * Throws only for spec defects (invalid repeats, a throwing grader or
-* extractor) and for a target-run envelope refusal; everything a run
-* does wrong lands in its record instead.
+* extractor); everything a run does wrong lands in its record, and a
+* target-run envelope refusal ends the series monotonically with the
+* completed repeats preserved (report.refusal).
 */
 async function runBenchmark(engine, spec, options = {}) {
 	if (!Number.isInteger(spec.repeats) || spec.repeats < 1) throw new TypeError(`BenchmarkSpec.repeats must be a positive integer, got ${String(spec.repeats)}`);
@@ -769,8 +770,18 @@ async function runBenchmark(engine, spec, options = {}) {
 	let startedAt;
 	let totalCostUsd = 0;
 	let totalJudgeCostUsd = 0;
+	let refusal;
 	for (let ordinal = 1; ordinal <= spec.repeats; ordinal += 1) {
-		options.envelope?.authorize(options.budgetUsd, `benchmark '${spec.name}' run ${String(ordinal)}`);
+		try {
+			options.envelope?.authorize(options.budgetUsd, `benchmark '${spec.name}' run ${String(ordinal)}`);
+		} catch (error) {
+			if (!(error instanceof SweepBudgetError)) throw error;
+			refusal = {
+				atOrdinal: ordinal,
+				detail: error.message
+			};
+			break;
+		}
 		const live = await collectRun(engine.run(spec.workflow, spec.args, {
 			name: `benchmark:${spec.name}:${String(ordinal)}`,
 			...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }
@@ -886,6 +897,7 @@ async function runBenchmark(engine, spec, options = {}) {
 		metrics: metricSeries,
 		totalCostUsd,
 		judgeCostUsd: totalJudgeCostUsd,
+		...refusal === void 0 ? {} : { refusal },
 		fingerprint: {
 			node: process.version,
 			platform: process.platform,
@@ -958,6 +970,16 @@ function armOf(suite) {
 		n: suite.results.length
 	};
 }
+/**
+* A measurement artifact in an arm invalidates the A/B comparison: an
+* envelope refusal truncated the arm, a judge budget event forced rows
+* to fail, or a target settled non-ok (provider failure, ceiling, host
+* cancellation). Grader failures on ok targets are the HONEST failure
+* signal and never contaminate.
+*/
+function contaminatedArm(suite) {
+	return suite.refusal !== void 0 || suite.results.some((result) => result.status !== "ok" || result.incomplete !== void 0);
+}
 function pool(arms) {
 	const n = arms.reduce((sum, arm) => sum + arm.n, 0);
 	const passed = arms.reduce((sum, arm) => sum + arm.passRate * arm.n, 0);
@@ -987,9 +1009,15 @@ async function runValueCheckpoint(checkpointPool, options) {
 		const treatmentTier = recommendation?.recommendedTier ?? ladder.startTier;
 		const baseMember = ladder.rungs[ladder.startTier];
 		const treatMember = ladder.rungs[treatmentTier];
-		if (baseMember === void 0 || treatMember === void 0) throw new Error(`checkpoint: ladder '${ladder.name}' lacks rung ${String(treatmentTier)}`);
-		const baseline = armOf(await runEvalSuite(await options.engineFor(baseMember), cases, options.suite ?? {}));
-		const treatment = treatmentTier === ladder.startTier ? baseline : armOf(await runEvalSuite(await options.engineFor(treatMember), cases, options.suite ?? {}));
+		if (baseMember === void 0 || treatMember === void 0) {
+			const missing = baseMember === void 0 ? ladder.startTier : treatmentTier;
+			throw new Error(`checkpoint: ladder '${ladder.name}' lacks rung ${String(missing)}`);
+		}
+		const baselineSuite = await runEvalSuite(await options.engineFor(baseMember), cases, options.suite ?? {});
+		const treatmentSuite = treatmentTier === ladder.startTier ? baselineSuite : await runEvalSuite(await options.engineFor(treatMember), cases, options.suite ?? {});
+		const baseline = armOf(baselineSuite);
+		const treatment = armOf(treatmentSuite);
+		const contaminated = contaminatedArm(baselineSuite) || contaminatedArm(treatmentSuite);
 		cells.push({
 			ladder: ladder.name,
 			taskClass,
@@ -998,7 +1026,8 @@ async function runValueCheckpoint(checkpointPool, options) {
 			recommended: recommendation !== void 0,
 			baseline,
 			treatment,
-			passed: rungRuleHolds(baseline, treatment)
+			...contaminated ? { contaminated: true } : {},
+			passed: !contaminated && rungRuleHolds(baseline, treatment)
 		});
 	}
 	const recommendedCells = cells.filter((cell) => cell.recommended);
@@ -1007,6 +1036,7 @@ async function runValueCheckpoint(checkpointPool, options) {
 	const pooledBaseline = pool(cells.map((cell) => cell.baseline));
 	const pooledTreatment = pool(cells.map((cell) => cell.treatment));
 	const pooledHolds = rungRuleHolds(pooledBaseline, pooledTreatment);
+	const contaminatedCells = cells.filter((cell) => cell.contaminated === true).length;
 	const criterion1 = {
 		cells,
 		cellsPassed,
@@ -1014,18 +1044,23 @@ async function runValueCheckpoint(checkpointPool, options) {
 		pooledBaseline,
 		pooledTreatment,
 		pooledHolds,
-		passed: majorityHolds && pooledHolds
+		...contaminatedCells === 0 ? {} : { contaminatedCells },
+		passed: majorityHolds && pooledHolds && contaminatedCells === 0
 	};
 	let criterion2;
 	if (options.orchestrateEngineFor !== void 0 && options.orchestratedCases !== void 0) {
 		const cases = options.orchestratedCases.map((entry) => entry.case);
 		const orchestratedSuite = options.orchestratedSuite ?? options.suite ?? {};
-		const baseline = armOf(await runEvalSuite(await options.orchestrateEngineFor(false), cases, orchestratedSuite));
-		const informed = armOf(await runEvalSuite(await options.orchestrateEngineFor(true), cases, orchestratedSuite));
+		const baselineSuite = await runEvalSuite(await options.orchestrateEngineFor(false), cases, orchestratedSuite);
+		const informedSuite = await runEvalSuite(await options.orchestrateEngineFor(true), cases, orchestratedSuite);
+		const baseline = armOf(baselineSuite);
+		const informed = armOf(informedSuite);
+		const contaminated = contaminatedArm(baselineSuite) || contaminatedArm(informedSuite);
 		criterion2 = {
 			baseline,
 			informed,
-			passed: informed.n > 0 && informed.passRate > 0 && agentTypeRuleHolds(baseline, informed)
+			...contaminated ? { contaminated: true } : {},
+			passed: !contaminated && informed.n > 0 && informed.passRate > 0 && agentTypeRuleHolds(baseline, informed)
 		};
 	}
 	return {
@@ -1042,9 +1077,9 @@ function renderCheckpointReport(report) {
 	const lines = [
 		`Measured-value checkpoint (OQ-09) at ${report.observedAt}: ` + (report.passed ? "PASSED" : "FAILED"),
 		"",
-		`Criterion 1 (rung selection): ${report.criterion1.passed ? "holds" : "fails"} (${String(report.criterion1.cellsPassed)}/${String(report.criterion1.cells.length)} cells, pooled ${report.criterion1.pooledHolds ? "holds" : "fails"})`
+		`Criterion 1 (rung selection): ${report.criterion1.passed ? "holds" : "fails"} (${String(report.criterion1.cellsPassed)}/${String(report.criterion1.cells.filter((cell) => cell.recommended).length)} recommended cells, pooled ${report.criterion1.pooledHolds ? "holds" : "fails"})`
 	];
-	for (const cell of report.criterion1.cells) lines.push(`* ${cell.ladder} :: ${cell.taskClass}: baseline tier ${String(cell.defaultTier)} ${percent(cell.baseline.passRate)} at ${usd(cell.baseline.totalCostUsd)}; treatment tier ${String(cell.treatmentTier)}${cell.recommended ? "" : " (no recommendation)"} ${percent(cell.treatment.passRate)} at ${usd(cell.treatment.totalCostUsd)}; ${cell.passed ? "pass" : "fail"} (n=${String(cell.baseline.n)})`);
+	for (const cell of report.criterion1.cells) lines.push(`* ${cell.ladder} :: ${cell.taskClass}: baseline tier ${String(cell.defaultTier)} ${percent(cell.baseline.passRate)} at ${usd(cell.baseline.totalCostUsd)}; treatment tier ${String(cell.treatmentTier)}${cell.recommended ? "" : " (no recommendation)"} ${percent(cell.treatment.passRate)} at ${usd(cell.treatment.totalCostUsd)}; ${cell.passed ? "pass" : "fail"}${cell.contaminated === true ? " (contaminated)" : ""} (n=${String(cell.baseline.n)})`);
 	if (report.criterion2 !== void 0) {
 		const c2 = report.criterion2;
 		lines.push("", `Criterion 2 (agentType selection): ${c2.passed ? "holds" : "fails"} (baseline ${percent(c2.baseline.passRate)} at ${usd(c2.baseline.totalCostUsd)}; card-informed ${percent(c2.informed.passRate)} at ${usd(c2.informed.totalCostUsd)}; n=${String(c2.baseline.n)})`);
@@ -1108,6 +1143,7 @@ async function runSweepMatrix(pool, options) {
 				...options.envelope === void 0 ? {} : { envelope: options.envelope }
 			});
 			const exhaustedRuns = suite.results.filter((result) => result.status === "exhausted").length;
+			const nonOkRuns = suite.results.filter((result) => result.status !== "ok" && result.status !== "exhausted").length;
 			const judgeIncompleteRuns = suite.results.filter((result) => result.incomplete !== void 0).length;
 			const incompleteReason = suite.refusal !== void 0 ? "envelope-exhausted" : suite.results.find((result) => result.incomplete !== void 0)?.incomplete?.reason;
 			const cell = {
@@ -1120,6 +1156,7 @@ async function runSweepMatrix(pool, options) {
 				totalCostUsd: suite.totalCostUsd,
 				caseNames: suite.results.map((result) => result.name),
 				...exhaustedRuns === 0 ? {} : { exhaustedRuns },
+				...nonOkRuns === 0 ? {} : { nonOkRuns },
 				...judgeIncompleteRuns === 0 ? {} : { judgeIncompleteRuns },
 				...suite.refusal === void 0 ? {} : {
 					envelopeExhausted: true,
@@ -1129,7 +1166,7 @@ async function runSweepMatrix(pool, options) {
 			};
 			cells.push(cell);
 			const polarity = cell.passRate >= thresholds.strength ? "strength" : cell.passRate <= thresholds.weakness ? "weakness" : void 0;
-			const complete = cell.n === cell.plannedN && exhaustedRuns === 0 && judgeIncompleteRuns === 0 && suite.refusal === void 0;
+			const complete = cell.n === cell.plannedN && exhaustedRuns === 0 && nonOkRuns === 0 && judgeIncompleteRuns === 0 && suite.refusal === void 0;
 			if (polarity !== void 0 && cell.n > 0 && complete) {
 				const epoch = options.modelEpochFor?.(member);
 				claims.push({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.81.2",
+  "version": "1.83.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.81.2",
-    "@rulvar/testing": "1.81.2"
+    "@rulvar/testing": "1.83.0",
+    "@rulvar/core": "1.83.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",