@rulvar/evals 1.17.0 → 1.18.0

package/dist/index.d.ts

@@ -105,6 +105,20 @@ interface EvalCaseResult {
   /** The target run's normalized usage. */
   usage: Usage;
   error?: WireError;
+  /**
+  * Present when grading stopped for a BUDGET reason (v1.17.0 review
+  * P1-5): the judge run hit its own per-run ceiling
+  * ('judge-exhausted') or the aggregate envelope refused a judge run
+  * before it started ('judge-refused'). The paid target evidence and
+  * its cost stay on this row, but the case can never count as passed
+  * and its cell emits no claim. Unexpected grader errors still throw:
+  * a grader that cannot grade for non-budget reasons is a defect of
+  * the suite, not a budget event.
+  */
+  incomplete?: {
+    reason: "judge-exhausted" | "judge-refused";
+    detail: string;
+  };
 }
 interface RunEvalCaseOptions {
   /** Display-name override; defaults to the workflow name. */
@@ -125,7 +139,9 @@ interface RunEvalCaseOptions {
 declare class EvalJudgeError extends Error {
   readonly judgeRun: string;
   readonly status: RunOutcome<Json>["status"];
-  constructor(judgeRun: string, status: RunOutcome<Json>["status"], detail?: string);
+  /** What the failing judge run actually spent (honest cost accounting). */
+  readonly costUsd: number;
+  constructor(judgeRun: string, status: RunOutcome<Json>["status"], detail?: string, costUsd?: number);
 }
 /**
 * Runs one EvalCase on the given engine: the target workflow as its own
@@ -137,11 +153,27 @@ declare function runEvalCase(engine: Engine, evalCase: EvalCase, options?: RunEv
 /** Aggregate view of a suite run. */
 interface EvalSuiteResult {
   results: EvalCaseResult[];
-  /** Fraction of cases with passed true; 0 for an empty suite. */
+  /** Fraction of result rows with passed true; 0 for an empty suite. */
   passRate: number;
   totalCostUsd: number;
-  /** Arithmetic mean over cases; 0 for an empty suite. */
+  /** Arithmetic mean over result rows; 0 for an empty suite. */
   meanLatencyMs: number;
+  /** Cases the caller asked for. */
+  plannedN: number;
+  /** Result rows actually produced (equals results.length). */
+  completedN: number;
+  /**
+  * Present when the aggregate envelope refused a TARGET run before it
+  * started (v1.17.0 review P1-5). The suite stops there and returns
+  * everything already measured instead of throwing: completed rows,
+  * their costs, and their names survive. Judge refusals never appear
+  * here; they normalize into the owning row's `incomplete` marker.
+  */
+  refusal?: {
+    runLabel: string;
+    atCase: string;
+    detail: string;
+  };
 }
 interface RunEvalSuiteOptions {
   budgetUsd?: number;
@@ -294,13 +326,18 @@ interface CanaryReport {
   /**
   * True only when every probe settled ok. A fingerprint containing a
   * non-ok probe status is a measurement artifact (budget exhaustion,
-  * transient provider failure), NOT evidence of model drift: never
-  * feed it to flipStaleOnCanaryDrift.
+  * an envelope refusal, transient provider failure), NOT evidence of
+  * model drift: never feed it to flipStaleOnCanaryDrift.
   */
   allOk: boolean;
+  /**
+  * One row per probe; 'refused' means the aggregate envelope refused
+  * the probe before it started (v1.17.0 review P1-5): the loop keeps
+  * walking so completed probe evidence survives, and allOk is false.
+  */
   probes: Array<{
     prompt: string;
-    status: RunOutcome<unknown>["status"];
+    status: RunOutcome<unknown>["status"] | "refused";
   }>;
 }
 /** The committed v1 normalization (OQ-06): NFC, trim, collapse whitespace. */
@@ -310,10 +347,12 @@ declare function normalizeCanaryOutput(output: unknown): string;
 * sequentially in declaration order, one run per probe, so recordings
 * replay deterministically. Each probe run carries the optional
 * immutable ceiling (options.budgetUsd) and authorizes it against the
-* optional envelope before starting. A non-ok probe enters the
-* fingerprint as `!status` and clears allOk: callers gate drift
-* flipping on allOk, because a budget-starved or transiently failing
-* probe fingerprints differently without the model having drifted.
+* optional envelope before starting; an envelope refusal records the
+* probe as 'refused' and keeps walking instead of throwing away the
+* completed probes. A non-ok or refused probe enters the fingerprint
+* as `!status` and clears allOk: callers gate drift flipping on allOk,
+* because a budget-starved or transiently failing probe fingerprints
+* differently without the model having drifted.
 */
 declare function runCanary(engine: Engine, probes: CanaryProbeSet, options?: CanaryRunOptions): Promise<CanaryReport>;
 /**
@@ -390,9 +429,12 @@ interface RunSweepOptions {
   * ceiling before starting, so the pool times cases times judge-call
   * product cannot exceed it, falsification pool growth included. An
   * envelope requires suite.budgetUsd (and suite.judgeBudgetUsd once a
-  * grader judges); a cell refused by the envelope lands in the report
-  * as envelopeExhausted with NO claim. Share the instance with the
-  * canary loop so probes draw from the same remainder.
+  * grader judges). Refusals are monotone (v1.17.0 review P1-5): a
+  * refused target stops that cell's walk but everything already
+  * measured stays on the cell (n, costs, caseNames), judge refusals
+  * normalize into their row's incomplete marker, and an incomplete
+  * cell emits NO claim. Share the instance with the canary loop so
+  * probes draw from the same remainder.
   */
   envelope?: SpendEnvelope;
   /** When given, emitted claims commit through the committer identity. */
@@ -408,7 +450,14 @@ interface SweepCellReport {
   effort?: Effort;
   taskClass: TaskClass;
   passRate: number;
+  /** Result rows actually measured (completed count). */
   n: number;
+  /**
+  * Cases this cell was asked to measure (v1.17.0 review P1-5). A cell
+  * with n < plannedN is incomplete: what ran stays reported, and the
+  * cell emits no claim.
+  */
+  plannedN: number;
   totalCostUsd: number;
   caseNames: string[];
   /**
@@ -421,10 +470,21 @@ interface SweepCellReport {
   */
   exhaustedRuns?: number;
   /**
-  * The aggregate envelope refused a run of this cell before it
-  * started; stats cover nothing reliable and the cell emits no claim.
+  * Count of result rows whose grading stopped on a judge budget event
+  * (per-run judge ceiling or envelope refusal of a judge run). The
+  * paid target evidence stays on those rows; the cell emits no claim.
+  */
+  judgeIncompleteRuns?: number;
+  /**
+  * The aggregate envelope refused a TARGET run of this cell before it
+  * started; everything measured up to that point stays reported and
+  * the cell emits no claim.
   */
   envelopeExhausted?: true;
+  /** Why the cell is incomplete, when it is. */
+  incompleteReason?: "envelope-exhausted" | "judge-exhausted" | "judge-refused";
+  /** The refused run's label, when the envelope refused one. */
+  refusedRunLabel?: string;
 }
 interface SweepReport {
   reportId: string;

package/dist/index.js

@@ -1,5 +1,100 @@
 import { ConfigError, KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
 import { createHash } from "node:crypto";
+//#region src/envelope.ts
+/**
+* The debit-only aggregate spend envelope (v1.16.2 review P1-2). A
+* sweep multiplies paid runs: pool members times cases for targets,
+* one judge run per GraderContext.judge call, one canary run per
+* probe per member, and the falsification union can grow the pool
+* beyond the config. Per-run ceilings alone do not bound that
+* product, so the envelope authorizes each run's IMMUTABLE ceiling
+* BEFORE the run starts: a run whose ceiling does not fit the
+* remainder is refused before any provider work.
+*
+* Authorizations are never returned: not when a run completes under
+* its ceiling, not on CAS retries (they run no paid work), and not on
+* replay. A replayed run authorizes exactly like a fresh one and then
+* spends nothing; the envelope bounds the authorized worst case, not
+* the observed spend, so replay never double-PAYS anything while the
+* accounting stays one-directional. The envelope lives for one
+* invocation and is never persisted.
+*
+* Accounting is integer micro-USD and conservative at the
+* representation boundary (v1.17.0 review P1-4): the cap converts DOWN
+* (floor), every debit converts UP (ceil), and a cap below one
+* micro-USD is rejected outright, so for any admitted sequence the sum
+* of the ORIGINAL ceilings can never exceed maxTotalUsd and no
+* positive ceiling ever debits zero. Amounts that are integer
+* micro-USD up to float noise stay exact, so 0.1 + 0.2 against a 0.3
+* envelope is a fit, not a float rejection.
+*/
+const MICRO = 1e6;
+/**
+* Relative tolerance for float noise around an integer micro-USD
+* amount: 0.3 * 1e6 is 299999.99999999994 and MUST count as 300000,
+* while a genuinely sub-micro 0.4 must not.
+*/
+const MICRO_NOISE = 1e-6;
+function microOf(usd, direction) {
+	const raw = usd * MICRO;
+	const nearest = Math.round(raw);
+	if (Math.abs(raw - nearest) <= MICRO_NOISE * Math.max(1, Math.abs(nearest))) return nearest;
+	return direction === "floor" ? Math.floor(raw) : Math.ceil(raw);
+}
+/** Thrown when authorizing a run's ceiling would exceed the envelope. */
+var SweepBudgetError = class extends Error {
+	/** What was about to start, e.g. `eval target 'sweep-math'`. */
+	runLabel;
+	/** The per-run ceiling that did not fit. */
+	ceilingUsd;
+	/** Total already authorized before this refusal. */
+	authorizedUsd;
+	maxTotalUsd;
+	constructor(runLabel, ceilingUsd, authorizedUsd, maxTotalUsd) {
+		super(`sweep envelope exhausted: authorizing $${String(ceilingUsd)} for ${runLabel} would exceed maxTotalUsd $${String(maxTotalUsd)} ($${String(authorizedUsd)} already authorized); the run was refused before any provider call`);
+		this.name = "SweepBudgetError";
+		this.runLabel = runLabel;
+		this.ceilingUsd = ceilingUsd;
+		this.authorizedUsd = authorizedUsd;
+		this.maxTotalUsd = maxTotalUsd;
+	}
+};
+/**
+* One envelope bounds one whole sweep invocation: share the instance
+* across the canary loop and runSweepMatrix so canary, target, and
+* judge runs all draw from the same remainder.
+*/
+var SpendEnvelope = class {
+	maxTotalUsd;
+	maxMicroUsd;
+	authorizedMicroUsd = 0;
+	constructor(maxTotalUsd) {
+		if (!Number.isFinite(maxTotalUsd) || maxTotalUsd <= 0) throw new ConfigError(`SpendEnvelope maxTotalUsd must be a positive finite number, got ${String(maxTotalUsd)}`);
+		this.maxTotalUsd = maxTotalUsd;
+		this.maxMicroUsd = microOf(maxTotalUsd, "floor");
+		if (this.maxMicroUsd < 1) throw new ConfigError(`SpendEnvelope maxTotalUsd ${String(maxTotalUsd)} is below the 1 micro-USD accounting granularity (\$0.000001); such an envelope could never admit a run`);
+	}
+	/** Total authorized so far (debit-only; never decreases). */
+	get authorizedUsd() {
+		return this.authorizedMicroUsd / MICRO;
+	}
+	get remainingUsd() {
+		return Math.max(0, this.maxMicroUsd - this.authorizedMicroUsd) / MICRO;
+	}
+	/**
+	* Authorizes one run's immutable ceiling or throws SweepBudgetError.
+	* An unbounded run cannot be authorized: under an envelope every run
+	* MUST carry an explicit positive ceiling, otherwise the aggregate
+	* bound would be unaccountable.
+	*/
+	authorize(ceilingUsd, runLabel) {
+		if (ceilingUsd === void 0 || !Number.isFinite(ceilingUsd) || ceilingUsd <= 0) throw new ConfigError(`the spend envelope requires an explicit positive per-run ceiling for ${runLabel}; got ${String(ceilingUsd)} (an unbounded run under an aggregate envelope would be unaccountable)`);
+		const micro = Math.max(1, microOf(ceilingUsd, "ceil"));
+		if (this.authorizedMicroUsd + micro > this.maxMicroUsd) throw new SweepBudgetError(runLabel, ceilingUsd, this.authorizedUsd, this.maxTotalUsd);
+		this.authorizedMicroUsd += micro;
+	}
+};
+//#endregion
 //#region src/case.ts
 /**
 * @rulvar/evals (M9-T02): EvalCase, the grader contract, and the case and
@@ -16,11 +111,14 @@ import { createHash } from "node:crypto";
 var EvalJudgeError = class extends Error {
 	judgeRun;
 	status;
-	constructor(judgeRun, status, detail) {
+	/** What the failing judge run actually spent (honest cost accounting). */
+	costUsd;
+	constructor(judgeRun, status, detail, costUsd = 0) {
 		super(`eval judge run '${judgeRun}' settled '${status}'${detail === void 0 ? "" : `: ${detail}`}`);
 		this.name = "EvalJudgeError";
 		this.judgeRun = judgeRun;
 		this.status = status;
+		this.costUsd = costUsd;
 	}
 };
 /**
@@ -61,18 +159,39 @@ async function runEvalCase(engine, evalCase, options = {}) {
 		}
 	};
 	const verdicts = [];
-	for (const grader of evalCase.graders) verdicts.push(await grader.grade(context));
+	let incomplete;
+	for (const grader of evalCase.graders) try {
+		verdicts.push(await grader.grade(context));
+	} catch (error) {
+		if (error instanceof SweepBudgetError) {
+			incomplete = {
+				reason: "judge-refused",
+				detail: error.message
+			};
+			break;
+		}
+		if (error instanceof EvalJudgeError && error.status === "exhausted") {
+			judgeCostUsd += error.costUsd;
+			incomplete = {
+				reason: "judge-exhausted",
+				detail: error.message
+			};
+			break;
+		}
+		throw error;
+	}
 	const latencyMs = timing.start !== void 0 && timing.end !== void 0 ? Math.max(0, Date.parse(timing.end) - Date.parse(timing.start)) : 0;
 	return {
 		name,
 		status: outcome.status,
-		passed: outcome.status === "ok" && verdicts.every((verdict) => verdict.passed),
+		passed: outcome.status === "ok" && incomplete === void 0 && verdicts.every((verdict) => verdict.passed),
 		verdicts,
 		costUsd: outcome.cost.totalUsd + judgeCostUsd,
 		judgeCostUsd,
 		latencyMs,
 		usage: outcome.usage,
-		...outcome.error === void 0 ? {} : { error: outcome.error }
+		...outcome.error === void 0 ? {} : { error: outcome.error },
+		...incomplete === void 0 ? {} : { incomplete }
 	};
 }
 async function runJudge(engine, judgeName, spec, budgetUsd) {
@@ -89,7 +208,7 @@ async function runJudge(engine, judgeName, spec, budgetUsd) {
 		name: workflowName,
 		...budgetUsd === void 0 ? {} : { budgetUsd }
 	}).result;
-	if (outcome.status !== "ok") throw new EvalJudgeError(workflowName, outcome.status, outcome.error?.message);
+	if (outcome.status !== "ok") throw new EvalJudgeError(workflowName, outcome.status, outcome.error?.message, outcome.cost.totalUsd);
 	return {
 		output: outcome.value ?? null,
 		costUsd: outcome.cost.totalUsd
@@ -103,23 +222,37 @@ async function runJudge(engine, judgeName, spec, budgetUsd) {
 async function runEvalSuite(engine, cases, options = {}) {
 	const seen = /* @__PURE__ */ new Map();
 	const results = [];
+	let refusal;
 	for (const evalCase of cases) {
 		const base = evalCase.workflow.name;
 		const ordinal = seen.get(base) ?? 0;
 		seen.set(base, ordinal + 1);
 		const name = ordinal === 0 ? base : `${base}#${ordinal}`;
-		results.push(await runEvalCase(engine, evalCase, {
-			name,
-			...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd },
-			...options.judgeBudgetUsd === void 0 ? {} : { judgeBudgetUsd: options.judgeBudgetUsd },
-			...options.envelope === void 0 ? {} : { envelope: options.envelope }
-		}));
+		try {
+			results.push(await runEvalCase(engine, evalCase, {
+				name,
+				...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd },
+				...options.judgeBudgetUsd === void 0 ? {} : { judgeBudgetUsd: options.judgeBudgetUsd },
+				...options.envelope === void 0 ? {} : { envelope: options.envelope }
+			}));
+		} catch (error) {
+			if (!(error instanceof SweepBudgetError)) throw error;
+			refusal = {
+				runLabel: error.runLabel,
+				atCase: name,
+				detail: error.message
+			};
+			break;
+		}
 	}
 	return {
 		results,
 		passRate: results.length === 0 ? 0 : results.filter((r) => r.passed).length / results.length,
 		totalCostUsd: results.reduce((sum, r) => sum + r.costUsd, 0),
-		meanLatencyMs: results.length === 0 ? 0 : results.reduce((sum, r) => sum + r.latencyMs, 0) / results.length
+		meanLatencyMs: results.length === 0 ? 0 : results.reduce((sum, r) => sum + r.latencyMs, 0) / results.length,
+		plannedN: cases.length,
+		completedN: results.length,
+		...refusal === void 0 ? {} : { refusal }
 	};
 }
 //#endregion
@@ -344,16 +477,28 @@ function normalizeCanaryOutput(output) {
 * sequentially in declaration order, one run per probe, so recordings
 * replay deterministically. Each probe run carries the optional
 * immutable ceiling (options.budgetUsd) and authorizes it against the
-* optional envelope before starting. A non-ok probe enters the
-* fingerprint as `!status` and clears allOk: callers gate drift
-* flipping on allOk, because a budget-starved or transiently failing
-* probe fingerprints differently without the model having drifted.
+* optional envelope before starting; an envelope refusal records the
+* probe as 'refused' and keeps walking instead of throwing away the
+* completed probes. A non-ok or refused probe enters the fingerprint
+* as `!status` and clears allOk: callers gate drift flipping on allOk,
+* because a budget-starved or transiently failing probe fingerprints
+* differently without the model having drifted.
 */
 async function runCanary(engine, probes, options = {}) {
 	const outputs = [];
 	const probeReports = [];
 	for (const [index, prompt] of probes.prompts.entries()) {
-		options.envelope?.authorize(options.budgetUsd, `canary probe ${String(index)}`);
+		try {
+			options.envelope?.authorize(options.budgetUsd, `canary probe ${String(index)}`);
+		} catch (error) {
+			if (!(error instanceof SweepBudgetError)) throw error;
+			probeReports.push({
+				prompt,
+				status: "refused"
+			});
+			outputs.push("!refused");
+			continue;
+		}
 		const workflow = defineWorkflow({ name: `kb-canary:${String(index)}` }, async (ctx) => await ctx.agent(prompt, { agentType: probes.agentType }));
 		const outcome = await engine.run(workflow, null, options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }).result;
 		probeReports.push({
@@ -424,82 +569,6 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 	throw lastCas ?? /* @__PURE__ */ new Error("flipStaleOnCanaryDrift: unreachable");
 }
 //#endregion
-//#region src/envelope.ts
-/**
-* The debit-only aggregate spend envelope (v1.16.2 review P1-2). A
-* sweep multiplies paid runs: pool members times cases for targets,
-* one judge run per GraderContext.judge call, one canary run per
-* probe per member, and the falsification union can grow the pool
-* beyond the config. Per-run ceilings alone do not bound that
-* product, so the envelope authorizes each run's IMMUTABLE ceiling
-* BEFORE the run starts: a run whose ceiling does not fit the
-* remainder is refused before any provider work.
-*
-* Authorizations are never returned: not when a run completes under
-* its ceiling, not on CAS retries (they run no paid work), and not on
-* replay. A replayed run authorizes exactly like a fresh one and then
-* spends nothing; the envelope bounds the authorized worst case, not
-* the observed spend, so replay never double-PAYS anything while the
-* accounting stays one-directional. The envelope lives for one
-* invocation and is never persisted.
-*
-* Accounting is integer micro-USD, so exact fits pass: 0.1 + 0.2
-* against a 0.3 envelope is a fit, not a float rejection.
-*/
-const MICRO = 1e6;
-/** Thrown when authorizing a run's ceiling would exceed the envelope. */
-var SweepBudgetError = class extends Error {
-	/** What was about to start, e.g. `eval target 'sweep-math'`. */
-	runLabel;
-	/** The per-run ceiling that did not fit. */
-	ceilingUsd;
-	/** Total already authorized before this refusal. */
-	authorizedUsd;
-	maxTotalUsd;
-	constructor(runLabel, ceilingUsd, authorizedUsd, maxTotalUsd) {
-		super(`sweep envelope exhausted: authorizing $${String(ceilingUsd)} for ${runLabel} would exceed maxTotalUsd $${String(maxTotalUsd)} ($${String(authorizedUsd)} already authorized); the run was refused before any provider call`);
-		this.name = "SweepBudgetError";
-		this.runLabel = runLabel;
-		this.ceilingUsd = ceilingUsd;
-		this.authorizedUsd = authorizedUsd;
-		this.maxTotalUsd = maxTotalUsd;
-	}
-};
-/**
-* One envelope bounds one whole sweep invocation: share the instance
-* across the canary loop and runSweepMatrix so canary, target, and
-* judge runs all draw from the same remainder.
-*/
-var SpendEnvelope = class {
-	maxTotalUsd;
-	maxMicroUsd;
-	authorizedMicroUsd = 0;
-	constructor(maxTotalUsd) {
-		if (!Number.isFinite(maxTotalUsd) || maxTotalUsd <= 0) throw new ConfigError(`SpendEnvelope maxTotalUsd must be a positive finite number, got ${String(maxTotalUsd)}`);
-		this.maxTotalUsd = maxTotalUsd;
-		this.maxMicroUsd = Math.round(maxTotalUsd * MICRO);
-	}
-	/** Total authorized so far (debit-only; never decreases). */
-	get authorizedUsd() {
-		return this.authorizedMicroUsd / MICRO;
-	}
-	get remainingUsd() {
-		return Math.max(0, this.maxMicroUsd - this.authorizedMicroUsd) / MICRO;
-	}
-	/**
-	* Authorizes one run's immutable ceiling or throws SweepBudgetError.
-	* An unbounded run cannot be authorized: under an envelope every run
-	* MUST carry an explicit positive ceiling, otherwise the aggregate
-	* bound would be unaccountable.
-	*/
-	authorize(ceilingUsd, runLabel) {
-		if (ceilingUsd === void 0 || !Number.isFinite(ceilingUsd) || ceilingUsd <= 0) throw new ConfigError(`the spend envelope requires an explicit positive per-run ceiling for ${runLabel}; got ${String(ceilingUsd)} (an unbounded run under an aggregate envelope would be unaccountable)`);
-		const micro = Math.round(ceilingUsd * MICRO);
-		if (this.authorizedMicroUsd + micro > this.maxMicroUsd) throw new SweepBudgetError(runLabel, ceilingUsd, this.authorizedUsd, this.maxTotalUsd);
-		this.authorizedMicroUsd += micro;
-	}
-};
-//#endregion
 //#region src/checkpoint.ts
 /**
 * The phases 1-2 measured-value checkpoint (M12-T01; the quantitative
@@ -704,40 +773,34 @@ async function runSweepMatrix(pool, options) {
 	for (const member of pool.models) {
 		const engine = await options.engineFor(member);
 		for (const [taskClass, bucket] of byTaskClass) {
-			let suite;
-			try {
-				suite = await runEvalSuite(engine, bucket.map((entry) => entry.case), {
-					...options.suite ?? {},
-					...options.envelope === void 0 ? {} : { envelope: options.envelope }
-				});
-			} catch (error) {
-				if (!(error instanceof SweepBudgetError)) throw error;
-				cells.push({
-					model: member.model,
-					...member.effort === void 0 ? {} : { effort: member.effort },
-					taskClass,
-					passRate: 0,
-					n: 0,
-					totalCostUsd: 0,
-					caseNames: [],
-					envelopeExhausted: true
-				});
-				continue;
-			}
+			const suite = await runEvalSuite(engine, bucket.map((entry) => entry.case), {
+				...options.suite ?? {},
+				...options.envelope === void 0 ? {} : { envelope: options.envelope }
+			});
 			const exhaustedRuns = suite.results.filter((result) => result.status === "exhausted").length;
+			const judgeIncompleteRuns = suite.results.filter((result) => result.incomplete !== void 0).length;
+			const incompleteReason = suite.refusal !== void 0 ? "envelope-exhausted" : suite.results.find((result) => result.incomplete !== void 0)?.incomplete?.reason;
 			const cell = {
 				model: member.model,
 				...member.effort === void 0 ? {} : { effort: member.effort },
 				taskClass,
 				passRate: suite.passRate,
-				n: suite.results.length,
+				n: suite.completedN,
+				plannedN: suite.plannedN,
 				totalCostUsd: suite.totalCostUsd,
 				caseNames: suite.results.map((result) => result.name),
-				...exhaustedRuns === 0 ? {} : { exhaustedRuns }
+				...exhaustedRuns === 0 ? {} : { exhaustedRuns },
+				...judgeIncompleteRuns === 0 ? {} : { judgeIncompleteRuns },
+				...suite.refusal === void 0 ? {} : {
+					envelopeExhausted: true,
+					refusedRunLabel: suite.refusal.runLabel
+				},
+				...incompleteReason === void 0 ? {} : { incompleteReason }
 			};
 			cells.push(cell);
 			const polarity = cell.passRate >= thresholds.strength ? "strength" : cell.passRate <= thresholds.weakness ? "weakness" : void 0;
-			if (polarity !== void 0 && cell.n > 0 && exhaustedRuns === 0) {
+			const complete = cell.n === cell.plannedN && exhaustedRuns === 0 && judgeIncompleteRuns === 0 && suite.refusal === void 0;
+			if (polarity !== void 0 && cell.n > 0 && complete) {
 				const epoch = options.modelEpochFor?.(member);
 				claims.push({
 					id: claimIdOf(options.reportId, member, taskClass),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.17.0",
+  "version": "1.18.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.17.0",
-    "@rulvar/testing": "1.17.0"
+    "@rulvar/core": "1.18.0",
+    "@rulvar/testing": "1.18.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",