@rulvar/evals 1.16.2 → 1.18.0

package/dist/index.d.ts

@@ -1,5 +1,39 @@
 import { CompiledWorkflow, DeclaredLadder, Effort, Engine, EvidenceRef, Json, JsonSchema, KnowledgeSnapshot, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow } from "@rulvar/core";
 
+//#region src/envelope.d.ts
+/** Thrown when authorizing a run's ceiling would exceed the envelope. */
+declare class SweepBudgetError extends Error {
+  /** What was about to start, e.g. `eval target 'sweep-math'`. */
+  readonly runLabel: string;
+  /** The per-run ceiling that did not fit. */
+  readonly ceilingUsd: number;
+  /** Total already authorized before this refusal. */
+  readonly authorizedUsd: number;
+  readonly maxTotalUsd: number;
+  constructor(runLabel: string, ceilingUsd: number, authorizedUsd: number, maxTotalUsd: number);
+}
+/**
+* One envelope bounds one whole sweep invocation: share the instance
+* across the canary loop and runSweepMatrix so canary, target, and
+* judge runs all draw from the same remainder.
+*/
+declare class SpendEnvelope {
+  readonly maxTotalUsd: number;
+  private readonly maxMicroUsd;
+  private authorizedMicroUsd;
+  constructor(maxTotalUsd: number);
+  /** Total authorized so far (debit-only; never decreases). */
+  get authorizedUsd(): number;
+  get remainingUsd(): number;
+  /**
+  * Authorizes one run's immutable ceiling or throws SweepBudgetError.
+  * An unbounded run cannot be authorized: under an envelope every run
+  * MUST carry an explicit positive ceiling, otherwise the aggregate
+  * bound would be unaccountable.
+  */
+  authorize(ceilingUsd: number | undefined, runLabel: string): void;
+}
+//#endregion
 //#region src/case.d.ts
 /**
 * One quality-measurement case. The shape is the
@@ -71,6 +105,20 @@ interface EvalCaseResult {
   /** The target run's normalized usage. */
   usage: Usage;
   error?: WireError;
+  /**
+  * Present when grading stopped for a BUDGET reason (v1.17.0 review
+  * P1-5): the judge run hit its own per-run ceiling
+  * ('judge-exhausted') or the aggregate envelope refused a judge run
+  * before it started ('judge-refused'). The paid target evidence and
+  * its cost stay on this row, but the case can never count as passed
+  * and its cell emits no claim. Unexpected grader errors still throw:
+  * a grader that cannot grade for non-budget reasons is a defect of
+  * the suite, not a budget event.
+  */
+  incomplete?: {
+    reason: "judge-exhausted" | "judge-refused";
+    detail: string;
+  };
 }
 interface RunEvalCaseOptions {
   /** Display-name override; defaults to the workflow name. */
@@ -79,12 +127,21 @@ interface RunEvalCaseOptions {
   budgetUsd?: number;
   /** Run ceiling for each judge run. */
   judgeBudgetUsd?: number;
+  /**
+  * Aggregate debit-only envelope (v1.16.2 review P1-2): every target
+  * and judge run authorizes its ceiling here BEFORE starting, and an
+  * envelope requires the matching per-run ceiling to be set. A
+  * refusal throws SweepBudgetError before any provider work.
+  */
+  envelope?: SpendEnvelope;
 }
 /** Thrown when a judge run does not settle ok. */
 declare class EvalJudgeError extends Error {
   readonly judgeRun: string;
   readonly status: RunOutcome<Json>["status"];
-  constructor(judgeRun: string, status: RunOutcome<Json>["status"], detail?: string);
+  /** What the failing judge run actually spent (honest cost accounting). */
+  readonly costUsd: number;
+  constructor(judgeRun: string, status: RunOutcome<Json>["status"], detail?: string, costUsd?: number);
 }
 /**
 * Runs one EvalCase on the given engine: the target workflow as its own
@@ -96,15 +153,33 @@ declare function runEvalCase(engine: Engine, evalCase: EvalCase, options?: RunEv
 /** Aggregate view of a suite run. */
 interface EvalSuiteResult {
   results: EvalCaseResult[];
-  /** Fraction of cases with passed true; 0 for an empty suite. */
+  /** Fraction of result rows with passed true; 0 for an empty suite. */
   passRate: number;
   totalCostUsd: number;
-  /** Arithmetic mean over cases; 0 for an empty suite. */
+  /** Arithmetic mean over result rows; 0 for an empty suite. */
   meanLatencyMs: number;
+  /** Cases the caller asked for. */
+  plannedN: number;
+  /** Result rows actually produced (equals results.length). */
+  completedN: number;
+  /**
+  * Present when the aggregate envelope refused a TARGET run before it
+  * started (v1.17.0 review P1-5). The suite stops there and returns
+  * everything already measured instead of throwing: completed rows,
+  * their costs, and their names survive. Judge refusals never appear
+  * here; they normalize into the owning row's `incomplete` marker.
+  */
+  refusal?: {
+    runLabel: string;
+    atCase: string;
+    detail: string;
+  };
 }
 interface RunEvalSuiteOptions {
   budgetUsd?: number;
   judgeBudgetUsd?: number;
+  /** See RunEvalCaseOptions.envelope; shared across every case of the suite. */
+  envelope?: SpendEnvelope;
 }
 /**
 * Runs cases sequentially (deterministic journal and cassette order) and
@@ -232,14 +307,59 @@ interface CanaryProbeSet {
   /** The fixed prompts; order matters and enters the fingerprint. */
   prompts: string[];
 }
+interface CanaryRunOptions {
+  /**
+  * Immutable ceiling per probe run (v1.16.2 review P1-2): every probe
+  * is an ordinary paid engine run and gets its own recorded
+  * RunMeta.budgetUsd.
+  */
+  budgetUsd?: number;
+  /**
+  * Aggregate debit-only envelope shared with the surrounding sweep;
+  * each probe authorizes budgetUsd BEFORE running, and an envelope
+  * requires budgetUsd to be set.
+  */
+  envelope?: SpendEnvelope;
+}
+interface CanaryReport {
+  fingerprint: string;
+  /**
+  * True only when every probe settled ok. A fingerprint containing a
+  * non-ok probe status is a measurement artifact (budget exhaustion,
+  * an envelope refusal, transient provider failure), NOT evidence of
+  * model drift: never feed it to flipStaleOnCanaryDrift.
+  */
+  allOk: boolean;
+  /**
+  * One row per probe; 'refused' means the aggregate envelope refused
+  * the probe before it started (v1.17.0 review P1-5): the loop keeps
+  * walking so completed probe evidence survives, and allOk is false.
+  */
+  probes: Array<{
+    prompt: string;
+    status: RunOutcome<unknown>["status"] | "refused";
+  }>;
+}
 /** The committed v1 normalization (OQ-06): NFC, trim, collapse whitespace. */
 declare function normalizeCanaryOutput(output: unknown): string;
 /**
-* Runs the fixed probe set through the ordinary engine and returns the
-* fingerprint. Probes run sequentially in declaration order, one run
-* per probe, so recordings replay deterministically.
+* Runs the fixed probe set through the ordinary engine. Probes run
+* sequentially in declaration order, one run per probe, so recordings
+* replay deterministically. Each probe run carries the optional
+* immutable ceiling (options.budgetUsd) and authorizes it against the
+* optional envelope before starting; an envelope refusal records the
+* probe as 'refused' and keeps walking instead of throwing away the
+* completed probes. A non-ok or refused probe enters the fingerprint
+* as `!status` and clears allOk: callers gate drift flipping on allOk,
+* because a budget-starved or transiently failing probe fingerprints
+* differently without the model having drifted.
 */
-declare function canaryFingerprint(engine: Engine, probes: CanaryProbeSet): Promise<string>;
+declare function runCanary(engine: Engine, probes: CanaryProbeSet, options?: CanaryRunOptions): Promise<CanaryReport>;
+/**
+* The fingerprint alone (the pre-v1.16.2-review surface, kept
+* compatible). Prefer runCanary: its allOk is the drift-flip gate.
+*/
+declare function canaryFingerprint(engine: Engine, probes: CanaryProbeSet, options?: CanaryRunOptions): Promise<string>;
 interface CanaryDriftReport {
   model: ModelRef;
   freshFingerprint: string;
@@ -253,7 +373,13 @@ interface CanaryDriftReport {
 * recorded canary fingerprint differs from the fresh one. Claims
 * without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
-* an idempotent noop. CAS-rebased like every maintenance commit.
+* an idempotent noop. CAS-rebased like every maintenance commit; the
+* retries run no engine work and pay nothing.
+*
+* Only pass fingerprints from an allOk probe set (runCanary): a
+* fingerprint containing a `!status` probe differs from any healthy
+* baseline by construction, and flipping on it would blame the model
+* for a budget ceiling or a transient provider failure.
 */
 declare function flipStaleOnCanaryDrift(store: ModelKnowledgeStore, model: ModelRef, freshFingerprint: string, options?: {
   attempts?: number;
@@ -297,6 +423,20 @@ interface RunSweepOptions {
   thresholds?: Partial<SweepThresholds>;
   /** Passed through to every suite run (budget, VCR hooks ride the engine). */
   suite?: RunEvalSuiteOptions;
+  /**
+  * Aggregate debit-only envelope over the WHOLE matrix (v1.16.2
+  * review P1-2): every target and judge run authorizes its immutable
+  * ceiling before starting, so the pool times cases times judge-call
+  * product cannot exceed it, falsification pool growth included. An
+  * envelope requires suite.budgetUsd (and suite.judgeBudgetUsd once a
+  * grader judges). Refusals are monotone (v1.17.0 review P1-5): a
+  * refused target stops that cell's walk but everything already
+  * measured stays on the cell (n, costs, caseNames), judge refusals
+  * normalize into their row's incomplete marker, and an incomplete
+  * cell emits NO claim. Share the instance with the canary loop so
+  * probes draw from the same remainder.
+  */
+  envelope?: SpendEnvelope;
   /** When given, emitted claims commit through the committer identity. */
   store?: ModelKnowledgeStore;
   /**
@@ -310,9 +450,41 @@ interface SweepCellReport {
   effort?: Effort;
   taskClass: TaskClass;
   passRate: number;
+  /** Result rows actually measured (completed count). */
   n: number;
+  /**
+  * Cases this cell was asked to measure (v1.17.0 review P1-5). A cell
+  * with n < plannedN is incomplete: what ran stays reported, and the
+  * cell emits no claim.
+  */
+  plannedN: number;
   totalCostUsd: number;
   caseNames: string[];
+  /**
+  * Count of case results whose TARGET run settled 'exhausted' (its
+  * per-run ceiling, not the envelope). A budget-starved measurement
+  * must not become a model belief, so any exhausted target suppresses
+  * the cell's claim even when the degraded passRate crosses a
+  * threshold: the alternative is committing a false weakness that
+  * blames the model for the ceiling.
+  */
+  exhaustedRuns?: number;
+  /**
+  * Count of result rows whose grading stopped on a judge budget event
+  * (per-run judge ceiling or envelope refusal of a judge run). The
+  * paid target evidence stays on those rows; the cell emits no claim.
+  */
+  judgeIncompleteRuns?: number;
+  /**
+  * The aggregate envelope refused a TARGET run of this cell before it
+  * started; everything measured up to that point stays reported and
+  * the cell emits no claim.
+  */
+  envelopeExhausted?: true;
+  /** Why the cell is incomplete, when it is. */
+  incompleteReason?: "envelope-exhausted" | "judge-exhausted" | "judge-refused";
+  /** The refused run's label, when the envelope refused one. */
+  refusedRunLabel?: string;
 }
 interface SweepReport {
   reportId: string;
@@ -417,4 +589,4 @@ declare function runValueCheckpoint(checkpointPool: CheckpointPool, options: Run
 /** The deterministic render for the M12 gate docs amendment. */
 declare function renderCheckpointReport(report: CheckpointReport): string;
 //#endregion
-export { type CanaryDriftReport, type CanaryProbeSet, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+export { type CanaryDriftReport, type CanaryProbeSet, type CanaryReport, type CanaryRunOptions, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/dist/index.js

@@ -1,5 +1,100 @@
-import { KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
+import { ConfigError, KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
 import { createHash } from "node:crypto";
+//#region src/envelope.ts
+/**
+* The debit-only aggregate spend envelope (v1.16.2 review P1-2). A
+* sweep multiplies paid runs: pool members times cases for targets,
+* one judge run per GraderContext.judge call, one canary run per
+* probe per member, and the falsification union can grow the pool
+* beyond the config. Per-run ceilings alone do not bound that
+* product, so the envelope authorizes each run's IMMUTABLE ceiling
+* BEFORE the run starts: a run whose ceiling does not fit the
+* remainder is refused before any provider work.
+*
+* Authorizations are never returned: not when a run completes under
+* its ceiling, not on CAS retries (they run no paid work), and not on
+* replay. A replayed run authorizes exactly like a fresh one and then
+* spends nothing; the envelope bounds the authorized worst case, not
+* the observed spend, so replay never double-PAYS anything while the
+* accounting stays one-directional. The envelope lives for one
+* invocation and is never persisted.
+*
+* Accounting is integer micro-USD and conservative at the
+* representation boundary (v1.17.0 review P1-4): the cap converts DOWN
+* (floor), every debit converts UP (ceil), and a cap below one
+* micro-USD is rejected outright, so for any admitted sequence the sum
+* of the ORIGINAL ceilings can never exceed maxTotalUsd and no
+* positive ceiling ever debits zero. Amounts that are integer
+* micro-USD up to float noise stay exact, so 0.1 + 0.2 against a 0.3
+* envelope is a fit, not a float rejection.
+*/
+const MICRO = 1e6;
+/**
+* Relative tolerance for float noise around an integer micro-USD
+* amount: 0.3 * 1e6 is 299999.99999999994 and MUST count as 300000,
+* while a genuinely sub-micro 0.4 must not.
+*/
+const MICRO_NOISE = 1e-6;
+function microOf(usd, direction) {
+	const raw = usd * MICRO;
+	const nearest = Math.round(raw);
+	if (Math.abs(raw - nearest) <= MICRO_NOISE * Math.max(1, Math.abs(nearest))) return nearest;
+	return direction === "floor" ? Math.floor(raw) : Math.ceil(raw);
+}
+/** Thrown when authorizing a run's ceiling would exceed the envelope. */
+var SweepBudgetError = class extends Error {
+	/** What was about to start, e.g. `eval target 'sweep-math'`. */
+	runLabel;
+	/** The per-run ceiling that did not fit. */
+	ceilingUsd;
+	/** Total already authorized before this refusal. */
+	authorizedUsd;
+	maxTotalUsd;
+	constructor(runLabel, ceilingUsd, authorizedUsd, maxTotalUsd) {
+		super(`sweep envelope exhausted: authorizing $${String(ceilingUsd)} for ${runLabel} would exceed maxTotalUsd $${String(maxTotalUsd)} ($${String(authorizedUsd)} already authorized); the run was refused before any provider call`);
+		this.name = "SweepBudgetError";
+		this.runLabel = runLabel;
+		this.ceilingUsd = ceilingUsd;
+		this.authorizedUsd = authorizedUsd;
+		this.maxTotalUsd = maxTotalUsd;
+	}
+};
+/**
+* One envelope bounds one whole sweep invocation: share the instance
+* across the canary loop and runSweepMatrix so canary, target, and
+* judge runs all draw from the same remainder.
+*/
+var SpendEnvelope = class {
+	maxTotalUsd;
+	maxMicroUsd;
+	authorizedMicroUsd = 0;
+	constructor(maxTotalUsd) {
+		if (!Number.isFinite(maxTotalUsd) || maxTotalUsd <= 0) throw new ConfigError(`SpendEnvelope maxTotalUsd must be a positive finite number, got ${String(maxTotalUsd)}`);
+		this.maxTotalUsd = maxTotalUsd;
+		this.maxMicroUsd = microOf(maxTotalUsd, "floor");
+		if (this.maxMicroUsd < 1) throw new ConfigError(`SpendEnvelope maxTotalUsd ${String(maxTotalUsd)} is below the 1 micro-USD accounting granularity (\$0.000001); such an envelope could never admit a run`);
+	}
+	/** Total authorized so far (debit-only; never decreases). */
+	get authorizedUsd() {
+		return this.authorizedMicroUsd / MICRO;
+	}
+	get remainingUsd() {
+		return Math.max(0, this.maxMicroUsd - this.authorizedMicroUsd) / MICRO;
+	}
+	/**
+	* Authorizes one run's immutable ceiling or throws SweepBudgetError.
+	* An unbounded run cannot be authorized: under an envelope every run
+	* MUST carry an explicit positive ceiling, otherwise the aggregate
+	* bound would be unaccountable.
+	*/
+	authorize(ceilingUsd, runLabel) {
+		if (ceilingUsd === void 0 || !Number.isFinite(ceilingUsd) || ceilingUsd <= 0) throw new ConfigError(`the spend envelope requires an explicit positive per-run ceiling for ${runLabel}; got ${String(ceilingUsd)} (an unbounded run under an aggregate envelope would be unaccountable)`);
+		const micro = Math.max(1, microOf(ceilingUsd, "ceil"));
+		if (this.authorizedMicroUsd + micro > this.maxMicroUsd) throw new SweepBudgetError(runLabel, ceilingUsd, this.authorizedUsd, this.maxTotalUsd);
+		this.authorizedMicroUsd += micro;
+	}
+};
+//#endregion
 //#region src/case.ts
 /**
 * @rulvar/evals (M9-T02): EvalCase, the grader contract, and the case and
@@ -16,11 +111,14 @@ import { createHash } from "node:crypto";
 var EvalJudgeError = class extends Error {
 	judgeRun;
 	status;
-	constructor(judgeRun, status, detail) {
+	/** What the failing judge run actually spent (honest cost accounting). */
+	costUsd;
+	constructor(judgeRun, status, detail, costUsd = 0) {
 		super(`eval judge run '${judgeRun}' settled '${status}'${detail === void 0 ? "" : `: ${detail}`}`);
 		this.name = "EvalJudgeError";
 		this.judgeRun = judgeRun;
 		this.status = status;
+		this.costUsd = costUsd;
 	}
 };
 /**
@@ -32,6 +130,7 @@ var EvalJudgeError = class extends Error {
 async function runEvalCase(engine, evalCase, options = {}) {
 	const name = options.name ?? evalCase.workflow.name;
 	const timing = {};
+	options.envelope?.authorize(options.budgetUsd, `eval target '${name}'`);
 	const handle = engine.run(evalCase.workflow, evalCase.args, {
 		name: `eval:${name}`,
 		...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }
@@ -53,24 +152,46 @@ async function runEvalCase(engine, evalCase, options = {}) {
 		async judge(spec) {
 			const ordinal = judgeOrdinal;
 			judgeOrdinal += 1;
+			options.envelope?.authorize(options.judgeBudgetUsd, `eval judge '${name}:${String(ordinal)}'`);
 			const judged = await runJudge(engine, `${name}:${ordinal}`, spec, options.judgeBudgetUsd);
 			judgeCostUsd += judged.costUsd;
 			return judged.output;
 		}
 	};
 	const verdicts = [];
-	for (const grader of evalCase.graders) verdicts.push(await grader.grade(context));
+	let incomplete;
+	for (const grader of evalCase.graders) try {
+		verdicts.push(await grader.grade(context));
+	} catch (error) {
+		if (error instanceof SweepBudgetError) {
+			incomplete = {
+				reason: "judge-refused",
+				detail: error.message
+			};
+			break;
+		}
+		if (error instanceof EvalJudgeError && error.status === "exhausted") {
+			judgeCostUsd += error.costUsd;
+			incomplete = {
+				reason: "judge-exhausted",
+				detail: error.message
+			};
+			break;
+		}
+		throw error;
+	}
 	const latencyMs = timing.start !== void 0 && timing.end !== void 0 ? Math.max(0, Date.parse(timing.end) - Date.parse(timing.start)) : 0;
 	return {
 		name,
 		status: outcome.status,
-		passed: outcome.status === "ok" && verdicts.every((verdict) => verdict.passed),
+		passed: outcome.status === "ok" && incomplete === void 0 && verdicts.every((verdict) => verdict.passed),
 		verdicts,
 		costUsd: outcome.cost.totalUsd + judgeCostUsd,
 		judgeCostUsd,
 		latencyMs,
 		usage: outcome.usage,
-		...outcome.error === void 0 ? {} : { error: outcome.error }
+		...outcome.error === void 0 ? {} : { error: outcome.error },
+		...incomplete === void 0 ? {} : { incomplete }
 	};
 }
 async function runJudge(engine, judgeName, spec, budgetUsd) {
@@ -87,7 +208,7 @@ async function runJudge(engine, judgeName, spec, budgetUsd) {
 		name: workflowName,
 		...budgetUsd === void 0 ? {} : { budgetUsd }
 	}).result;
-	if (outcome.status !== "ok") throw new EvalJudgeError(workflowName, outcome.status, outcome.error?.message);
+	if (outcome.status !== "ok") throw new EvalJudgeError(workflowName, outcome.status, outcome.error?.message, outcome.cost.totalUsd);
 	return {
 		output: outcome.value ?? null,
 		costUsd: outcome.cost.totalUsd
@@ -101,22 +222,37 @@ async function runJudge(engine, judgeName, spec, budgetUsd) {
 async function runEvalSuite(engine, cases, options = {}) {
 	const seen = /* @__PURE__ */ new Map();
 	const results = [];
+	let refusal;
 	for (const evalCase of cases) {
 		const base = evalCase.workflow.name;
 		const ordinal = seen.get(base) ?? 0;
 		seen.set(base, ordinal + 1);
 		const name = ordinal === 0 ? base : `${base}#${ordinal}`;
-		results.push(await runEvalCase(engine, evalCase, {
-			name,
-			...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd },
-			...options.judgeBudgetUsd === void 0 ? {} : { judgeBudgetUsd: options.judgeBudgetUsd }
-		}));
+		try {
+			results.push(await runEvalCase(engine, evalCase, {
+				name,
+				...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd },
+				...options.judgeBudgetUsd === void 0 ? {} : { judgeBudgetUsd: options.judgeBudgetUsd },
+				...options.envelope === void 0 ? {} : { envelope: options.envelope }
+			}));
+		} catch (error) {
+			if (!(error instanceof SweepBudgetError)) throw error;
+			refusal = {
+				runLabel: error.runLabel,
+				atCase: name,
+				detail: error.message
+			};
+			break;
+		}
 	}
 	return {
 		results,
 		passRate: results.length === 0 ? 0 : results.filter((r) => r.passed).length / results.length,
 		totalCostUsd: results.reduce((sum, r) => sum + r.costUsd, 0),
-		meanLatencyMs: results.length === 0 ? 0 : results.reduce((sum, r) => sum + r.latencyMs, 0) / results.length
+		meanLatencyMs: results.length === 0 ? 0 : results.reduce((sum, r) => sum + r.latencyMs, 0) / results.length,
+		plannedN: cases.length,
+		completedN: results.length,
+		...refusal === void 0 ? {} : { refusal }
 	};
 }
 //#endregion
@@ -337,26 +473,66 @@ function normalizeCanaryOutput(output) {
 	return (typeof output === "string" ? output : JSON.stringify(output ?? null)).normalize("NFC").trim().replace(/\s+/gu, " ");
 }
 /**
-* Runs the fixed probe set through the ordinary engine and returns the
-* fingerprint. Probes run sequentially in declaration order, one run
-* per probe, so recordings replay deterministically.
+* Runs the fixed probe set through the ordinary engine. Probes run
+* sequentially in declaration order, one run per probe, so recordings
+* replay deterministically. Each probe run carries the optional
+* immutable ceiling (options.budgetUsd) and authorizes it against the
+* optional envelope before starting; an envelope refusal records the
+* probe as 'refused' and keeps walking instead of throwing away the
+* completed probes. A non-ok or refused probe enters the fingerprint
+* as `!status` and clears allOk: callers gate drift flipping on allOk,
+* because a budget-starved or transiently failing probe fingerprints
+* differently without the model having drifted.
 */
-async function canaryFingerprint(engine, probes) {
+async function runCanary(engine, probes, options = {}) {
 	const outputs = [];
+	const probeReports = [];
 	for (const [index, prompt] of probes.prompts.entries()) {
+		try {
+			options.envelope?.authorize(options.budgetUsd, `canary probe ${String(index)}`);
+		} catch (error) {
+			if (!(error instanceof SweepBudgetError)) throw error;
+			probeReports.push({
+				prompt,
+				status: "refused"
+			});
+			outputs.push("!refused");
+			continue;
+		}
 		const workflow = defineWorkflow({ name: `kb-canary:${String(index)}` }, async (ctx) => await ctx.agent(prompt, { agentType: probes.agentType }));
-		const outcome = await engine.run(workflow, null).result;
+		const outcome = await engine.run(workflow, null, options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }).result;
+		probeReports.push({
+			prompt,
+			status: outcome.status
+		});
 		outputs.push(outcome.status === "ok" ? normalizeCanaryOutput(outcome.value) : `!${outcome.status}`);
 	}
 	const body = JSON.stringify([probes.prompts.length, outputs]);
-	return createHash("sha256").update(body, "utf8").digest("hex");
+	return {
+		fingerprint: createHash("sha256").update(body, "utf8").digest("hex"),
+		allOk: probeReports.every((probe) => probe.status === "ok"),
+		probes: probeReports
+	};
+}
+/**
+* The fingerprint alone (the pre-v1.16.2-review surface, kept
+* compatible). Prefer runCanary: its allOk is the drift-flip gate.
+*/
+async function canaryFingerprint(engine, probes, options = {}) {
+	return (await runCanary(engine, probes, options)).fingerprint;
 }
 /**
 * Flips the model's ACTIVE eval-measured claims to stale when their
 * recorded canary fingerprint differs from the fresh one. Claims
 * without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
-* an idempotent noop. CAS-rebased like every maintenance commit.
+* an idempotent noop. CAS-rebased like every maintenance commit; the
+* retries run no engine work and pay nothing.
+*
+* Only pass fingerprints from an allOk probe set (runCanary): a
+* fingerprint containing a `!status` probe differs from any healthy
+* baseline by construction, and flipping on it would blame the model
+* for a budget ceiling or a transient provider failure.
 */
 async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 	const attempts = options?.attempts ?? 3;
@@ -549,6 +725,16 @@ function renderCheckpointReport(report) {
 }
 //#endregion
 //#region src/sweeps.ts
+/**
+* Matrix sweeps (M11-T02). The deconfounder of the whole
+* knowledge feature: a FIXED eval matrix (workflow x model x
+* taskClass), independent of current routing, measured through the
+* ordinary engine (journaled, budgeted, VCR-recordable), emitting
+* eval-measured claims through the eval-committer identity.
+*
+* Sweep volume is never authorized by proposal volume: the model pool
+* and the case list are EXPLICIT caller data (fixed pools only).
+*/
 const SWEEP_THRESHOLD_DEFAULTS = {
 	strength: .9,
 	weakness: .5
@@ -575,6 +761,7 @@ async function runSweepMatrix(pool, options) {
 		...SWEEP_THRESHOLD_DEFAULTS,
 		...options.thresholds
 	};
+	if (options.envelope !== void 0 && options.suite?.budgetUsd === void 0) throw new ConfigError("runSweepMatrix: an aggregate envelope requires suite.budgetUsd (the per-target ceiling); unbounded targets under an envelope would be unaccountable");
 	const byTaskClass = /* @__PURE__ */ new Map();
 	for (const entry of pool.cases) {
 		const bucket = byTaskClass.get(entry.taskClass) ?? [];
@@ -586,19 +773,34 @@ async function runSweepMatrix(pool, options) {
 	for (const member of pool.models) {
 		const engine = await options.engineFor(member);
 		for (const [taskClass, bucket] of byTaskClass) {
-			const suite = await runEvalSuite(engine, bucket.map((entry) => entry.case), options.suite ?? {});
+			const suite = await runEvalSuite(engine, bucket.map((entry) => entry.case), {
+				...options.suite ?? {},
+				...options.envelope === void 0 ? {} : { envelope: options.envelope }
+			});
+			const exhaustedRuns = suite.results.filter((result) => result.status === "exhausted").length;
+			const judgeIncompleteRuns = suite.results.filter((result) => result.incomplete !== void 0).length;
+			const incompleteReason = suite.refusal !== void 0 ? "envelope-exhausted" : suite.results.find((result) => result.incomplete !== void 0)?.incomplete?.reason;
 			const cell = {
 				model: member.model,
 				...member.effort === void 0 ? {} : { effort: member.effort },
 				taskClass,
 				passRate: suite.passRate,
-				n: suite.results.length,
+				n: suite.completedN,
+				plannedN: suite.plannedN,
 				totalCostUsd: suite.totalCostUsd,
-				caseNames: suite.results.map((result) => result.name)
+				caseNames: suite.results.map((result) => result.name),
+				...exhaustedRuns === 0 ? {} : { exhaustedRuns },
+				...judgeIncompleteRuns === 0 ? {} : { judgeIncompleteRuns },
+				...suite.refusal === void 0 ? {} : {
+					envelopeExhausted: true,
+					refusedRunLabel: suite.refusal.runLabel
+				},
+				...incompleteReason === void 0 ? {} : { incompleteReason }
 			};
 			cells.push(cell);
 			const polarity = cell.passRate >= thresholds.strength ? "strength" : cell.passRate <= thresholds.weakness ? "weakness" : void 0;
-			if (polarity !== void 0 && cell.n > 0) {
+			const complete = cell.n === cell.plannedN && exhaustedRuns === 0 && judgeIncompleteRuns === 0 && suite.refusal === void 0;
+			if (polarity !== void 0 && cell.n > 0 && complete) {
 				const epoch = options.modelEpochFor?.(member);
 				claims.push({
 					id: claimIdOf(options.reportId, member, taskClass),
@@ -639,4 +841,4 @@ async function runSweepMatrix(pool, options) {
 	return report;
 }
 //#endregion
-export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.16.2",
+  "version": "1.18.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/testing": "1.16.2",
-    "@rulvar/core": "1.16.2"
+    "@rulvar/core": "1.18.0",
+    "@rulvar/testing": "1.18.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",