@rulvar/evals 1.16.1 → 1.17.0

package/dist/index.d.ts

@@ -1,5 +1,39 @@
 import { CompiledWorkflow, DeclaredLadder, Effort, Engine, EvidenceRef, Json, JsonSchema, KnowledgeSnapshot, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow } from "@rulvar/core";
 
+//#region src/envelope.d.ts
+/** Thrown when authorizing a run's ceiling would exceed the envelope. */
+declare class SweepBudgetError extends Error {
+  /** What was about to start, e.g. `eval target 'sweep-math'`. */
+  readonly runLabel: string;
+  /** The per-run ceiling that did not fit. */
+  readonly ceilingUsd: number;
+  /** Total already authorized before this refusal. */
+  readonly authorizedUsd: number;
+  readonly maxTotalUsd: number;
+  constructor(runLabel: string, ceilingUsd: number, authorizedUsd: number, maxTotalUsd: number);
+}
+/**
+* One envelope bounds one whole sweep invocation: share the instance
+* across the canary loop and runSweepMatrix so canary, target, and
+* judge runs all draw from the same remainder.
+*/
+declare class SpendEnvelope {
+  readonly maxTotalUsd: number;
+  private readonly maxMicroUsd;
+  private authorizedMicroUsd;
+  constructor(maxTotalUsd: number);
+  /** Total authorized so far (debit-only; never decreases). */
+  get authorizedUsd(): number;
+  get remainingUsd(): number;
+  /**
+  * Authorizes one run's immutable ceiling or throws SweepBudgetError.
+  * An unbounded run cannot be authorized: under an envelope every run
+  * MUST carry an explicit positive ceiling, otherwise the aggregate
+  * bound would be unaccountable.
+  */
+  authorize(ceilingUsd: number | undefined, runLabel: string): void;
+}
+//#endregion
 //#region src/case.d.ts
 /**
 * One quality-measurement case. The shape is the
@@ -79,6 +113,13 @@ interface RunEvalCaseOptions {
   budgetUsd?: number;
   /** Run ceiling for each judge run. */
   judgeBudgetUsd?: number;
+  /**
+  * Aggregate debit-only envelope (v1.16.2 review P1-2): every target
+  * and judge run authorizes its ceiling here BEFORE starting, and an
+  * envelope requires the matching per-run ceiling to be set. A
+  * refusal throws SweepBudgetError before any provider work.
+  */
+  envelope?: SpendEnvelope;
 }
 /** Thrown when a judge run does not settle ok. */
 declare class EvalJudgeError extends Error {
@@ -105,6 +146,8 @@ interface EvalSuiteResult {
 interface RunEvalSuiteOptions {
   budgetUsd?: number;
   judgeBudgetUsd?: number;
+  /** See RunEvalCaseOptions.envelope; shared across every case of the suite. */
+  envelope?: SpendEnvelope;
 }
 /**
 * Runs cases sequentially (deterministic journal and cassette order) and
@@ -232,14 +275,52 @@ interface CanaryProbeSet {
   /** The fixed prompts; order matters and enters the fingerprint. */
   prompts: string[];
 }
+interface CanaryRunOptions {
+  /**
+  * Immutable ceiling per probe run (v1.16.2 review P1-2): every probe
+  * is an ordinary paid engine run and gets its own recorded
+  * RunMeta.budgetUsd.
+  */
+  budgetUsd?: number;
+  /**
+  * Aggregate debit-only envelope shared with the surrounding sweep;
+  * each probe authorizes budgetUsd BEFORE running, and an envelope
+  * requires budgetUsd to be set.
+  */
+  envelope?: SpendEnvelope;
+}
+interface CanaryReport {
+  fingerprint: string;
+  /**
+  * True only when every probe settled ok. A fingerprint containing a
+  * non-ok probe status is a measurement artifact (budget exhaustion,
+  * transient provider failure), NOT evidence of model drift: never
+  * feed it to flipStaleOnCanaryDrift.
+  */
+  allOk: boolean;
+  probes: Array<{
+    prompt: string;
+    status: RunOutcome<unknown>["status"];
+  }>;
+}
 /** The committed v1 normalization (OQ-06): NFC, trim, collapse whitespace. */
 declare function normalizeCanaryOutput(output: unknown): string;
 /**
-* Runs the fixed probe set through the ordinary engine and returns the
-* fingerprint. Probes run sequentially in declaration order, one run
-* per probe, so recordings replay deterministically.
+* Runs the fixed probe set through the ordinary engine. Probes run
+* sequentially in declaration order, one run per probe, so recordings
+* replay deterministically. Each probe run carries the optional
+* immutable ceiling (options.budgetUsd) and authorizes it against the
+* optional envelope before starting. A non-ok probe enters the
+* fingerprint as `!status` and clears allOk: callers gate drift
+* flipping on allOk, because a budget-starved or transiently failing
+* probe fingerprints differently without the model having drifted.
+*/
+declare function runCanary(engine: Engine, probes: CanaryProbeSet, options?: CanaryRunOptions): Promise<CanaryReport>;
+/**
+* The fingerprint alone (the pre-v1.16.2-review surface, kept
+* compatible). Prefer runCanary: its allOk is the drift-flip gate.
 */
-declare function canaryFingerprint(engine: Engine, probes: CanaryProbeSet): Promise<string>;
+declare function canaryFingerprint(engine: Engine, probes: CanaryProbeSet, options?: CanaryRunOptions): Promise<string>;
 interface CanaryDriftReport {
   model: ModelRef;
   freshFingerprint: string;
@@ -253,7 +334,13 @@ interface CanaryDriftReport {
 * recorded canary fingerprint differs from the fresh one. Claims
 * without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
-* an idempotent noop. CAS-rebased like every maintenance commit.
+* an idempotent noop. CAS-rebased like every maintenance commit; the
+* retries run no engine work and pay nothing.
+*
+* Only pass fingerprints from an allOk probe set (runCanary): a
+* fingerprint containing a `!status` probe differs from any healthy
+* baseline by construction, and flipping on it would blame the model
+* for a budget ceiling or a transient provider failure.
 */
 declare function flipStaleOnCanaryDrift(store: ModelKnowledgeStore, model: ModelRef, freshFingerprint: string, options?: {
   attempts?: number;
@@ -297,6 +384,17 @@ interface RunSweepOptions {
   thresholds?: Partial<SweepThresholds>;
   /** Passed through to every suite run (budget, VCR hooks ride the engine). */
   suite?: RunEvalSuiteOptions;
+  /**
+  * Aggregate debit-only envelope over the WHOLE matrix (v1.16.2
+  * review P1-2): every target and judge run authorizes its immutable
+  * ceiling before starting, so the pool times cases times judge-call
+  * product cannot exceed it, falsification pool growth included. An
+  * envelope requires suite.budgetUsd (and suite.judgeBudgetUsd once a
+  * grader judges); a cell refused by the envelope lands in the report
+  * as envelopeExhausted with NO claim. Share the instance with the
+  * canary loop so probes draw from the same remainder.
+  */
+  envelope?: SpendEnvelope;
   /** When given, emitted claims commit through the committer identity. */
   store?: ModelKnowledgeStore;
   /**
@@ -313,6 +411,20 @@ interface SweepCellReport {
   n: number;
   totalCostUsd: number;
   caseNames: string[];
+  /**
+  * Count of case results whose TARGET run settled 'exhausted' (its
+  * per-run ceiling, not the envelope). A budget-starved measurement
+  * must not become a model belief, so any exhausted target suppresses
+  * the cell's claim even when the degraded passRate crosses a
+  * threshold: the alternative is committing a false weakness that
+  * blames the model for the ceiling.
+  */
+  exhaustedRuns?: number;
+  /**
+  * The aggregate envelope refused a run of this cell before it
+  * started; stats cover nothing reliable and the cell emits no claim.
+  */
+  envelopeExhausted?: true;
 }
 interface SweepReport {
   reportId: string;
@@ -417,4 +529,4 @@ declare function runValueCheckpoint(checkpointPool: CheckpointPool, options: Run
 /** The deterministic render for the M12 gate docs amendment. */
 declare function renderCheckpointReport(report: CheckpointReport): string;
 //#endregion
-export { type CanaryDriftReport, type CanaryProbeSet, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+export { type CanaryDriftReport, type CanaryProbeSet, type CanaryReport, type CanaryRunOptions, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
+import { ConfigError, KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
 import { createHash } from "node:crypto";
 //#region src/case.ts
 /**
@@ -32,6 +32,7 @@ var EvalJudgeError = class extends Error {
 async function runEvalCase(engine, evalCase, options = {}) {
 	const name = options.name ?? evalCase.workflow.name;
 	const timing = {};
+	options.envelope?.authorize(options.budgetUsd, `eval target '${name}'`);
 	const handle = engine.run(evalCase.workflow, evalCase.args, {
 		name: `eval:${name}`,
 		...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }
@@ -53,6 +54,7 @@ async function runEvalCase(engine, evalCase, options = {}) {
 		async judge(spec) {
 			const ordinal = judgeOrdinal;
 			judgeOrdinal += 1;
+			options.envelope?.authorize(options.judgeBudgetUsd, `eval judge '${name}:${String(ordinal)}'`);
 			const judged = await runJudge(engine, `${name}:${ordinal}`, spec, options.judgeBudgetUsd);
 			judgeCostUsd += judged.costUsd;
 			return judged.output;
@@ -109,7 +111,8 @@ async function runEvalSuite(engine, cases, options = {}) {
 		results.push(await runEvalCase(engine, evalCase, {
 			name,
 			...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd },
-			...options.judgeBudgetUsd === void 0 ? {} : { judgeBudgetUsd: options.judgeBudgetUsd }
+			...options.judgeBudgetUsd === void 0 ? {} : { judgeBudgetUsd: options.judgeBudgetUsd },
+			...options.envelope === void 0 ? {} : { envelope: options.envelope }
 		}));
 	}
 	return {
@@ -337,26 +340,54 @@ function normalizeCanaryOutput(output) {
 	return (typeof output === "string" ? output : JSON.stringify(output ?? null)).normalize("NFC").trim().replace(/\s+/gu, " ");
 }
 /**
-* Runs the fixed probe set through the ordinary engine and returns the
-* fingerprint. Probes run sequentially in declaration order, one run
-* per probe, so recordings replay deterministically.
+* Runs the fixed probe set through the ordinary engine. Probes run
+* sequentially in declaration order, one run per probe, so recordings
+* replay deterministically. Each probe run carries the optional
+* immutable ceiling (options.budgetUsd) and authorizes it against the
+* optional envelope before starting. A non-ok probe enters the
+* fingerprint as `!status` and clears allOk: callers gate drift
+* flipping on allOk, because a budget-starved or transiently failing
+* probe fingerprints differently without the model having drifted.
 */
-async function canaryFingerprint(engine, probes) {
+async function runCanary(engine, probes, options = {}) {
 	const outputs = [];
+	const probeReports = [];
 	for (const [index, prompt] of probes.prompts.entries()) {
+		options.envelope?.authorize(options.budgetUsd, `canary probe ${String(index)}`);
 		const workflow = defineWorkflow({ name: `kb-canary:${String(index)}` }, async (ctx) => await ctx.agent(prompt, { agentType: probes.agentType }));
-		const outcome = await engine.run(workflow, null).result;
+		const outcome = await engine.run(workflow, null, options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }).result;
+		probeReports.push({
+			prompt,
+			status: outcome.status
+		});
 		outputs.push(outcome.status === "ok" ? normalizeCanaryOutput(outcome.value) : `!${outcome.status}`);
 	}
 	const body = JSON.stringify([probes.prompts.length, outputs]);
-	return createHash("sha256").update(body, "utf8").digest("hex");
+	return {
+		fingerprint: createHash("sha256").update(body, "utf8").digest("hex"),
+		allOk: probeReports.every((probe) => probe.status === "ok"),
+		probes: probeReports
+	};
+}
+/**
+* The fingerprint alone (the pre-v1.16.2-review surface, kept
+* compatible). Prefer runCanary: its allOk is the drift-flip gate.
+*/
+async function canaryFingerprint(engine, probes, options = {}) {
+	return (await runCanary(engine, probes, options)).fingerprint;
 }
 /**
 * Flips the model's ACTIVE eval-measured claims to stale when their
 * recorded canary fingerprint differs from the fresh one. Claims
 * without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
-* an idempotent noop. CAS-rebased like every maintenance commit.
+* an idempotent noop. CAS-rebased like every maintenance commit; the
+* retries run no engine work and pay nothing.
+*
+* Only pass fingerprints from an allOk probe set (runCanary): a
+* fingerprint containing a `!status` probe differs from any healthy
+* baseline by construction, and flipping on it would blame the model
+* for a budget ceiling or a transient provider failure.
 */
 async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 	const attempts = options?.attempts ?? 3;
@@ -393,6 +424,82 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 	throw lastCas ?? /* @__PURE__ */ new Error("flipStaleOnCanaryDrift: unreachable");
 }
 //#endregion
+//#region src/envelope.ts
+/**
+* The debit-only aggregate spend envelope (v1.16.2 review P1-2). A
+* sweep multiplies paid runs: pool members times cases for targets,
+* one judge run per GraderContext.judge call, one canary run per
+* probe per member, and the falsification union can grow the pool
+* beyond the config. Per-run ceilings alone do not bound that
+* product, so the envelope authorizes each run's IMMUTABLE ceiling
+* BEFORE the run starts: a run whose ceiling does not fit the
+* remainder is refused before any provider work.
+*
+* Authorizations are never returned: not when a run completes under
+* its ceiling, not on CAS retries (they run no paid work), and not on
+* replay. A replayed run authorizes exactly like a fresh one and then
+* spends nothing; the envelope bounds the authorized worst case, not
+* the observed spend, so replay never double-PAYS anything while the
+* accounting stays one-directional. The envelope lives for one
+* invocation and is never persisted.
+*
+* Accounting is integer micro-USD, so exact fits pass: 0.1 + 0.2
+* against a 0.3 envelope is a fit, not a float rejection.
+*/
+const MICRO = 1e6;
+/** Thrown when authorizing a run's ceiling would exceed the envelope. */
+var SweepBudgetError = class extends Error {
+	/** What was about to start, e.g. `eval target 'sweep-math'`. */
+	runLabel;
+	/** The per-run ceiling that did not fit. */
+	ceilingUsd;
+	/** Total already authorized before this refusal. */
+	authorizedUsd;
+	maxTotalUsd;
+	constructor(runLabel, ceilingUsd, authorizedUsd, maxTotalUsd) {
+		super(`sweep envelope exhausted: authorizing $${String(ceilingUsd)} for ${runLabel} would exceed maxTotalUsd $${String(maxTotalUsd)} ($${String(authorizedUsd)} already authorized); the run was refused before any provider call`);
+		this.name = "SweepBudgetError";
+		this.runLabel = runLabel;
+		this.ceilingUsd = ceilingUsd;
+		this.authorizedUsd = authorizedUsd;
+		this.maxTotalUsd = maxTotalUsd;
+	}
+};
+/**
+* One envelope bounds one whole sweep invocation: share the instance
+* across the canary loop and runSweepMatrix so canary, target, and
+* judge runs all draw from the same remainder.
+*/
+var SpendEnvelope = class {
+	maxTotalUsd;
+	maxMicroUsd;
+	authorizedMicroUsd = 0;
+	constructor(maxTotalUsd) {
+		if (!Number.isFinite(maxTotalUsd) || maxTotalUsd <= 0) throw new ConfigError(`SpendEnvelope maxTotalUsd must be a positive finite number, got ${String(maxTotalUsd)}`);
+		this.maxTotalUsd = maxTotalUsd;
+		this.maxMicroUsd = Math.round(maxTotalUsd * MICRO);
+	}
+	/** Total authorized so far (debit-only; never decreases). */
+	get authorizedUsd() {
+		return this.authorizedMicroUsd / MICRO;
+	}
+	get remainingUsd() {
+		return Math.max(0, this.maxMicroUsd - this.authorizedMicroUsd) / MICRO;
+	}
+	/**
+	* Authorizes one run's immutable ceiling or throws SweepBudgetError.
+	* An unbounded run cannot be authorized: under an envelope every run
+	* MUST carry an explicit positive ceiling, otherwise the aggregate
+	* bound would be unaccountable.
+	*/
+	authorize(ceilingUsd, runLabel) {
+		if (ceilingUsd === void 0 || !Number.isFinite(ceilingUsd) || ceilingUsd <= 0) throw new ConfigError(`the spend envelope requires an explicit positive per-run ceiling for ${runLabel}; got ${String(ceilingUsd)} (an unbounded run under an aggregate envelope would be unaccountable)`);
+		const micro = Math.round(ceilingUsd * MICRO);
+		if (this.authorizedMicroUsd + micro > this.maxMicroUsd) throw new SweepBudgetError(runLabel, ceilingUsd, this.authorizedUsd, this.maxTotalUsd);
+		this.authorizedMicroUsd += micro;
+	}
+};
+//#endregion
 //#region src/checkpoint.ts
 /**
 * The phases 1-2 measured-value checkpoint (M12-T01; the quantitative
@@ -549,6 +656,16 @@ function renderCheckpointReport(report) {
 }
 //#endregion
 //#region src/sweeps.ts
+/**
+* Matrix sweeps (M11-T02). The deconfounder of the whole
+* knowledge feature: a FIXED eval matrix (workflow x model x
+* taskClass), independent of current routing, measured through the
+* ordinary engine (journaled, budgeted, VCR-recordable), emitting
+* eval-measured claims through the eval-committer identity.
+*
+* Sweep volume is never authorized by proposal volume: the model pool
+* and the case list are EXPLICIT caller data (fixed pools only).
+*/
 const SWEEP_THRESHOLD_DEFAULTS = {
 	strength: .9,
 	weakness: .5
@@ -575,6 +692,7 @@ async function runSweepMatrix(pool, options) {
 		...SWEEP_THRESHOLD_DEFAULTS,
 		...options.thresholds
 	};
+	if (options.envelope !== void 0 && options.suite?.budgetUsd === void 0) throw new ConfigError("runSweepMatrix: an aggregate envelope requires suite.budgetUsd (the per-target ceiling); unbounded targets under an envelope would be unaccountable");
 	const byTaskClass = /* @__PURE__ */ new Map();
 	for (const entry of pool.cases) {
 		const bucket = byTaskClass.get(entry.taskClass) ?? [];
@@ -586,7 +704,27 @@ async function runSweepMatrix(pool, options) {
 	for (const member of pool.models) {
 		const engine = await options.engineFor(member);
 		for (const [taskClass, bucket] of byTaskClass) {
-			const suite = await runEvalSuite(engine, bucket.map((entry) => entry.case), options.suite ?? {});
+			let suite;
+			try {
+				suite = await runEvalSuite(engine, bucket.map((entry) => entry.case), {
+					...options.suite ?? {},
+					...options.envelope === void 0 ? {} : { envelope: options.envelope }
+				});
+			} catch (error) {
+				if (!(error instanceof SweepBudgetError)) throw error;
+				cells.push({
+					model: member.model,
+					...member.effort === void 0 ? {} : { effort: member.effort },
+					taskClass,
+					passRate: 0,
+					n: 0,
+					totalCostUsd: 0,
+					caseNames: [],
+					envelopeExhausted: true
+				});
+				continue;
+			}
+			const exhaustedRuns = suite.results.filter((result) => result.status === "exhausted").length;
 			const cell = {
 				model: member.model,
 				...member.effort === void 0 ? {} : { effort: member.effort },
@@ -594,11 +732,12 @@ async function runSweepMatrix(pool, options) {
 				passRate: suite.passRate,
 				n: suite.results.length,
 				totalCostUsd: suite.totalCostUsd,
-				caseNames: suite.results.map((result) => result.name)
+				caseNames: suite.results.map((result) => result.name),
+				...exhaustedRuns === 0 ? {} : { exhaustedRuns }
 			};
 			cells.push(cell);
 			const polarity = cell.passRate >= thresholds.strength ? "strength" : cell.passRate <= thresholds.weakness ? "weakness" : void 0;
-			if (polarity !== void 0 && cell.n > 0) {
+			if (polarity !== void 0 && cell.n > 0 && exhaustedRuns === 0) {
 				const epoch = options.modelEpochFor?.(member);
 				claims.push({
 					id: claimIdOf(options.reportId, member, taskClass),
@@ -639,4 +778,4 @@ async function runSweepMatrix(pool, options) {
 	return report;
 }
 //#endregion
-export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.16.1",
+  "version": "1.17.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.16.1",
-    "@rulvar/testing": "1.16.1"
+    "@rulvar/core": "1.17.0",
+    "@rulvar/testing": "1.17.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",