@rulvar/evals 1.0.0 → 1.2.0

package/dist/index.d.ts

@@ -1,8 +1,8 @@
-import { CompiledWorkflow, Effort, Engine, EvidenceRef, Json, JsonSchema, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow } from "@rulvar/core";
+import { CompiledWorkflow, DeclaredLadder, Effort, Engine, EvidenceRef, Json, JsonSchema, KnowledgeSnapshot, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow } from "@rulvar/core";
 
 //#region src/case.d.ts
 /**
-* One quality-measurement case (docs/09, section 7.1). The shape is the
+* One quality-measurement case. The shape is the
 * documented interface verbatim; display names derive from the workflow
 * name (the suite runner disambiguates duplicates by ordinal).
 */
@@ -24,7 +24,7 @@ interface GraderVerdict {
 /**
 * A judge invocation specification. The judge runs through the engine as
 * an ordinary journaled, budgeted invocation; model selection is subject
-* to the router quality floors (docs/04, section "Role quality floors"),
+* to the router quality floors,
 * and @rulvar/evals ships NO default judge model: weak defaults for
 * judging are forbidden, so the model is always explicit.
 */
@@ -65,7 +65,7 @@ interface EvalCaseResult {
   judgeCostUsd: number;
   /**
   * run:start to run:end of the target run, from event timestamps; no
-  * separate measurement channel exists (docs/09, section 7.2).
+  * separate measurement channel exists.
   */
   latencyMs: number;
   /** The target run's normalized usage. */
@@ -162,7 +162,7 @@ declare function rubricGrader(criteria: RubricCriterion[], options?: RubricGrade
 /** The default judge verdict shape. */
 declare const JUDGE_VERDICT_SCHEMA: JsonSchema;
 interface JudgeGraderOptions {
-  /** Judge model; required, never defaulted (docs/04 role quality floors). */
+  /** Judge model; required, never defaulted (role quality floors). */
   model: ModelSpec;
   /** What to judge: the criteria prose embedded into the judge prompt. */
   instruction: string;
@@ -213,10 +213,10 @@ interface EvalCommitterOptions {
   committerId: string;
   /** The emitting sweep report; every claim's gate references it. */
   reportId: string;
-  /** CAS-rebase attempts (docs/05, 5.4); default 3. */
+  /** CAS-rebase attempts; default 3. */
   attempts?: number;
 }
-/** One measured claim, TTL applied per the docs/05 decay table. */
+/** One measured claim; claimExpiry applies the TTL from the decay table. */
 declare function evalMeasuredClaim(input: MeasuredClaimInput, committerId: string): ModelClaim;
 /**
 * Commits measured claims through the eval-committer gate with the
@@ -250,9 +250,8 @@ interface CanaryDriftReport {
 }
 /**
 * Flips the model's ACTIVE eval-measured claims to stale when their
-* recorded canary fingerprint differs from the fresh one (docs/05:
-* "a fingerprint change immediately flips the model's eval claims to
-* stale"). Claims without a recorded fingerprint have no baseline and
+* recorded canary fingerprint differs from the fresh one. Claims
+* without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
 * an idempotent noop. CAS-rebased like every maintenance commit.
 */
@@ -284,7 +283,7 @@ interface SweepThresholds {
 interface RunSweepOptions {
   /** Deterministic, caller-minted; every claim's evidence and gate reference it. */
   reportId: string;
-  /** The dedicated identity (docs/05, 5.4). */
+  /** The dedicated committer identity. */
   committerId: string;
   /** ISO date of the sweep; the TTL table applies from it (no wall clock inside). */
   observedAt: string;
@@ -332,4 +331,90 @@ declare const SWEEP_THRESHOLD_DEFAULTS: SweepThresholds;
 */
 declare function runSweepMatrix(pool: SweepPool, options: RunSweepOptions): Promise<SweepReport>;
 //#endregion
-export { type CanaryDriftReport, type CanaryProbeSet, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type RubricCriterion, type RubricGraderOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix };
+//#region src/checkpoint.d.ts
+/** One declared checkpoint ladder: rungs are concrete pool members. */
+interface CheckpointLadder extends DeclaredLadder {
+  name: string;
+  startTier: number;
+  rungs: SweepModel[];
+}
+interface CheckpointPool {
+  ladders: CheckpointLadder[];
+  /** The measurement half; the seeding sweep MUST NOT have seen these. */
+  evalCases: SweepCase[];
+}
+interface OrchestratedCase {
+  /** The workflow drives an orchestrate-role run; graders judge its outcome. */
+  case: EvalCase;
+}
+interface RunCheckpointOptions {
+  /** The claims snapshot produced by the seeding sweep (disjoint cases). */
+  snapshot: KnowledgeSnapshot;
+  /** ISO date of the evaluation (recorded in the report; no wall clock inside). */
+  observedAt: string;
+  /** An engine per concrete pool member (the caller owns adapters and budgets). */
+  engineFor: (member: SweepModel) => Engine | Promise<Engine>;
+  /**
+  * Criterion 2 engines: withKnowledge true configures the SAME store
+  * snapshot behind stores.modelKnowledge; false omits it entirely.
+  */
+  orchestrateEngineFor?: (withKnowledge: boolean) => Engine | Promise<Engine>;
+  orchestratedCases?: OrchestratedCase[];
+  suite?: RunEvalSuiteOptions;
+  /**
+  * Orchestrated runs need room for the orchestrator cap math (the
+  * run ceiling must host the finalize reserve): their
+  * suite options default to `suite` but usually carry a larger
+  * budgetUsd.
+  */
+  orchestratedSuite?: RunEvalSuiteOptions;
+}
+interface CheckpointArm {
+  passRate: number;
+  totalCostUsd: number;
+  n: number;
+}
+interface CheckpointCell {
+  ladder: string;
+  taskClass: TaskClass;
+  defaultTier: number;
+  /** The tier the treatment arm ran at (default when no recommendation). */
+  treatmentTier: number;
+  recommended: boolean;
+  baseline: CheckpointArm;
+  treatment: CheckpointArm;
+  passed: boolean;
+}
+interface CriterionOneReport {
+  cells: CheckpointCell[];
+  cellsPassed: number;
+  majorityHolds: boolean;
+  pooledBaseline: CheckpointArm;
+  pooledTreatment: CheckpointArm;
+  pooledHolds: boolean;
+  passed: boolean;
+}
+interface CriterionTwoReport {
+  baseline: CheckpointArm;
+  informed: CheckpointArm;
+  passed: boolean;
+}
+interface CheckpointReport {
+  observedAt: string;
+  criterion1: CriterionOneReport;
+  criterion2?: CriterionTwoReport;
+  /** Both criteria (criterion 2 counts as failed when unmeasured). */
+  passed: boolean;
+}
+/** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
+declare function rungRuleHolds(baseline: CheckpointArm, treatment: CheckpointArm): boolean;
+/**
+* Runs the checkpoint over the fixed pool. Sequential in declaration
+* order (deterministic cassette consumption when recorded); every cell
+* runs baseline then treatment.
+*/
+declare function runValueCheckpoint(checkpointPool: CheckpointPool, options: RunCheckpointOptions): Promise<CheckpointReport>;
+/** The deterministic render for the M12 gate docs amendment. */
+declare function renderCheckpointReport(report: CheckpointReport): string;
+//#endregion
+export { type CanaryDriftReport, type CanaryProbeSet, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { KnowledgeCasError, claimExpiry, defineWorkflow } from "@rulvar/core";
+import { KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
 import { createHash } from "node:crypto";
 //#region src/case.ts
 /**
@@ -6,9 +6,9 @@ import { createHash } from "node:crypto";
 * suite runners. A separate quality-measurement package built strictly on
 * the public APIs (L6).
 *
-* Owning spec: docs/09-observability-testing-spec.md, section
-* "@rulvar/evals"; determinism rules in docs/11-testing-strategy.md,
-* section "Eval CI": judge graders run THROUGH the engine, so judge calls
+* Determinism rules (https://docs.rulvar.com/guide/evals and
+* https://docs.rulvar.com/guide/testing): judge graders run THROUGH the
+* engine, so judge calls
 * are journaled, budgeted, and VCR-recordable, and an eval suite replays
 * deterministically from cassettes with zero live calls.
 */
@@ -254,14 +254,14 @@ function judgeGrader(options) {
 //#endregion
 //#region src/committer.ts
 /**
-* The eval-committer identity (M11-T01; docs/05, sections "Data model"
-* and "Commit discipline"). The pipeline-side commit path: builds
+* The eval-committer identity (M11-T01; https://docs.rulvar.com/guide/model-knowledge).
+* The pipeline-side commit path: builds
 * eval-committer-gated ops (the coherence square: class eval-measured,
 * author eval-pipeline, metrics present) and commits them with the
 * documented CAS-rebase recipe. Humans never call this; their path is
 * the human gate and it structurally cannot carry metrics.
 */
-/** One measured claim, TTL applied per the docs/05 decay table. */
+/** One measured claim; claimExpiry applies the TTL from the decay table. */
 function evalMeasuredClaim(input, committerId) {
 	return {
 		id: input.id,
@@ -318,8 +318,7 @@ async function commitEvalMeasured(store, claims, options) {
 //#endregion
 //#region src/canary.ts
 /**
-* The canary fingerprint (M11-T04; docs/05, section "Grounding and
-* decay"; OQ-06). The optional compensation for silent alias
+* The canary fingerprint (M11-T04; OQ-06). The optional compensation for silent alias
 * re-pointing that modelEpoch honestly cannot catch: a FIXED probe set
 * at temperature 0, run through the ordinary engine (journaled,
 * budgeted, VCR-recordable), hashed over normalized outputs. A
@@ -353,9 +352,8 @@ async function canaryFingerprint(engine, probes) {
 }
 /**
 * Flips the model's ACTIVE eval-measured claims to stale when their
-* recorded canary fingerprint differs from the fresh one (docs/05:
-* "a fingerprint change immediately flips the model's eval claims to
-* stale"). Claims without a recorded fingerprint have no baseline and
+* recorded canary fingerprint differs from the fresh one. Claims
+* without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
 * an idempotent noop. CAS-rebased like every maintenance commit.
 */
@@ -394,6 +392,161 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 	throw lastCas ?? /* @__PURE__ */ new Error("flipStaleOnCanaryDrift: unreachable");
 }
 //#endregion
+//#region src/checkpoint.ts
+/**
+* The phases 1-2 measured-value checkpoint (M12-T01; the quantitative
+* criteria of OQ-09, closed at M11-T06). The M12 gate: kb_propose
+* and the proposal loop ship ONLY if the knowledge card demonstrably
+* improves tier and agentType selection on eval cases.
+*
+* Two experiments, both A/B under identical fixed pools:
+*
+* 1. RUNG SELECTION, per (ladder, taskClass) cell: the baseline arm
+*    runs every eval case at the ladder's DEFAULT start tier; the
+*    treatment arm runs at the tier recommended by
+*    compileVerifiedLayer over the store's claims (default when no
+*    recommendation). A cell passes when the treatment reaches a pass
+*    rate at least equal to the baseline at no more than 90 percent
+*    of its cost, OR at least 5 points above it at no more than its
+*    cost. Criterion 1 holds when a MAJORITY of cells pass AND the
+*    pooled aggregate passes the same rule.
+*
+* 2. AGENTTYPE SELECTION, pooled: the same orchestrate-role cases run
+*    with and without the knowledge store configured (the card docks
+*    into the spawn tool description when configured). Criterion 2
+*    holds when the card-informed arm matches or beats the baseline
+*    pass rate at no more than 105 percent of its cost, OR beats it
+*    by at least 15 points at no more than 115 percent of its cost
+*    (the quality branch; OQ-09 as amended 2026-07-12: the baseline
+*    fails CHEAPLY, so the flat cost bar tightened exactly when the
+*    card was winning on quality).
+*
+* The checkpoint PASSES only when both criteria hold. Methodology
+* guard: the claims the treatment consumes MUST come from a seeding
+* sweep over a DISJOINT case set (the seed/eval split is the caller's
+* pool contract), or the measurement is leakage.
+*/
+/** IEEE754 guard for the rule boundaries (0.8 + 0.05 exceeds 0.85). */
+const EPSILON = 1e-9;
+/**
+* The OQ-09 criterion 2 rule (as amended 2026-07-12): match-or-beat at
+* 105 percent of baseline cost, OR at least 15 points better at 115
+* percent (the quality branch: the baseline fails cheaply, so the flat
+* bar tightened exactly when the card won on quality). The vacuous-pass
+* guard stays with the caller.
+*/
+function agentTypeRuleHolds(baseline, informed) {
+	const matchesCheaply = informed.passRate >= baseline.passRate - EPSILON && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON;
+	const clearlyBetterNearCost = informed.passRate >= baseline.passRate + .15 - EPSILON && informed.totalCostUsd <= 1.15 * baseline.totalCostUsd + EPSILON;
+	return matchesCheaply || clearlyBetterNearCost;
+}
+/** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
+function rungRuleHolds(baseline, treatment) {
+	const equalOrBetterCheaper = treatment.passRate >= baseline.passRate - EPSILON && treatment.totalCostUsd <= .9 * baseline.totalCostUsd + EPSILON;
+	const clearlyBetterAtCost = treatment.passRate >= baseline.passRate + .05 - EPSILON && treatment.totalCostUsd <= baseline.totalCostUsd + EPSILON;
+	return equalOrBetterCheaper || clearlyBetterAtCost;
+}
+function armOf(suite) {
+	return {
+		passRate: suite.passRate,
+		totalCostUsd: suite.totalCostUsd,
+		n: suite.results.length
+	};
+}
+function pool(arms) {
+	const n = arms.reduce((sum, arm) => sum + arm.n, 0);
+	const passed = arms.reduce((sum, arm) => sum + arm.passRate * arm.n, 0);
+	const cost = arms.reduce((sum, arm) => sum + arm.totalCostUsd, 0);
+	return {
+		passRate: n === 0 ? 0 : passed / n,
+		totalCostUsd: cost,
+		n
+	};
+}
+/**
+* Runs the checkpoint over the fixed pool. Sequential in declaration
+* order (deterministic cassette consumption when recorded); every cell
+* runs baseline then treatment.
+*/
+async function runValueCheckpoint(checkpointPool, options) {
+	const recommendations = compileVerifiedLayer(options.snapshot.claims.filter((claim) => claim.status === "active"), checkpointPool.ladders);
+	const byTaskClass = /* @__PURE__ */ new Map();
+	for (const entry of checkpointPool.evalCases) {
+		const bucket = byTaskClass.get(entry.taskClass) ?? [];
+		bucket.push(entry.case);
+		byTaskClass.set(entry.taskClass, bucket);
+	}
+	const cells = [];
+	for (const ladder of checkpointPool.ladders) for (const [taskClass, cases] of byTaskClass) {
+		const recommendation = recommendations.find((row) => row.ladder === ladder.name && row.taskClass === taskClass);
+		const treatmentTier = recommendation?.recommendedTier ?? ladder.startTier;
+		const baseMember = ladder.rungs[ladder.startTier];
+		const treatMember = ladder.rungs[treatmentTier];
+		if (baseMember === void 0 || treatMember === void 0) throw new Error(`checkpoint: ladder '${ladder.name}' lacks rung ${String(treatmentTier)}`);
+		const baseline = armOf(await runEvalSuite(await options.engineFor(baseMember), cases, options.suite ?? {}));
+		const treatment = treatmentTier === ladder.startTier ? baseline : armOf(await runEvalSuite(await options.engineFor(treatMember), cases, options.suite ?? {}));
+		cells.push({
+			ladder: ladder.name,
+			taskClass,
+			defaultTier: ladder.startTier,
+			treatmentTier,
+			recommended: recommendation !== void 0,
+			baseline,
+			treatment,
+			passed: rungRuleHolds(baseline, treatment)
+		});
+	}
+	const recommendedCells = cells.filter((cell) => cell.recommended);
+	const cellsPassed = recommendedCells.filter((cell) => cell.passed).length;
+	const majorityHolds = recommendedCells.length > 0 && cellsPassed * 2 > recommendedCells.length;
+	const pooledBaseline = pool(cells.map((cell) => cell.baseline));
+	const pooledTreatment = pool(cells.map((cell) => cell.treatment));
+	const pooledHolds = rungRuleHolds(pooledBaseline, pooledTreatment);
+	const criterion1 = {
+		cells,
+		cellsPassed,
+		majorityHolds,
+		pooledBaseline,
+		pooledTreatment,
+		pooledHolds,
+		passed: majorityHolds && pooledHolds
+	};
+	let criterion2;
+	if (options.orchestrateEngineFor !== void 0 && options.orchestratedCases !== void 0) {
+		const cases = options.orchestratedCases.map((entry) => entry.case);
+		const orchestratedSuite = options.orchestratedSuite ?? options.suite ?? {};
+		const baseline = armOf(await runEvalSuite(await options.orchestrateEngineFor(false), cases, orchestratedSuite));
+		const informed = armOf(await runEvalSuite(await options.orchestrateEngineFor(true), cases, orchestratedSuite));
+		criterion2 = {
+			baseline,
+			informed,
+			passed: informed.n > 0 && informed.passRate > 0 && agentTypeRuleHolds(baseline, informed)
+		};
+	}
+	return {
+		observedAt: options.observedAt,
+		criterion1,
+		...criterion2 === void 0 ? {} : { criterion2 },
+		passed: criterion1.passed && criterion2 !== void 0 && criterion2.passed
+	};
+}
+const percent = (rate) => `${(rate * 100).toFixed(1)}%`;
+const usd = (value) => `$${value.toFixed(4)}`;
+/** The deterministic render for the M12 gate docs amendment. */
+function renderCheckpointReport(report) {
+	const lines = [
+		`Measured-value checkpoint (OQ-09) at ${report.observedAt}: ` + (report.passed ? "PASSED" : "FAILED"),
+		"",
+		`Criterion 1 (rung selection): ${report.criterion1.passed ? "holds" : "fails"} (${String(report.criterion1.cellsPassed)}/${String(report.criterion1.cells.length)} cells, pooled ${report.criterion1.pooledHolds ? "holds" : "fails"})`
+	];
+	for (const cell of report.criterion1.cells) lines.push(`* ${cell.ladder} :: ${cell.taskClass}: baseline tier ${String(cell.defaultTier)} ${percent(cell.baseline.passRate)} at ${usd(cell.baseline.totalCostUsd)}; treatment tier ${String(cell.treatmentTier)}${cell.recommended ? "" : " (no recommendation)"} ${percent(cell.treatment.passRate)} at ${usd(cell.treatment.totalCostUsd)}; ${cell.passed ? "pass" : "fail"} (n=${String(cell.baseline.n)})`);
+	if (report.criterion2 !== void 0) {
+		const c2 = report.criterion2;
+		lines.push("", `Criterion 2 (agentType selection): ${c2.passed ? "holds" : "fails"} (baseline ${percent(c2.baseline.passRate)} at ${usd(c2.baseline.totalCostUsd)}; card-informed ${percent(c2.informed.passRate)} at ${usd(c2.informed.totalCostUsd)}; n=${String(c2.baseline.n)})`);
+	} else lines.push("", "Criterion 2 (agentType selection): NOT MEASURED (counts as failed)");
+	return lines.join("\n");
+}
+//#endregion
 //#region src/sweeps.ts
 const SWEEP_THRESHOLD_DEFAULTS = {
 	strength: .9,
@@ -485,4 +638,4 @@ async function runSweepMatrix(pool, options) {
 	return report;
 }
 //#endregion
-export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix };
+export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/testing": "1.0.0",
-    "@rulvar/core": "1.0.0"
+    "@rulvar/core": "1.2.0",
+    "@rulvar/testing": "1.2.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",