@rulvar/evals 1.0.0 → 1.1.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { CompiledWorkflow, Effort, Engine, EvidenceRef, Json, JsonSchema, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow } from "@rulvar/core";
+import { CompiledWorkflow, DeclaredLadder, Effort, Engine, EvidenceRef, Json, JsonSchema, KnowledgeSnapshot, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow } from "@rulvar/core";
 
 //#region src/case.d.ts
 /**
@@ -332,4 +332,90 @@ declare const SWEEP_THRESHOLD_DEFAULTS: SweepThresholds;
 */
 declare function runSweepMatrix(pool: SweepPool, options: RunSweepOptions): Promise<SweepReport>;
 //#endregion
-export { type CanaryDriftReport, type CanaryProbeSet, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type RubricCriterion, type RubricGraderOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix };
+//#region src/checkpoint.d.ts
+/** One declared checkpoint ladder: rungs are concrete pool members. */
+interface CheckpointLadder extends DeclaredLadder {
+  name: string;
+  startTier: number;
+  rungs: SweepModel[];
+}
+interface CheckpointPool {
+  ladders: CheckpointLadder[];
+  /** The measurement half; the seeding sweep MUST NOT have seen these. */
+  evalCases: SweepCase[];
+}
+interface OrchestratedCase {
+  /** The workflow drives an orchestrate-role run; graders judge its outcome. */
+  case: EvalCase;
+}
+interface RunCheckpointOptions {
+  /** The claims snapshot produced by the seeding sweep (disjoint cases). */
+  snapshot: KnowledgeSnapshot;
+  /** ISO date of the evaluation (recorded in the report; no wall clock inside). */
+  observedAt: string;
+  /** An engine per concrete pool member (the caller owns adapters and budgets). */
+  engineFor: (member: SweepModel) => Engine | Promise<Engine>;
+  /**
+  * Criterion 2 engines: withKnowledge true configures the SAME store
+  * snapshot behind stores.modelKnowledge; false omits it entirely.
+  */
+  orchestrateEngineFor?: (withKnowledge: boolean) => Engine | Promise<Engine>;
+  orchestratedCases?: OrchestratedCase[];
+  suite?: RunEvalSuiteOptions;
+  /**
+  * Orchestrated runs need room for the orchestrator cap math (the
+  * run ceiling must host the finalize reserve; docs/07, 12.2): their
+  * suite options default to `suite` but usually carry a larger
+  * budgetUsd.
+  */
+  orchestratedSuite?: RunEvalSuiteOptions;
+}
+interface CheckpointArm {
+  passRate: number;
+  totalCostUsd: number;
+  n: number;
+}
+interface CheckpointCell {
+  ladder: string;
+  taskClass: TaskClass;
+  defaultTier: number;
+  /** The tier the treatment arm ran at (default when no recommendation). */
+  treatmentTier: number;
+  recommended: boolean;
+  baseline: CheckpointArm;
+  treatment: CheckpointArm;
+  passed: boolean;
+}
+interface CriterionOneReport {
+  cells: CheckpointCell[];
+  cellsPassed: number;
+  majorityHolds: boolean;
+  pooledBaseline: CheckpointArm;
+  pooledTreatment: CheckpointArm;
+  pooledHolds: boolean;
+  passed: boolean;
+}
+interface CriterionTwoReport {
+  baseline: CheckpointArm;
+  informed: CheckpointArm;
+  passed: boolean;
+}
+interface CheckpointReport {
+  observedAt: string;
+  criterion1: CriterionOneReport;
+  criterion2?: CriterionTwoReport;
+  /** Both criteria (criterion 2 counts as failed when unmeasured). */
+  passed: boolean;
+}
+/** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
+declare function rungRuleHolds(baseline: CheckpointArm, treatment: CheckpointArm): boolean;
+/**
+* Runs the checkpoint over the fixed pool. Sequential in declaration
+* order (deterministic cassette consumption when recorded); every cell
+* runs baseline then treatment.
+*/
+declare function runValueCheckpoint(checkpointPool: CheckpointPool, options: RunCheckpointOptions): Promise<CheckpointReport>;
+/** The deterministic render for the M12 gate docs amendment. */
+declare function renderCheckpointReport(report: CheckpointReport): string;
+//#endregion
+export { type CanaryDriftReport, type CanaryProbeSet, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { KnowledgeCasError, claimExpiry, defineWorkflow } from "@rulvar/core";
+import { KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow } from "@rulvar/core";
 import { createHash } from "node:crypto";
 //#region src/case.ts
 /**
@@ -394,6 +394,146 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 	throw lastCas ?? /* @__PURE__ */ new Error("flipStaleOnCanaryDrift: unreachable");
 }
 //#endregion
+//#region src/checkpoint.ts
+/**
+* The phases 1-2 measured-value checkpoint (M12-T01; docs/05, section
+* "Phases and placement"; the quantitative criteria of OQ-09,
+* 14-open-questions.md, closed at M11-T06). The M12 gate: kb_propose
+* and the proposal loop ship ONLY if the knowledge card demonstrably
+* improves tier and agentType selection on eval cases.
+*
+* Two experiments, both A/B under identical fixed pools:
+*
+* 1. RUNG SELECTION, per (ladder, taskClass) cell: the baseline arm
+*    runs every eval case at the ladder's DEFAULT start tier; the
+*    treatment arm runs at the tier recommended by
+*    compileVerifiedLayer over the store's claims (default when no
+*    recommendation). A cell passes when the treatment reaches a pass
+*    rate at least equal to the baseline at no more than 90 percent
+*    of its cost, OR at least 5 points above it at no more than its
+*    cost. Criterion 1 holds when a MAJORITY of cells pass AND the
+*    pooled aggregate passes the same rule.
+*
+* 2. AGENTTYPE SELECTION, pooled: the same orchestrate-role cases run
+*    with and without the knowledge store configured (the card docks
+*    into the spawn tool description when configured). Criterion 2
+*    holds when the card-informed arm matches or beats the baseline
+*    pass rate at no more than 105 percent of its cost.
+*
+* The checkpoint PASSES only when both criteria hold. Methodology
+* guard: the claims the treatment consumes MUST come from a seeding
+* sweep over a DISJOINT case set (the seed/eval split is the caller's
+* pool contract), or the measurement is leakage.
+*/
+/** IEEE754 guard for the rule boundaries (0.8 + 0.05 exceeds 0.85). */
+const EPSILON = 1e-9;
+/** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
+function rungRuleHolds(baseline, treatment) {
+	const equalOrBetterCheaper = treatment.passRate >= baseline.passRate - EPSILON && treatment.totalCostUsd <= .9 * baseline.totalCostUsd + EPSILON;
+	const clearlyBetterAtCost = treatment.passRate >= baseline.passRate + .05 - EPSILON && treatment.totalCostUsd <= baseline.totalCostUsd + EPSILON;
+	return equalOrBetterCheaper || clearlyBetterAtCost;
+}
+function armOf(suite) {
+	return {
+		passRate: suite.passRate,
+		totalCostUsd: suite.totalCostUsd,
+		n: suite.results.length
+	};
+}
+function pool(arms) {
+	const n = arms.reduce((sum, arm) => sum + arm.n, 0);
+	const passed = arms.reduce((sum, arm) => sum + arm.passRate * arm.n, 0);
+	const cost = arms.reduce((sum, arm) => sum + arm.totalCostUsd, 0);
+	return {
+		passRate: n === 0 ? 0 : passed / n,
+		totalCostUsd: cost,
+		n
+	};
+}
+/**
+* Runs the checkpoint over the fixed pool. Sequential in declaration
+* order (deterministic cassette consumption when recorded); every cell
+* runs baseline then treatment.
+*/
+async function runValueCheckpoint(checkpointPool, options) {
+	const recommendations = compileVerifiedLayer(options.snapshot.claims.filter((claim) => claim.status === "active"), checkpointPool.ladders);
+	const byTaskClass = /* @__PURE__ */ new Map();
+	for (const entry of checkpointPool.evalCases) {
+		const bucket = byTaskClass.get(entry.taskClass) ?? [];
+		bucket.push(entry.case);
+		byTaskClass.set(entry.taskClass, bucket);
+	}
+	const cells = [];
+	for (const ladder of checkpointPool.ladders) for (const [taskClass, cases] of byTaskClass) {
+		const recommendation = recommendations.find((row) => row.ladder === ladder.name && row.taskClass === taskClass);
+		const treatmentTier = recommendation?.recommendedTier ?? ladder.startTier;
+		const baseMember = ladder.rungs[ladder.startTier];
+		const treatMember = ladder.rungs[treatmentTier];
+		if (baseMember === void 0 || treatMember === void 0) throw new Error(`checkpoint: ladder '${ladder.name}' lacks rung ${String(treatmentTier)}`);
+		const baseline = armOf(await runEvalSuite(await options.engineFor(baseMember), cases, options.suite ?? {}));
+		const treatment = treatmentTier === ladder.startTier ? baseline : armOf(await runEvalSuite(await options.engineFor(treatMember), cases, options.suite ?? {}));
+		cells.push({
+			ladder: ladder.name,
+			taskClass,
+			defaultTier: ladder.startTier,
+			treatmentTier,
+			recommended: recommendation !== void 0,
+			baseline,
+			treatment,
+			passed: rungRuleHolds(baseline, treatment)
+		});
+	}
+	const recommendedCells = cells.filter((cell) => cell.recommended);
+	const cellsPassed = recommendedCells.filter((cell) => cell.passed).length;
+	const majorityHolds = recommendedCells.length > 0 && cellsPassed * 2 > recommendedCells.length;
+	const pooledBaseline = pool(cells.map((cell) => cell.baseline));
+	const pooledTreatment = pool(cells.map((cell) => cell.treatment));
+	const pooledHolds = rungRuleHolds(pooledBaseline, pooledTreatment);
+	const criterion1 = {
+		cells,
+		cellsPassed,
+		majorityHolds,
+		pooledBaseline,
+		pooledTreatment,
+		pooledHolds,
+		passed: majorityHolds && pooledHolds
+	};
+	let criterion2;
+	if (options.orchestrateEngineFor !== void 0 && options.orchestratedCases !== void 0) {
+		const cases = options.orchestratedCases.map((entry) => entry.case);
+		const orchestratedSuite = options.orchestratedSuite ?? options.suite ?? {};
+		const baseline = armOf(await runEvalSuite(await options.orchestrateEngineFor(false), cases, orchestratedSuite));
+		const informed = armOf(await runEvalSuite(await options.orchestrateEngineFor(true), cases, orchestratedSuite));
+		criterion2 = {
+			baseline,
+			informed,
+			passed: informed.n > 0 && informed.passRate > 0 && informed.passRate >= baseline.passRate && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON
+		};
+	}
+	return {
+		observedAt: options.observedAt,
+		criterion1,
+		...criterion2 === void 0 ? {} : { criterion2 },
+		passed: criterion1.passed && criterion2 !== void 0 && criterion2.passed
+	};
+}
+const percent = (rate) => `${(rate * 100).toFixed(1)}%`;
+const usd = (value) => `$${value.toFixed(4)}`;
+/** The deterministic render for the M12 gate docs amendment. */
+function renderCheckpointReport(report) {
+	const lines = [
+		`Measured-value checkpoint (OQ-09) at ${report.observedAt}: ` + (report.passed ? "PASSED" : "FAILED"),
+		"",
+		`Criterion 1 (rung selection): ${report.criterion1.passed ? "holds" : "fails"} (${String(report.criterion1.cellsPassed)}/${String(report.criterion1.cells.length)} cells, pooled ${report.criterion1.pooledHolds ? "holds" : "fails"})`
+	];
+	for (const cell of report.criterion1.cells) lines.push(`* ${cell.ladder} :: ${cell.taskClass}: baseline tier ${String(cell.defaultTier)} ${percent(cell.baseline.passRate)} at ${usd(cell.baseline.totalCostUsd)}; treatment tier ${String(cell.treatmentTier)}${cell.recommended ? "" : " (no recommendation)"} ${percent(cell.treatment.passRate)} at ${usd(cell.treatment.totalCostUsd)}; ${cell.passed ? "pass" : "fail"} (n=${String(cell.baseline.n)})`);
+	if (report.criterion2 !== void 0) {
+		const c2 = report.criterion2;
+		lines.push("", `Criterion 2 (agentType selection): ${c2.passed ? "holds" : "fails"} (baseline ${percent(c2.baseline.passRate)} at ${usd(c2.baseline.totalCostUsd)}; card-informed ${percent(c2.informed.passRate)} at ${usd(c2.informed.totalCostUsd)}; n=${String(c2.baseline.n)})`);
+	} else lines.push("", "Criterion 2 (agentType selection): NOT MEASURED (counts as failed)");
+	return lines.join("\n");
+}
+//#endregion
 //#region src/sweeps.ts
 const SWEEP_THRESHOLD_DEFAULTS = {
 	strength: .9,
@@ -485,4 +625,4 @@ async function runSweepMatrix(pool, options) {
 	return report;
 }
 //#endregion
-export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix };
+export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/testing": "1.0.0",
-    "@rulvar/core": "1.0.0"
+    "@rulvar/core": "1.1.0",
+    "@rulvar/testing": "1.1.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",