@rulvar/evals 1.1.0 → 1.3.0

package/dist/index.d.ts

@@ -2,7 +2,7 @@ import { CompiledWorkflow, DeclaredLadder, Effort, Engine, EvidenceRef, Json, Js
 
 //#region src/case.d.ts
 /**
-* One quality-measurement case (docs/09, section 7.1). The shape is the
+* One quality-measurement case. The shape is the
 * documented interface verbatim; display names derive from the workflow
 * name (the suite runner disambiguates duplicates by ordinal).
 */
@@ -24,7 +24,7 @@ interface GraderVerdict {
 /**
 * A judge invocation specification. The judge runs through the engine as
 * an ordinary journaled, budgeted invocation; model selection is subject
-* to the router quality floors (docs/04, section "Role quality floors"),
+* to the router quality floors,
 * and @rulvar/evals ships NO default judge model: weak defaults for
 * judging are forbidden, so the model is always explicit.
 */
@@ -65,7 +65,7 @@ interface EvalCaseResult {
   judgeCostUsd: number;
   /**
   * run:start to run:end of the target run, from event timestamps; no
-  * separate measurement channel exists (docs/09, section 7.2).
+  * separate measurement channel exists.
   */
   latencyMs: number;
   /** The target run's normalized usage. */
@@ -162,7 +162,7 @@ declare function rubricGrader(criteria: RubricCriterion[], options?: RubricGrade
 /** The default judge verdict shape. */
 declare const JUDGE_VERDICT_SCHEMA: JsonSchema;
 interface JudgeGraderOptions {
-  /** Judge model; required, never defaulted (docs/04 role quality floors). */
+  /** Judge model; required, never defaulted (role quality floors). */
   model: ModelSpec;
   /** What to judge: the criteria prose embedded into the judge prompt. */
   instruction: string;
@@ -213,10 +213,10 @@ interface EvalCommitterOptions {
   committerId: string;
   /** The emitting sweep report; every claim's gate references it. */
   reportId: string;
-  /** CAS-rebase attempts (docs/05, 5.4); default 3. */
+  /** CAS-rebase attempts; default 3. */
   attempts?: number;
 }
-/** One measured claim, TTL applied per the docs/05 decay table. */
+/** One measured claim; claimExpiry applies the TTL from the decay table. */
 declare function evalMeasuredClaim(input: MeasuredClaimInput, committerId: string): ModelClaim;
 /**
 * Commits measured claims through the eval-committer gate with the
@@ -250,9 +250,8 @@ interface CanaryDriftReport {
 }
 /**
 * Flips the model's ACTIVE eval-measured claims to stale when their
-* recorded canary fingerprint differs from the fresh one (docs/05:
-* "a fingerprint change immediately flips the model's eval claims to
-* stale"). Claims without a recorded fingerprint have no baseline and
+* recorded canary fingerprint differs from the fresh one. Claims
+* without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
 * an idempotent noop. CAS-rebased like every maintenance commit.
 */
@@ -284,7 +283,7 @@ interface SweepThresholds {
 interface RunSweepOptions {
   /** Deterministic, caller-minted; every claim's evidence and gate reference it. */
   reportId: string;
-  /** The dedicated identity (docs/05, 5.4). */
+  /** The dedicated committer identity. */
   committerId: string;
   /** ISO date of the sweep; the TTL table applies from it (no wall clock inside). */
   observedAt: string;
@@ -364,7 +363,7 @@ interface RunCheckpointOptions {
   suite?: RunEvalSuiteOptions;
   /**
   * Orchestrated runs need room for the orchestrator cap math (the
-  * run ceiling must host the finalize reserve; docs/07, 12.2): their
+  * run ceiling must host the finalize reserve): their
   * suite options default to `suite` but usually carry a larger
   * budgetUsd.
   */

package/dist/index.js

@@ -6,9 +6,9 @@ import { createHash } from "node:crypto";
 * suite runners. A separate quality-measurement package built strictly on
 * the public APIs (L6).
 *
-* Owning spec: docs/09-observability-testing-spec.md, section
-* "@rulvar/evals"; determinism rules in docs/11-testing-strategy.md,
-* section "Eval CI": judge graders run THROUGH the engine, so judge calls
+* Determinism rules (https://docs.rulvar.com/guide/evals and
+* https://docs.rulvar.com/guide/testing): judge graders run THROUGH the
+* engine, so judge calls
 * are journaled, budgeted, and VCR-recordable, and an eval suite replays
 * deterministically from cassettes with zero live calls.
 */
@@ -254,14 +254,14 @@ function judgeGrader(options) {
 //#endregion
 //#region src/committer.ts
 /**
-* The eval-committer identity (M11-T01; docs/05, sections "Data model"
-* and "Commit discipline"). The pipeline-side commit path: builds
+* The eval-committer identity (M11-T01; https://docs.rulvar.com/guide/model-knowledge).
+* The pipeline-side commit path: builds
 * eval-committer-gated ops (the coherence square: class eval-measured,
 * author eval-pipeline, metrics present) and commits them with the
 * documented CAS-rebase recipe. Humans never call this; their path is
 * the human gate and it structurally cannot carry metrics.
 */
-/** One measured claim, TTL applied per the docs/05 decay table. */
+/** One measured claim; claimExpiry applies the TTL from the decay table. */
 function evalMeasuredClaim(input, committerId) {
 	return {
 		id: input.id,
@@ -318,8 +318,7 @@ async function commitEvalMeasured(store, claims, options) {
 //#endregion
 //#region src/canary.ts
 /**
-* The canary fingerprint (M11-T04; docs/05, section "Grounding and
-* decay"; OQ-06). The optional compensation for silent alias
+* The canary fingerprint (M11-T04; OQ-06). The optional compensation for silent alias
 * re-pointing that modelEpoch honestly cannot catch: a FIXED probe set
 * at temperature 0, run through the ordinary engine (journaled,
 * budgeted, VCR-recordable), hashed over normalized outputs. A
@@ -353,9 +352,8 @@ async function canaryFingerprint(engine, probes) {
 }
 /**
 * Flips the model's ACTIVE eval-measured claims to stale when their
-* recorded canary fingerprint differs from the fresh one (docs/05:
-* "a fingerprint change immediately flips the model's eval claims to
-* stale"). Claims without a recorded fingerprint have no baseline and
+* recorded canary fingerprint differs from the fresh one. Claims
+* without a recorded fingerprint have no baseline and
 * stay untouched (the documented no-probe posture); a second run is
 * an idempotent noop. CAS-rebased like every maintenance commit.
 */
@@ -396,9 +394,8 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 //#endregion
 //#region src/checkpoint.ts
 /**
-* The phases 1-2 measured-value checkpoint (M12-T01; docs/05, section
-* "Phases and placement"; the quantitative criteria of OQ-09,
-* 14-open-questions.md, closed at M11-T06). The M12 gate: kb_propose
+* The phases 1-2 measured-value checkpoint (M12-T01; the quantitative
+* criteria of OQ-09, closed at M11-T06). The M12 gate: kb_propose
 * and the proposal loop ship ONLY if the knowledge card demonstrably
 * improves tier and agentType selection on eval cases.
 *
@@ -418,7 +415,11 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 *    with and without the knowledge store configured (the card docks
 *    into the spawn tool description when configured). Criterion 2
 *    holds when the card-informed arm matches or beats the baseline
-*    pass rate at no more than 105 percent of its cost.
+*    pass rate at no more than 105 percent of its cost, OR beats it
+*    by at least 15 points at no more than 115 percent of its cost
+*    (the quality branch; OQ-09 as amended 2026-07-12: the baseline
+*    fails CHEAPLY, so the flat cost bar tightened exactly when the
+*    card was winning on quality).
 *
 * The checkpoint PASSES only when both criteria hold. Methodology
 * guard: the claims the treatment consumes MUST come from a seeding
@@ -427,6 +428,18 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
 */
 /** IEEE754 guard for the rule boundaries (0.8 + 0.05 exceeds 0.85). */
 const EPSILON = 1e-9;
+/**
+* The OQ-09 criterion 2 rule (as amended 2026-07-12): match-or-beat at
+* 105 percent of baseline cost, OR at least 15 points better at 115
+* percent (the quality branch: the baseline fails cheaply, so the flat
+* bar tightened exactly when the card won on quality). The vacuous-pass
+* guard stays with the caller.
+*/
+function agentTypeRuleHolds(baseline, informed) {
+	const matchesCheaply = informed.passRate >= baseline.passRate - EPSILON && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON;
+	const clearlyBetterNearCost = informed.passRate >= baseline.passRate + .15 - EPSILON && informed.totalCostUsd <= 1.15 * baseline.totalCostUsd + EPSILON;
+	return matchesCheaply || clearlyBetterNearCost;
+}
 /** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
 function rungRuleHolds(baseline, treatment) {
 	const equalOrBetterCheaper = treatment.passRate >= baseline.passRate - EPSILON && treatment.totalCostUsd <= .9 * baseline.totalCostUsd + EPSILON;
@@ -507,7 +520,7 @@ async function runValueCheckpoint(checkpointPool, options) {
 		criterion2 = {
 			baseline,
 			informed,
-			passed: informed.n > 0 && informed.passRate > 0 && informed.passRate >= baseline.passRate && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON
+			passed: informed.n > 0 && informed.passRate > 0 && agentTypeRuleHolds(baseline, informed)
 		};
 	}
 	return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.1.0",
-    "@rulvar/testing": "1.1.0"
+    "@rulvar/core": "1.3.0",
+    "@rulvar/testing": "1.3.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",