@rulvar/evals 1.243.0 → 1.244.0

package/dist/index.d.ts

@@ -849,9 +849,14 @@ declare function runFaultInjection(options?: RunFaultInjectionOptions): Promise<
 * figure whose premise contradicts the declared input (2,000 slots
 * computed from a 30 minute window where the input declares a 20
 * minute burst), and 'cost-basis' prints a locally estimated total as
-* the provider's bill.
+* the provider's bill. The fourth comparison experiment named the
+* decisive one (RV3909): 'stale-doctrine-echo' is a draft echoing a
+* DOCUMENTED doctrine while the pool holds the diverging source fact,
+* both sides cited; the experiment's answer echoed the retired
+* budget-immutability wording from a guide six weeks stale, and no
+* judge could flag it because the pool never carried the source side.
 */
-type ClaimCorpusClass = "live-fact" | "package-identity" | "inverted-default" | "numeric-range" | "negation" | "bounded-coverage" | "modality-overclaim" | "scope-ambiguity" | "bound-conflation" | "derived-premise" | "cost-basis";
+type ClaimCorpusClass = "live-fact" | "package-identity" | "inverted-default" | "numeric-range" | "negation" | "bounded-coverage" | "modality-overclaim" | "scope-ambiguity" | "bound-conflation" | "derived-premise" | "cost-basis" | "stale-doctrine-echo";
 /** One adversarial case: a draft, its contradicting evidence, and the mechanical expectations. */
 interface ClaimCorpusCase {
   id: string;

package/dist/index.js

@@ -3950,12 +3950,13 @@ const sectionalRepairRound = {
 		const { compositions, judges } = tailSpans(entries);
 		const verdictRows = entries.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finish_validation").map((entry) => entry.value);
 		const roundPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
-		const sectionalPrompt = roundPrompts.length === 2 && roundPrompts[0]?.includes("SECTIONAL ROUND") === false && roundPrompts[1]?.includes("SECTIONAL ROUND") === true && roundPrompts[1]?.includes("\"## Verdict\"") === true;
+		const sectionalPrompt = roundPrompts.length === 2 && roundPrompts[0]?.includes("SECTIONAL ROUND") === false && roundPrompts[1]?.includes("SECTIONAL ROUND") === true && roundPrompts[1]?.includes("\"## Verdict\"") === true && roundPrompts[0]?.includes("retained evidence discipline") === false && roundPrompts[1]?.includes("retained evidence discipline") === true;
 		const byteIdentity = value?.result === SECTIONAL_SPLICED && String(value.result).startsWith(SECTIONAL_PREFIX);
+		const lineage = value?.claimConsistencyMeta?.passes === 2 && value.claimConsistencyMeta.firstPassFindings === 1 && value.claimConsistencyMeta.semanticRepairRounds === 1;
 		return {
 			observation: {
-				matched: outcome.status === "ok" && byteIdentity && value.claimConsistencyMeta?.findings === 0 && verdictRows.map((row) => row.verdict).join(",") === "accepted,accepted" && verdictRows.map((row) => String(row.repairsUsed)).join(",") === "0,0" && sectionalPrompt && compositions.length === 2 && judges.length === 2,
-				detail: `run '${outcome.status}' shipped ${value?.result === SECTIONAL_SPLICED ? "the spliced whole" : "an unexpected document"}; byte identity=${String(byteIdentity)}; sectional prompt=${String(sectionalPrompt)}; verdicts [${verdictRows.map((row) => row.verdict).join(",")}]; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
+				matched: outcome.status === "ok" && byteIdentity && value.claimConsistencyMeta?.findings === 0 && lineage && verdictRows.map((row) => row.verdict).join(",") === "accepted,accepted" && verdictRows.map((row) => String(row.repairsUsed)).join(",") === "0,0" && sectionalPrompt && compositions.length === 2 && judges.length === 2,
+				detail: `run '${outcome.status}' shipped ${value?.result === SECTIONAL_SPLICED ? "the spliced whole" : "an unexpected document"}; byte identity=${String(byteIdentity)}; sectional prompt=${String(sectionalPrompt)}; lineage=${String(lineage)}; verdicts [${verdictRows.map((row) => row.verdict).join(",")}]; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
 			},
 			artifacts: [jsonArtifact("outcome.json", {
 				status: outcome.status,
@@ -4146,10 +4147,11 @@ const SCENARIOS = [
 			const judgeSawPatched = adapter.calls.filter((call) => call.label === "claim-consistency-judge-final").map((call) => call.prompt)[0]?.includes(`(run ${PATCH_RUN_ID})`) === true;
 			const roundPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
 			const lessonCarried = roundPrompts.length === 2 && roundPrompts[0]?.includes("HOST VALIDATION LESSONS") === false && roundPrompts[1]?.includes("HOST VALIDATION LESSONS") === true && roundPrompts[1]?.includes("evidence-grade") === true;
+			const aggregate = value?.deterministicPatches?.decisions === 1 && typeof value.deterministicPatches.patches === "number" && value.deterministicPatches.patches >= 1 && /^[0-9a-f]{64}$/u.test(String(value.deterministicPatches.lastAfterHash)) && value.deterministicPatches.lastAfterHash !== value.deterministicPatches.lastBeforeHash;
 			return {
 				observation: {
-					matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && value.claimConsistencyMeta?.findings === 0 && verdicts === "accepted,accepted" && pools === "0,0" && patch?.outcome === "accepted" && patch.healed?.map((healed) => healed.name).join(",") === "evidence-grade" && judgeSawPatched && lessonCarried && compositions.length === 2 && judges.length === 2,
-					detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the clean round document" : "an unexpected document"}; verdicts [${verdicts}], repairsUsed [${pools}] (no wire, no pool spent); patch outcome='${String(patch?.outcome)}'; judge saw patched=${String(judgeSawPatched)}; lesson carried=${String(lessonCarried)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
+					matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && value.claimConsistencyMeta?.findings === 0 && aggregate && verdicts === "accepted,accepted" && pools === "0,0" && patch?.outcome === "accepted" && patch.healed?.map((healed) => healed.name).join(",") === "evidence-grade" && judgeSawPatched && lessonCarried && compositions.length === 2 && judges.length === 2,
+					detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the clean round document" : "an unexpected document"}; verdicts [${verdicts}], repairsUsed [${pools}] (no wire, no pool spent); patch outcome='${String(patch?.outcome)}'; envelope aggregate=${String(aggregate)}; judge saw patched=${String(judgeSawPatched)}; lesson carried=${String(lessonCarried)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
 				},
 				artifacts: [jsonArtifact("outcome.json", {
 					status: outcome.status,
@@ -4159,7 +4161,106 @@ const SCENARIOS = [
 			};
 		}
 	},
-	claimJudgeDeadArmedRefusal
+	claimJudgeDeadArmedRefusal,
+	{
+		name: "acceptance-reserve-refusal",
+		doctrine: "budget.acceptanceReserve 'require' refuses the run typed BEFORE the first wire when the declared acceptance tail (synthesis hold + judge passes + mechanical repair + round composition + working room) does not fit the effective cap, and the refusal journals every term; the same config under the default warn posture dispatches",
+		async run() {
+			const store = new InMemoryStore();
+			const adapter = new FakeAdapter({ agents: { "*": "never dispatched" } });
+			const outcome = await createEngine({
+				adapters: [adapter],
+				stores: { journal: store },
+				defaults: { routing: {
+					loop: FAKE_MODEL_REF,
+					orchestrate: FAKE_MODEL_REF,
+					synthesize: FAKE_MODEL_REF,
+					extract: FAKE_MODEL_REF
+				} }
+			}).run(makeOrchestratorWorkflow("audit the executor", {
+				synthesis: { estCost: .15 },
+				claimConsistency: {
+					stage: "final",
+					onFound: "repair",
+					judge: { estCost: .2 }
+				},
+				finishValidation: {
+					validators: [{
+						name: "anything",
+						validate: () => ({ ok: true })
+					}],
+					estRepairCostUsd: .1
+				},
+				budget: {
+					synthesisReserveUsd: 1,
+					acceptanceReserve: "require"
+				}
+			}), void 0, {
+				runId: "fault-acceptance-reserve",
+				budgetUsd: 10
+			}).result;
+			const refusal = (await store.load("fault-acceptance-reserve")).find((entry) => entry.value?.decisionType === "acceptance_reserve_refused");
+			const terms = refusal?.value;
+			return {
+				observation: {
+					matched: outcome.status === "error" && (outcome.error?.message ?? "").includes("acceptanceReserve 'require'") && adapter.calls.length === 0 && terms !== void 0 && Math.abs((terms.requiredUsd ?? 0) - 2.15) < 1e-9 && Math.abs((terms.effectiveCapUsd ?? 0) - 2) < 1e-9 && terms.judgePasses === 2,
+					detail: `run '${outcome.status}'; dispatches=${String(adapter.calls.length)}; refusal decision=${String(refusal !== void 0)} (required ${String(terms?.requiredUsd)} vs cap ${String(terms?.effectiveCapUsd)}, ` + `judge passes ${String(terms?.judgePasses)}); ${outcome.error?.message ?? ""}`.slice(0, 400)
+				},
+				artifacts: [jsonArtifact("refusal.json", {
+					error: outcome.error ?? null,
+					decision: refusal?.value ?? null
+				})]
+			};
+		}
+	},
+	{
+		name: "budget-policy-immutable",
+		doctrine: "budgetPolicy 'immutable-lifetime' refuses a resume-time ceiling override typed before ownership, raise and lower alike, with zero provider dispatches and zero durable mutations; a bare resume of the same run stays a pure replay",
+		async run() {
+			const store = new InMemoryStore();
+			const seeded = await createEngine({
+				adapters: [new FakeAdapter({ agents: { "*": "answered" } })],
+				stores: { journal: store },
+				defaults: ROUTING
+			}).run(echoWorkflow, void 0, {
+				runId: "fault-budget-policy",
+				budgetUsd: 1,
+				budgetPolicy: "immutable-lifetime"
+			}).result;
+			const entriesBefore = (await store.load("fault-budget-policy")).length;
+			const overrideAdapter = new FakeAdapter({ agents: { "*": "never dispatched" } });
+			let refusalText = "the override was not refused";
+			let refusedTyped = false;
+			try {
+				await createEngine({
+					adapters: [overrideAdapter],
+					stores: { journal: store },
+					defaults: ROUTING
+				}).resume("fault-budget-policy", echoWorkflow, { run: { budgetUsd: 2 } }).result;
+			} catch (thrown) {
+				refusalText = errorText(thrown);
+				refusedTyped = thrown instanceof ConfigError && refusalText.includes("immutable-lifetime");
+			}
+			const entriesAfterRefusal = (await store.load("fault-budget-policy")).length;
+			const metaAfterRefusal = await store.getMeta("fault-budget-policy");
+			const bare = await createEngine({
+				adapters: [new FakeAdapter({ agents: { "*": "never dispatched either" } })],
+				stores: { journal: store },
+				defaults: ROUTING
+			}).resume("fault-budget-policy", echoWorkflow).result;
+			return {
+				observation: {
+					matched: seeded.status === "ok" && refusedTyped && overrideAdapter.calls.length === 0 && entriesAfterRefusal === entriesBefore && metaAfterRefusal?.budgetUsd === 1 && metaAfterRefusal.budgetPolicy === "immutable-lifetime" && bare.status === "ok",
+					detail: `seed '${seeded.status}'; refusal typed=${String(refusedTyped)} (${refusalText.slice(0, 160)}); override dispatches=${String(overrideAdapter.calls.length)}; entries ${String(entriesBefore)} -> ${String(entriesAfterRefusal)}; meta budgetUsd=${String(metaAfterRefusal?.budgetUsd)} policy=${String(metaAfterRefusal?.budgetPolicy)}; bare resume '${bare.status}'`
+				},
+				artifacts: [jsonArtifact("refusal.json", {
+					refusalText,
+					entriesBefore,
+					entriesAfterRefusal
+				}), jsonArtifact("meta.json", metaAfterRefusal ?? null)]
+			};
+		}
+	}
 ];
 /** The scenario names in run order. */
 const FAULT_SCENARIO_NAMES = SCENARIOS.map((scenario) => scenario.name);
@@ -4377,6 +4478,21 @@ const CLAIM_CORPUS = [
 		},
 		runFactTerms: ["provider bill", "charged and settled"],
 		expect: { minRunFactPairs: 1 }
+	},
+	{
+		id: "stale-doctrine-echo-budget-override",
+		class: "stale-doctrine-echo",
+		draft: "The run budget ceiling is documented as immutable after start: resume accepts no budget parameter, so an exhausted run can never continue under a raised ceiling (docs/guide/budgets.md:43; packages/core/src/engine/engine.ts:588).",
+		pool: [{
+			nodeId: "agent:3",
+			text: "The budgets guide historically taught \"immutable after start\" (docs/guide/budgets.md:43), but ResumeOptions.run raises budgetUsd and maxInFlightExposureUsd at resume, validated and journaled as a run_budget_override decision with a typed floor at the settled spend (packages/core/src/engine/engine.ts:588-604); the doctrine in force is immutability WITHIN a segment."
+		}],
+		critical: ["packages/core/src/engine/engine.ts"],
+		expect: {
+			minPairs: 1,
+			anchors: ["packages/core/src/engine/engine.ts:588"],
+			coverage: "full"
+		}
 	}
 ];
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.243.0",
+  "version": "1.244.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,11 +22,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.243.0",
-    "@rulvar/openai": "1.243.0",
-    "@rulvar/testing": "1.243.0",
-    "@rulvar/anthropic": "1.243.0",
-    "@rulvar/plan": "1.243.0"
+    "@rulvar/anthropic": "1.244.0",
+    "@rulvar/openai": "1.244.0",
+    "@rulvar/core": "1.244.0",
+    "@rulvar/plan": "1.244.0",
+    "@rulvar/testing": "1.244.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",