@rulvar/evals 1.240.0 → 1.241.0

package/dist/index.js

@@ -1,5 +1,5 @@
 import { createRequire } from "node:module";
-import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, citedValueValidator, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, evidenceGradeValidator, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
+import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, citedValueValidator, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, evidenceGradeValidator, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, minMatchesValidator, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
 import { createHash } from "node:crypto";
 import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -3492,6 +3492,8 @@ const TAIL_DRAFT_INVERTED = "draft: an audit-write failure does not turn success
 const TAIL_FINAL_INVERTED = "final: an audit-write failure does not turn success into failure [src/exec.ts:256-296].";
 const TAIL_FINAL_STILL_INVERTED = "final: the repaired text still flips the recorded outcome [src/exec.ts:256-296].";
 const TAIL_FINAL_CLEAN = "final: a failed audit write does not mask success [src/exec.ts:256-296].";
+const TAIL_FINAL_UNGROUNDED = "final: repaired wording with the provenance stripped.";
+const TAIL_FINAL_UNGROUNDED_INITIAL = "final: the initial wording arrived without any anchor.";
 const TAIL_FINDS = { contradictions: [{
 	pair: 0,
 	reason: "the draft inverts the recorded reading"
@@ -3671,6 +3673,107 @@ const SCENARIOS = [
 			};
 		}
 	},
+	{
+		name: "repair-round-host-rejection",
+		doctrine: "a repair round that dispatched and lost its candidate to the finish contract fails typed as a host rejection (RV3601): the message names the dispatch, never 'could not dispatch'; data carries roundDispatched true, repairsUsed 1, the judge meta beside the findings, and the finish verdict verbatim (failed validators, candidateHash, candidateChars); the outcome lifts the findings beside the meta",
+		async run() {
+			const { engine, store } = tailEngine(tailAdapter({
+				judge: () => TAIL_FINDS,
+				finals: [
+					TAIL_FINAL_INVERTED,
+					TAIL_FINAL_UNGROUNDED,
+					TAIL_FINAL_UNGROUNDED
+				]
+			}));
+			const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
+				...TAIL_OPTS,
+				claimConsistency: {
+					stage: "final",
+					onFound: "repair"
+				},
+				finishValidation: {
+					validators: [minMatchesValidator({
+						pattern: "src/[a-z]+\\.ts:\\d+",
+						min: 1,
+						name: "provenance-anchor"
+					})],
+					maxRepairs: 1
+				}
+			}), void 0, {
+				runId: "fault-repair-host-rejection",
+				budgetUsd: 10
+			}).result;
+			const data = outcome.error?.data ?? {};
+			const entries = await store.load("fault-repair-host-rejection");
+			const { compositions, judges } = tailSpans(entries);
+			const liftedFindings = outcome.claimContradictions;
+			const message = outcome.error?.message ?? "";
+			return {
+				observation: {
+					matched: outcome.status === "error" && message.includes("dispatched and its repaired candidate failed host validation") && !message.includes("could not dispatch") && data.source === "orchestrator_claim_consistency" && data.roundDispatched === true && data.repairsUsed === 1 && data.claimConsistencyMeta?.findings === 1 && Array.isArray(data.claimContradictions) && data.claimContradictions.length === 1 && typeof data.finishValidation?.callId === "string" && data.finishValidation.repairsUsed === 1 && data.finishValidation.maxRepairs === 1 && typeof data.finishValidation.candidateHash === "string" && typeof data.finishValidation.candidateChars === "number" && JSON.stringify(data.finishValidation.failed ?? null).includes("provenance-anchor") && Array.isArray(liftedFindings) && liftedFindings.length === 1 && compositions.length === 2 && judges.length === 1,
+					detail: `run '${outcome.status}': roundDispatched=${String(data.roundDispatched)}, repairsUsed=${String(data.repairsUsed)}, meta findings=${String(data.claimConsistencyMeta?.findings)}, finish verdict carries hash=${String(typeof data.finishValidation?.candidateHash === "string")} chars=${String(data.finishValidation?.candidateChars)}, outcome lifts findings=${String(Array.isArray(liftedFindings))}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
+				},
+				artifacts: [jsonArtifact("outcome.json", {
+					status: outcome.status,
+					error: outcome.error ?? null
+				}), jsonArtifact("journal.json", entries)]
+			};
+		}
+	},
+	{
+		name: "repair-round-own-pool",
+		doctrine: "the mechanical repair pool belongs to one composition invocation (RV3602): the bounded claim repair round enters with the full maxRepairs even after the initial composition spent its own, so the frozen third comparison sequence converges to ok/complete with verdicts repair/accepted twice and repairsUsed restarting at the invocation boundary, never a host rejection born of an inherited spent pool; the round's prompt also carries the HOST VALIDATION LESSONS block naming the failure the run already paid for (RV3603), while the initial composition carries none",
+		async run() {
+			let judgeCalls = 0;
+			const adapter = tailAdapter({
+				judge: () => (judgeCalls += 1) === 1 ? TAIL_FINDS : TAIL_AGREES,
+				finals: [
+					TAIL_FINAL_UNGROUNDED_INITIAL,
+					TAIL_FINAL_INVERTED,
+					TAIL_FINAL_UNGROUNDED,
+					TAIL_FINAL_CLEAN
+				]
+			});
+			const { engine, store } = tailEngine(adapter);
+			const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
+				...TAIL_OPTS,
+				claimConsistency: {
+					stage: "final",
+					onFound: "repair"
+				},
+				finishValidation: {
+					validators: [minMatchesValidator({
+						pattern: "src/[a-z]+\\.ts:\\d+",
+						min: 1,
+						name: "provenance-anchor"
+					})],
+					maxRepairs: 1
+				}
+			}), void 0, {
+				runId: "fault-repair-own-pool",
+				budgetUsd: 10
+			}).result;
+			const value = outcome.value;
+			const entries = await store.load("fault-repair-own-pool");
+			const { compositions, judges } = tailSpans(entries);
+			const verdictRows = entries.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finish_validation").map((entry) => entry.value);
+			const verdicts = verdictRows.map((row) => row.verdict).join(",");
+			const pools = verdictRows.map((row) => String(row.repairsUsed)).join(",");
+			const compositionPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
+			const lessonCarried = compositionPrompts.length === 4 && compositionPrompts[0]?.includes("HOST VALIDATION LESSONS") === false && compositionPrompts[2]?.includes("HOST VALIDATION LESSONS") === true && compositionPrompts[2]?.includes("provenance-anchor") === true;
+			return {
+				observation: {
+					matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && value.claimConsistencyMeta?.findings === 0 && verdicts === "repair,accepted,repair,accepted" && pools === "0,1,0,1" && (value.rejectedFinishCandidates?.length ?? 0) === 2 && value.rejectedFinishCandidates?.every((row) => row.verdict === "repair") === true && lessonCarried && compositions.length === 2 && judges.length === 2,
+					detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the repaired document" : "an unexpected document"}; verdicts [${verdicts}], repairsUsed [${pools}] (fresh pool at the invocation boundary); lesson carried=${String(lessonCarried)}; meta findings=${String(value?.claimConsistencyMeta?.findings)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
+				},
+				artifacts: [jsonArtifact("outcome.json", {
+					status: outcome.status,
+					value: outcome.value ?? null,
+					envelope: outcome.envelope
+				}), jsonArtifact("journal.json", entries)]
+			};
+		}
+	},
 	{
 		name: "claim-judge-dead-armed-refusal",
 		doctrine: "a final stage judge that dies on the wire under an armed posture fails the run typed for both 'fail' and 'repair' (RV3307), each message naming its armed posture, with exactly one composition paid and no repair round dispatched, never a silent settle over findings nobody ruled on",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.240.0",
+  "version": "1.241.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,11 +22,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/anthropic": "1.240.0",
-    "@rulvar/openai": "1.240.0",
-    "@rulvar/core": "1.240.0",
-    "@rulvar/plan": "1.240.0",
-    "@rulvar/testing": "1.240.0"
+    "@rulvar/anthropic": "1.241.0",
+    "@rulvar/plan": "1.241.0",
+    "@rulvar/core": "1.241.0",
+    "@rulvar/testing": "1.241.0",
+    "@rulvar/openai": "1.241.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",
@@ -46,6 +46,7 @@
     "build": "tsdown",
     "typecheck": "tsc --noEmit",
     "lint": "eslint .",
+    "test": "pnpm -w exec vitest run --project @rulvar/evals",
     "pack-check": "publint --pack pnpm && attw --pack . --profile esm-only"
   }
 }