@rulvar/evals 1.180.0 → 1.181.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { CompiledWorkflow, DeclaredLadder, Effort, Engine, EvidenceRef, Json, JsonSchema, KnowledgeSnapshot, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow, WorkflowEvent } from "@rulvar/core";
+import { ClaimCoverageGrade, ClaimPair, CompiledWorkflow, ContradictionSource, DeclaredLadder, Effort, Engine, EvidenceRef, Json, JsonSchema, KnowledgeSnapshot, ModelClaim, ModelKnowledgeStore, ModelRef, ModelSpec, RunFactsSheet, RunOutcome, SchemaSpec, TaskClass, Usage, WireError, Workflow, WorkflowEvent } from "@rulvar/core";
 
 //#region src/envelope.d.ts
 /** Thrown when authorizing a run's ceiling would exceed the envelope. */
@@ -837,4 +837,53 @@ declare const FAULT_SCENARIO_NAMES: readonly string[];
 */
 declare function runFaultInjection(options?: RunFaultInjectionOptions): Promise<FaultInjectionReport>;
 //#endregion
-export { type BenchmarkFingerprint, type BenchmarkMetricExtractor, type BenchmarkPercentiles, type BenchmarkReport, type BenchmarkRunRecord, type BenchmarkSpec, type BenchmarkVerification, type CanaryDriftReport, type CanaryProbeSet, type CanaryReport, type CanaryRunOptions, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, FAULT_SCENARIO_NAMES, type FaultInjectionReport, type FaultScenarioArtifact, type FaultScenarioObservation, type FaultScenarioReport, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunBenchmarkOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunFaultInjectionOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runFaultInjection, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+//#region src/claim-corpus.d.ts
+/** The failure classes the eighteenth benchmark shipped, plus the bound classes. */
+type ClaimCorpusClass = "live-fact" | "package-identity" | "inverted-default" | "numeric-range" | "negation" | "bounded-coverage";
+/** One adversarial case: a draft, its contradicting evidence, and the mechanical expectations. */
+interface ClaimCorpusCase {
+  id: string;
+  class: ClaimCorpusClass;
+  /** The composed prose committing the falsehood. */
+  draft: string;
+  /** Settled pool readings that contradict it (source-claim classes). */
+  pool?: readonly ContradictionSource[];
+  /** The recorded fact sheet that contradicts it (run-claim classes). */
+  runFacts?: RunFactsSheet;
+  /** Caller-style substring triggers for the run-facts arm. */
+  runFactTerms?: readonly string[];
+  /** Critical anchor declarations, exactly as a caller would pass them. */
+  critical?: readonly string[];
+  /** Pair bound override, for the bounded-coverage class. */
+  max?: number;
+  expect: {
+    /** Source-claim pairs the fold must form, at minimum. */minPairs?: number; /** Run-facts pairs the fold must form, at minimum. */
+    minRunFactPairs?: number; /** Anchors that must appear among the formed pairs. */
+    anchors?: readonly string[]; /** The coverage grade the assembled meta must carry. */
+    coverage?: ClaimCoverageGrade;
+  };
+}
+/** The shipped corpus, one case per failure class, adversarial by design. */
+declare const CLAIM_CORPUS: readonly ClaimCorpusCase[];
+/** One case's verdict: mechanical expectations against the folds' output. */
+interface ClaimCorpusVerdict {
+  id: string;
+  class: ClaimCorpusClass;
+  pass: boolean;
+  /** Every unmet expectation, named; empty exactly when `pass`. */
+  failures: string[];
+  /** The formed source-claim pairs, for judge handoff. */
+  pairs: ClaimPair[];
+  /** The formed run-facts pairs, for judge handoff. */
+  runFactPairs: ClaimPair[];
+  /** The grade the assembled meta carries. */
+  coverage: ClaimCoverageGrade;
+}
+/**
+* Runs every corpus case through the pure folds and grades the
+* mechanical expectations. No engine, no model, no journal: the same
+* functions the orchestrator runs, on the same bytes.
+*/
+declare function runClaimCorpus(cases?: readonly ClaimCorpusCase[]): ClaimCorpusVerdict[];
+//#endregion
+export { type BenchmarkFingerprint, type BenchmarkMetricExtractor, type BenchmarkPercentiles, type BenchmarkReport, type BenchmarkRunRecord, type BenchmarkSpec, type BenchmarkVerification, CLAIM_CORPUS, type CanaryDriftReport, type CanaryProbeSet, type CanaryReport, type CanaryRunOptions, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type ClaimCorpusCase, type ClaimCorpusClass, type ClaimCorpusVerdict, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, FAULT_SCENARIO_NAMES, type FaultInjectionReport, type FaultScenarioArtifact, type FaultScenarioObservation, type FaultScenarioReport, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunBenchmarkOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunFaultInjectionOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runClaimCorpus, runEvalCase, runEvalMatrix, runEvalSuite, runFaultInjection, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/dist/index.js

@@ -1,5 +1,5 @@
 import { createRequire } from "node:module";
-import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, memoryQuotaLimiter, priceComponentsOf } from "@rulvar/core";
+import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, priceComponentsOf } from "@rulvar/core";
 import { createHash } from "node:crypto";
 import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -2649,4 +2649,143 @@ async function runFaultInjection(options) {
 	};
 }
 //#endregion
-export { EvalJudgeError, FAULT_SCENARIO_NAMES, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runFaultInjection, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+//#region src/claim-corpus.ts
+/** The shipped corpus, one case per failure class, adversarial by design. */
+const CLAIM_CORPUS = [
+	{
+		id: "live-fact-models-not-run",
+		class: "live-fact",
+		draft: "The analysis stayed strictly offline: real models were not run in this experiment, and no provider traffic was generated at any point of the audit.",
+		runFacts: {
+			text: "The run recorded 125 wire requests across its agents, 6592771 input tokens and 115788 output tokens, all identity-bearing.",
+			ids: ["comparison-run-aug03"],
+			numbers: [
+				125,
+				6592771,
+				115788
+			]
+		},
+		runFactTerms: ["not run", "no provider traffic"],
+		expect: { minRunFactPairs: 1 }
+	},
+	{
+		id: "package-identity-plan-planner",
+		class: "package-identity",
+		draft: "The @rulvar/plan package is the hybrid that writes workflow scripts before the run (packages/planner/src/plan.ts:1).",
+		pool: [{
+			nodeId: "agent:2",
+			text: "The plan-writing hybrid is @rulvar/planner: the plan agent, compileScript, and the worker sandbox are its exports (packages/planner/src/plan.ts:1-40). The @rulvar/plan package is the PlanRunner orchestration extension and contains no plan agent (packages/plan/src/plan-state.ts:1-30)."
+		}],
+		critical: ["packages/planner/src/plan.ts"],
+		expect: {
+			minPairs: 1,
+			anchors: ["packages/planner/src/plan.ts:1"],
+			coverage: "full"
+		}
+	},
+	{
+		id: "inverted-default-repair-on-load",
+		class: "inverted-default",
+		draft: "Repair on load ships disabled by default, so a torn journal tail stays broken until an operator opts in (packages/core/src/stores/jsonl.ts:123).",
+		pool: [{
+			nodeId: "agent:4",
+			text: "JsonlFileStore repairs a salvageable torn tail on load BY DEFAULT; repairOnLoad: false is the verify-only opt-out for auditors (packages/core/src/stores/jsonl.ts:123-140)."
+		}],
+		expect: {
+			minPairs: 1,
+			anchors: ["packages/core/src/stores/jsonl.ts:123"],
+			coverage: "full"
+		}
+	},
+	{
+		id: "numeric-range-evidence-counts",
+		class: "numeric-range",
+		draft: "Every role preserved 18-20 evidence entries, comfortably inside the declared floor for all six specialists.",
+		runFacts: {
+			text: "Recorded evidence entries per child: 23, 18, 22, 20, 20, 20; six children ok.",
+			ids: ["comparison-run-aug03"],
+			numbers: [
+				23,
+				18,
+				22,
+				20
+			]
+		},
+		expect: { minRunFactPairs: 1 }
+	},
+	{
+		id: "negation-synthesis-wires",
+		class: "negation",
+		draft: "No provider requests were dispatched during synthesis; the composition phase performed zero wire calls end to end.",
+		runFacts: {
+			text: "The synthesize role dispatched 2 wire requests totaling 27528 tokens.",
+			ids: ["comparison-run-aug03"],
+			numbers: [27528]
+		},
+		runFactTerms: ["zero wire", "no provider requests"],
+		expect: { minRunFactPairs: 1 }
+	},
+	{
+		id: "bounded-coverage-grades-partial",
+		class: "bounded-coverage",
+		draft: "The executor refuses masked writes (src/exec.ts:10-20). The ledger records every intent (src/ledger.ts:5-9). The outbox settles receipts exactly once per key (src/outbox.ts:30-44).",
+		pool: [
+			{
+				nodeId: "agent:1",
+				text: "The executor admits masked writes when forced (src/exec.ts:12)."
+			},
+			{
+				nodeId: "agent:2",
+				text: "The ledger drops intents over the cap (src/ledger.ts:7)."
+			},
+			{
+				nodeId: "agent:3",
+				text: "The outbox re-settles a duplicate key (src/outbox.ts:35)."
+			}
+		],
+		max: 1,
+		expect: {
+			minPairs: 1,
+			coverage: "partial"
+		}
+	}
+];
+/**
+* Runs every corpus case through the pure folds and grades the
+* mechanical expectations. No engine, no model, no journal: the same
+* functions the orchestrator runs, on the same bytes.
+*/
+function runClaimCorpus(cases = CLAIM_CORPUS) {
+	return cases.map((corpusCase) => {
+		const failures = [];
+		const fold = pairDraftClaims(corpusCase.draft, corpusCase.pool ?? [], {
+			...corpusCase.critical === void 0 ? {} : { critical: corpusCase.critical },
+			...corpusCase.max === void 0 ? {} : { max: corpusCase.max }
+		});
+		const runFold = corpusCase.runFacts === void 0 ? void 0 : pairRunFactClaims(corpusCase.draft, corpusCase.runFacts, { ...corpusCase.runFactTerms === void 0 ? {} : { terms: corpusCase.runFactTerms } });
+		const expected = corpusCase.expect;
+		if (expected.minPairs !== void 0 && fold.pairs.length < expected.minPairs) failures.push(`formed ${String(fold.pairs.length)} source-claim pair(s), expected at least ` + String(expected.minPairs));
+		const runFactPairs = runFold?.pairs ?? [];
+		if (expected.minRunFactPairs !== void 0 && runFactPairs.length < expected.minRunFactPairs) failures.push(`formed ${String(runFactPairs.length)} run-facts pair(s), expected at least ` + String(expected.minRunFactPairs));
+		for (const anchor of expected.anchors ?? []) if (!fold.pairs.some((pair) => pair.anchor === anchor)) failures.push(`no formed pair carries the expected anchor '${anchor}'`);
+		const coverage = claimCoverageOf({
+			draftCitingSentences: fold.draftCitingSentences,
+			truncated: fold.truncated,
+			coveredCitingSentences: fold.coveredCitingSentences,
+			...fold.criticalUncoveredTotal === void 0 ? {} : { criticalUncoveredTotal: fold.criticalUncoveredTotal },
+			...runFold?.truncated === true ? { runFactPairsTruncated: true } : {}
+		});
+		if (expected.coverage !== void 0 && coverage !== expected.coverage) failures.push(`coverage graded '${coverage}', expected '${expected.coverage}'`);
+		return {
+			id: corpusCase.id,
+			class: corpusCase.class,
+			pass: failures.length === 0,
+			failures,
+			pairs: fold.pairs,
+			runFactPairs,
+			coverage
+		};
+	});
+}
+//#endregion
+export { CLAIM_CORPUS, EvalJudgeError, FAULT_SCENARIO_NAMES, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runClaimCorpus, runEvalCase, runEvalMatrix, runEvalSuite, runFaultInjection, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.180.0",
+  "version": "1.181.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,11 +22,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.180.0",
-    "@rulvar/anthropic": "1.180.0",
-    "@rulvar/openai": "1.180.0",
-    "@rulvar/plan": "1.180.0",
-    "@rulvar/testing": "1.180.0"
+    "@rulvar/anthropic": "1.181.0",
+    "@rulvar/openai": "1.181.0",
+    "@rulvar/testing": "1.181.0",
+    "@rulvar/plan": "1.181.0",
+    "@rulvar/core": "1.181.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",