@rulvar/evals 1.123.0 → 1.124.0

package/dist/index.d.ts

@@ -785,4 +785,47 @@ declare function runValueCheckpoint(checkpointPool: CheckpointPool, options: Run
 /** The deterministic render for the M12 gate docs amendment. */
 declare function renderCheckpointReport(report: CheckpointReport): string;
 //#endregion
-export { type BenchmarkFingerprint, type BenchmarkMetricExtractor, type BenchmarkPercentiles, type BenchmarkReport, type BenchmarkRunRecord, type BenchmarkSpec, type BenchmarkVerification, type CanaryDriftReport, type CanaryProbeSet, type CanaryReport, type CanaryRunOptions, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunBenchmarkOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+//#region src/fault-injection.d.ts
+/** One machine-checkable observation of a driven branch. */
+interface FaultScenarioObservation {
+  /** The documented typed observable was produced exactly. */
+  matched: boolean;
+  /** What was actually observed, quoting the typed surfaces. */
+  detail: string;
+}
+/** One artifact a scenario leaves, JSON or raw text. */
+interface FaultScenarioArtifact {
+  name: string;
+  content: string;
+}
+interface FaultScenarioReport {
+  scenario: string;
+  /** The never-observed-live branch this scenario exists to drive. */
+  doctrine: string;
+  observation: FaultScenarioObservation;
+  artifacts: FaultScenarioArtifact[];
+}
+interface FaultInjectionReport {
+  scenarios: FaultScenarioReport[];
+  /** Every scenario matched its documented observable. */
+  allMatched: boolean;
+  /** The artifact files written, when `artifactsDir` was given. */
+  artifactFiles?: string[];
+}
+interface RunFaultInjectionOptions {
+  /** Write one `<scenario>.json` artifact bundle per scenario here. */
+  artifactsDir?: string;
+  /** Run only these scenarios; an unknown name is a typed ConfigError. */
+  only?: readonly string[];
+}
+/** The scenario names in run order. */
+declare const FAULT_SCENARIO_NAMES: readonly string[];
+/**
+* Runs the fault-injection scenarios sequentially and reports each
+* driven branch's observation; with `artifactsDir`, writes one
+* `<scenario>.json` bundle per scenario (the observation plus every
+* artifact), the experiment-grade trace a review can cite.
+*/
+declare function runFaultInjection(options?: RunFaultInjectionOptions): Promise<FaultInjectionReport>;
+//#endregion
+export { type BenchmarkFingerprint, type BenchmarkMetricExtractor, type BenchmarkPercentiles, type BenchmarkReport, type BenchmarkRunRecord, type BenchmarkSpec, type BenchmarkVerification, type CanaryDriftReport, type CanaryProbeSet, type CanaryReport, type CanaryRunOptions, type CheckpointArm, type CheckpointCell, type CheckpointLadder, type CheckpointPool, type CheckpointReport, type CriterionOneReport, type CriterionTwoReport, type EvalCase, type EvalCaseResult, type EvalCommitterOptions, EvalJudgeError, type EvalMatrixReport, type EvalSuiteResult, FAULT_SCENARIO_NAMES, type FaultInjectionReport, type FaultScenarioArtifact, type FaultScenarioObservation, type FaultScenarioReport, type GoldenGraderOptions, type Grader, type GraderContext, type GraderVerdict, JUDGE_VERDICT_SCHEMA, type JudgeGraderOptions, type JudgeSpec, type MatrixCell, type MatrixCellReport, type MeasuredClaimInput, type OrchestratedCase, type RubricCriterion, type RubricGraderOptions, type RunBenchmarkOptions, type RunCheckpointOptions, type RunEvalCaseOptions, type RunEvalSuiteOptions, type RunFaultInjectionOptions, type RunSweepOptions, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, type SweepCase, type SweepCellReport, type SweepModel, type SweepPool, type SweepReport, type SweepThresholds, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runFaultInjection, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/dist/index.js

@@ -1,6 +1,10 @@
 import { createRequire } from "node:module";
-import { ConfigError, KnowledgeCasError, claimExpiry, compileVerifiedLayer, defineWorkflow, hashRunOutput, lastRunSettle } from "@rulvar/core";
+import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, claimExpiry, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, journalPricingSnapshot, lastRunSettle, memoryQuotaLimiter } from "@rulvar/core";
 import { createHash } from "node:crypto";
+import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { FAKE_MODEL_REF, FakeAdapter } from "@rulvar/testing";
 //#region src/envelope.ts
 /**
 * The debit-only aggregate spend envelope (v1.16.2 review P1-2). A
@@ -1208,4 +1212,346 @@ async function runSweepMatrix(pool, options) {
 	return report;
 }
 //#endregion
-export { EvalJudgeError, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runSweepMatrix, runValueCheckpoint, rungRuleHolds };
+//#region src/fault-injection.ts
+/**
+* The fault-injection kit (RV811): the comparison experiments left a
+* standing list of fail-closed branches never observed live, and a
+* branch nobody has ever driven is a claim, not a guarantee. Each
+* scenario here DELIBERATELY drives one such branch on the real engine
+* with scripted adapters (zero provider calls, zero keys), verifies the
+* documented typed observable, and leaves experiment-grade artifacts
+* (the outcome, the journal, the raw bytes where the fault is a byte
+* fault). Fail closed like everything else in this package: a scenario
+* whose branch stops producing its documented observable reports
+* `matched: false` and the whole report says so, instead of the list
+* quietly becoming untested again.
+*/
+const ROUTING = { routing: { loop: FAKE_MODEL_REF } };
+const echoWorkflow = defineWorkflow({ name: "fault-kit-echo" }, async (ctx) => {
+	return await ctx.agent("one small step");
+});
+const twoStepWorkflow = defineWorkflow({ name: "fault-kit-two-step" }, async (ctx) => {
+	return {
+		first: await ctx.agent("first step"),
+		second: await ctx.agent("second step")
+	};
+});
+function jsonArtifact(name, value) {
+	return {
+		name,
+		content: JSON.stringify(value, null, 2)
+	};
+}
+function errorText(thrown) {
+	return thrown instanceof Error ? `${thrown.name}: ${thrown.message}` : String(thrown);
+}
+/**
+* RV711: the opt-in in-flight exposure cap refuses a dispatch whose
+* worst-case estimate does not fit, BEFORE any provider call, with the
+* typed transient refusal (never a claimed ceiling crossing).
+*/
+const inFlightExposure = {
+	name: "in-flight-exposure-refusal",
+	doctrine: "maxInFlightExposureUsd refuses the dispatch typed before any provider call (BudgetExhaustedError, reason 'in-flight-exposure'), never a silent wait and never a claimed budget-ceiling crossing",
+	async run() {
+		const adapter = new FakeAdapter({
+			agents: { "*": "never dispatched" },
+			capsOverrides: { pricing: {
+				inputUsdPerMTok: 3,
+				outputUsdPerMTok: 15
+			} }
+		});
+		const store = new InMemoryStore();
+		const outcome = await createEngine({
+			adapters: [adapter],
+			stores: { journal: store },
+			defaults: ROUTING
+		}).run(echoWorkflow, void 0, {
+			runId: "fault-exposure",
+			maxInFlightExposureUsd: 1e-4
+		}).result;
+		const message = outcome.error?.message ?? "";
+		return {
+			observation: {
+				matched: (outcome.status === "exhausted" || outcome.status === "error") && message.includes("in flight exposure cap reached") && message.includes("refused before any provider call"),
+				detail: `run status '${outcome.status}'; ${message}`
+			},
+			artifacts: [jsonArtifact("outcome.json", {
+				status: outcome.status,
+				error: outcome.error
+			}), jsonArtifact("journal.json", await store.load("fault-exposure"))]
+		};
+	}
+};
+/**
+* RV704: two rules with one canonical content key are refused at the
+* shared construction chokepoint, because memory buckets by index while
+* keyed stores bucket by key, and one configuration must never admit
+* differently per storage.
+*/
+const duplicateQuotaRule = {
+	name: "duplicate-quota-rule",
+	doctrine: "a duplicated quota rule is refused typed at construction (ConfigError naming both indexes and the shared rule key), never admitted differently per storage backend",
+	run() {
+		let detail = "memoryQuotaLimiter accepted the duplicated rule set";
+		let matched = false;
+		try {
+			memoryQuotaLimiter([{
+				provider: "fake",
+				requestsPerMinute: 10
+			}, {
+				provider: "fake",
+				requestsPerMinute: 10
+			}]);
+		} catch (thrown) {
+			detail = errorText(thrown);
+			matched = thrown instanceof ConfigError && detail.includes("duplicates") && detail.includes("delete the duplicate");
+		}
+		return Promise.resolve({
+			observation: {
+				matched,
+				detail
+			},
+			artifacts: [jsonArtifact("refusal.json", { detail })]
+		});
+	}
+};
+/** Runs the echo workflow into a fresh JsonlFileStore dir and returns the pieces. */
+async function seededJsonlRun(runId) {
+	const dir = mkdtempSync(join(tmpdir(), "rulvar-fault-jsonl-"));
+	const store = new JsonlFileStore({ dir });
+	const outcome = await createEngine({
+		adapters: [new FakeAdapter({ agents: { "*": "stored answer" } })],
+		stores: { journal: store },
+		defaults: ROUTING
+	}).run(echoWorkflow, void 0, { runId }).result;
+	if (outcome.status !== "ok") throw new Error(`fault kit: the seeding run settled '${outcome.status}' instead of ok`);
+	const entries = await store.load(runId);
+	const file = readdirSync(dir).filter((name) => name.includes(runId) && name.endsWith(".jsonl")).map((name) => join(dir, name))[0];
+	if (file === void 0) throw new Error(`fault kit: no journal file found for '${runId}' in ${dir}`);
+	return {
+		dir,
+		file,
+		entries
+	};
+}
+const SCENARIOS = [
+	inFlightExposure,
+	duplicateQuotaRule,
+	{
+		name: "torn-jsonl-tail",
+		doctrine: "a torn trailing JSONL line (crash mid-append) is discarded on load, every whole record is salvaged, and the file is repaired so the tear never accumulates",
+		async run() {
+			const { dir, file, entries } = await seededJsonlRun("fault-torn");
+			appendFileSync(file, "{\"seq\": 9999, \"kind\": \"agent\", \"torn\": tr", "utf8");
+			const bytesBefore = readFileSync(file, "utf8");
+			const loaded = await new JsonlFileStore({ dir }).load("fault-torn");
+			const bytesAfter = readFileSync(file, "utf8");
+			const everyLineParses = bytesAfter.split("\n").filter((line) => line !== "").every((line) => {
+				try {
+					JSON.parse(line);
+					return true;
+				} catch {
+					return false;
+				}
+			});
+			return {
+				observation: {
+					matched: loaded.length === entries.length && everyLineParses && !bytesAfter.includes("\"torn\": tr"),
+					detail: `salvaged ${String(loaded.length)} of ${String(entries.length)} whole entries; the torn fragment was discarded and the file repaired in place (every remaining line parses: ${String(everyLineParses)})`
+				},
+				artifacts: [
+					{
+						name: "journal-torn.jsonl",
+						content: bytesBefore
+					},
+					{
+						name: "journal-repaired.jsonl",
+						content: bytesAfter
+					},
+					jsonArtifact("counts.json", {
+						before: entries.length,
+						salvaged: loaded.length
+					})
+				]
+			};
+		}
+	},
+	{
+		name: "glued-jsonl-tail",
+		doctrine: "whole JSONL records glued onto one trailing line are accepted data: the load salvages every glued record instead of discarding them as a torn fragment",
+		async run() {
+			const { dir, file, entries } = await seededJsonlRun("fault-glued");
+			const lines = readFileSync(file, "utf8").split("\n").filter((line) => line !== "");
+			writeFileSync(file, [...lines.slice(0, -2), `${lines[lines.length - 2] ?? ""}${lines[lines.length - 1] ?? ""}`].join("\n"), "utf8");
+			const bytesBefore = readFileSync(file, "utf8");
+			const loaded = await new JsonlFileStore({ dir }).load("fault-glued");
+			return {
+				observation: {
+					matched: loaded.length === entries.length && lines.length === entries.length,
+					detail: `glued tail: ${String(loaded.length)} of ${String(entries.length)} records salvaged, the two glued records included, none discarded`
+				},
+				artifacts: [{
+					name: "journal-glued.jsonl",
+					content: bytesBefore
+				}, jsonArtifact("counts.json", {
+					before: entries.length,
+					salvaged: loaded.length
+				})]
+			};
+		}
+	},
+	{
+		name: "crash-resume-settle-boundary",
+		doctrine: "a journal cut immediately after an agent terminal entry (the settle-boundary crash window) resumes to ok: the settled agent replays with zero live calls and only the unsettled remainder re-runs",
+		async run() {
+			const liveAdapter = new FakeAdapter({ agents: { "*": (call) => `answer: ${call.prompt}` } });
+			const storeA = new InMemoryStore();
+			const first = await createEngine({
+				adapters: [liveAdapter],
+				stores: { journal: storeA },
+				defaults: ROUTING
+			}).run(twoStepWorkflow, void 0, { runId: "fault-boundary" }).result;
+			if (first.status !== "ok") throw new Error(`fault kit: the seeding run settled '${first.status}' instead of ok`);
+			const entries = await storeA.load("fault-boundary");
+			const settleIndex = entries.findIndex((entry) => entry.kind === "agent" && entry.status === "ok");
+			const cut = entries.slice(0, settleIndex + 1);
+			const storeB = new InMemoryStore();
+			for (const entry of cut) await storeB.append("fault-boundary", entry);
+			let liveCalls = 0;
+			const resumed = await createEngine({
+				adapters: [new FakeAdapter({ agents: { "*": (call) => {
+					liveCalls += 1;
+					return `answer: ${call.prompt}`;
+				} } })],
+				stores: { journal: storeB },
+				defaults: ROUTING
+			}).resume("fault-boundary", twoStepWorkflow).result;
+			return {
+				observation: {
+					matched: resumed.status === "ok" && liveCalls === 1,
+					detail: `resumed '${resumed.status}' from the post-settle crash window (journal cut at seq ${String(cut[cut.length - 1]?.seq ?? -1)} of ${String(entries.length)} entries): liveCalls=${String(liveCalls)}, the settled step replayed free`
+				},
+				artifacts: [jsonArtifact("journal-cut.json", cut), jsonArtifact("resume-outcome.json", {
+					status: resumed.status,
+					liveCalls,
+					value: resumed.value ?? null
+				})]
+			};
+		}
+	},
+	{
+		name: "pricing-rotation-uncovered-tail",
+		doctrine: "after a price-table rotation that drops the model, the pinned segment still prices under its own pin while the uncovered tail folds unpriced (undefined), never silently at the stale pinned rates",
+		async run() {
+			const adapter = new FakeAdapter({ agents: { "*": "priced answer" } });
+			const store = new InMemoryStore();
+			const outcome = await createEngine({
+				adapters: [adapter],
+				stores: { journal: store },
+				defaults: ROUTING,
+				pricing: {
+					pricingVersion: "fault-v1",
+					models: { [FAKE_MODEL_REF]: {
+						inputUsdPerMTok: 3,
+						outputUsdPerMTok: 15
+					} }
+				}
+			}).run(echoWorkflow, void 0, { runId: "fault-rotation" }).result;
+			if (outcome.status !== "ok") throw new Error(`fault kit: the seeding run settled '${outcome.status}' instead of ok`);
+			const snapshot = journalPricingSnapshot(await store.load("fault-rotation"));
+			const composed = snapshot?.composedPriceUsd(() => void 0);
+			const usage = {
+				inputTokens: 1e3,
+				outputTokens: 100,
+				cacheReadTokens: 0,
+				cacheWriteTokens: 0
+			};
+			const pinnedSegmentUsd = composed?.(FAKE_MODEL_REF, usage, 1);
+			const uncoveredTailUsd = composed?.(FAKE_MODEL_REF, usage, (snapshot?.pinnedThroughSeq ?? 0) + 1);
+			const ghostUsd = composed?.("ghost:model", usage, 1);
+			return {
+				observation: {
+					matched: typeof pinnedSegmentUsd === "number" && pinnedSegmentUsd > 0 && uncoveredTailUsd === void 0 && ghostUsd === void 0,
+					detail: `pinned segment priced under '${snapshot?.pricingVersion ?? "none"}' (${String(pinnedSegmentUsd)} USD); the rotated tail and the ghost model fold unpriced (undefined), surfaced, never a silent zero`
+				},
+				artifacts: [jsonArtifact("pricing-snapshot.json", {
+					pricingVersion: snapshot?.pricingVersion,
+					pinnedThroughSeq: snapshot?.pinnedThroughSeq,
+					segments: snapshot?.segments
+				}), jsonArtifact("fold.json", {
+					pinnedSegmentUsd: pinnedSegmentUsd ?? null,
+					uncoveredTailUsd: uncoveredTailUsd ?? null,
+					ghostUsd: ghostUsd ?? null
+				})]
+			};
+		}
+	},
+	{
+		name: "unknown-provider-id",
+		doctrine: "routing to an unregistered provider id fails typed naming the id; nothing dispatches and nothing settles ok",
+		async run() {
+			const adapter = new FakeAdapter({ agents: { "*": "never used" } });
+			const store = new InMemoryStore();
+			let detail = "";
+			let matched = false;
+			try {
+				const outcome = await createEngine({
+					adapters: [adapter],
+					stores: { journal: store },
+					defaults: { routing: { loop: "ghost:model" } }
+				}).run(echoWorkflow, void 0, { runId: "fault-ghost" }).result;
+				detail = `run status '${outcome.status}': ${outcome.error?.message ?? ""}`;
+				matched = outcome.status === "error" && (outcome.error?.message ?? "").includes("ghost");
+			} catch (thrown) {
+				detail = errorText(thrown);
+				matched = detail.includes("ghost");
+			}
+			return {
+				observation: {
+					matched,
+					detail
+				},
+				artifacts: [jsonArtifact("refusal.json", { detail })]
+			};
+		}
+	}
+];
+/** The scenario names in run order. */
+const FAULT_SCENARIO_NAMES = SCENARIOS.map((scenario) => scenario.name);
+/**
+* Runs the fault-injection scenarios sequentially and reports each
+* driven branch's observation; with `artifactsDir`, writes one
+* `<scenario>.json` bundle per scenario (the observation plus every
+* artifact), the experiment-grade trace a review can cite.
+*/
+async function runFaultInjection(options) {
+	const known = new Map(SCENARIOS.map((scenario) => [scenario.name, scenario]));
+	for (const name of options?.only ?? []) if (!known.has(name)) throw new ConfigError(`unknown fault scenario '${name}'; known: ${FAULT_SCENARIO_NAMES.join(", ")}`);
+	const selected = options?.only === void 0 ? SCENARIOS : SCENARIOS.filter((scenario) => options.only?.includes(scenario.name));
+	const scenarios = [];
+	const artifactFiles = [];
+	for (const scenario of selected) {
+		const { observation, artifacts } = await scenario.run();
+		const report = {
+			scenario: scenario.name,
+			doctrine: scenario.doctrine,
+			observation,
+			artifacts
+		};
+		scenarios.push(report);
+		if (options?.artifactsDir !== void 0) {
+			mkdirSync(options.artifactsDir, { recursive: true });
+			const file = join(options.artifactsDir, `${scenario.name}.json`);
+			writeFileSync(file, JSON.stringify(report, null, 2), "utf8");
+			artifactFiles.push(file);
+		}
+	}
+	return {
+		scenarios,
+		allMatched: scenarios.every((report) => report.observation.matched),
+		...options?.artifactsDir === void 0 ? {} : { artifactFiles }
+	};
+}
+//#endregion
+export { EvalJudgeError, FAULT_SCENARIO_NAMES, JUDGE_VERDICT_SCHEMA, SWEEP_THRESHOLD_DEFAULTS, SpendEnvelope, SweepBudgetError, agentTypeRuleHolds, canaryFingerprint, commitEvalMeasured, evalMeasuredClaim, flipStaleOnCanaryDrift, goldenGrader, judgeGrader, normalizeCanaryOutput, renderCheckpointReport, rubricGrader, runBenchmark, runCanary, runEvalCase, runEvalMatrix, runEvalSuite, runFaultInjection, runSweepMatrix, runValueCheckpoint, rungRuleHolds };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.123.0",
+  "version": "1.124.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,8 +22,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.123.0",
-    "@rulvar/testing": "1.123.0"
+    "@rulvar/core": "1.124.0",
+    "@rulvar/testing": "1.124.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",