@rulvar/evals 1.195.0 → 1.197.0

package/dist/index.js

@@ -1,5 +1,5 @@
 import { createRequire } from "node:module";
-import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, priceComponentsOf } from "@rulvar/core";
+import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
 import { createHash } from "node:crypto";
 import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -2607,6 +2607,203 @@ const SCENARIOS = [
 				})]
 			};
 		}
+	},
+	{
+		name: "benchmark-primary-preflight-parity",
+		doctrine: "the admission projection holds the synthesis reserve exactly like the live gates (RV1901): the benchmark primary configuration projects 2 of 4 seats with the equation terms exposed and the roster shortfall named admission-below-roster-floor, never the 5/5 green wave the live gate is bound to refuse",
+		run() {
+			const report = preflightEstimate({
+				engine: {
+					adapters: [new FakeAdapter({ agents: { "*": "unused" } })],
+					defaults: { routing: {
+						loop: FAKE_MODEL_REF,
+						orchestrate: FAKE_MODEL_REF,
+						synthesize: FAKE_MODEL_REF
+					} }
+				},
+				run: { budgetUsd: 6 },
+				orchestrator: {
+					budget: {
+						capUsd: 4.5,
+						capFraction: 1,
+						synthesisReserveUsd: 1
+					},
+					synthesis: { limits: { maxTurns: 2 } },
+					acceptance: { minSpawnedChildren: 4 }
+				},
+				spawns: [
+					"product",
+					"finops",
+					"durability",
+					"adversarial"
+				].map((label) => ({
+					label,
+					estCost: .62
+				}))
+			});
+			const denied = report.admission.wave.filter((row) => !row.admitted);
+			const floorFinding = report.findings.find((finding) => finding.code === "admission-below-roster-floor");
+			const matched = report.admission.admitted === 3 && report.admission.denied === 2 && denied.every((row) => row.deniedBy === "budget") && report.admission.synthesisReserveUsd === 1 && Math.abs((report.admission.wave[0]?.reserveUsd ?? 0) - 3.5) < 1e-9 && Math.abs((report.admission.wave[0]?.heldAtEvaluationUsd ?? 0) - 1) < 1e-9 && floorFinding?.severity === "error";
+			return Promise.resolve({
+				observation: {
+					matched,
+					detail: `the wave seats ${String(report.admission.admitted)} of 5 rows (${String(report.admission.denied)} denied by budget), synthesis hold ${String(report.admission.synthesisReserveUsd)} USD, roster finding '${String(floorFinding?.code)}' at severity '${String(floorFinding?.severity)}'`
+				},
+				artifacts: [jsonArtifact("preflight.json", report)]
+			});
+		}
+	},
+	{
+		name: "benchmark-recovery-root-exposure",
+		doctrine: "a root turn refused by the in-flight exposure cap beside live children parks and completes after a hold releases (RV1902), every child terminal precedes run_settle (RV1903), and the terminal envelope and the invoice cardinality agree on the wire count (RV1904): the recovery arm terminal-failed on every one of these",
+		async run() {
+			let releaseChildren = () => {};
+			const childrenGate = new Promise((resolve) => {
+				releaseChildren = resolve;
+			});
+			const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
+			let orchTurn = 0;
+			const calls = [];
+			const toolEvents = (name, args, call) => [
+				{
+					type: "tool-call-start",
+					id: `id-${String(call)}-0`,
+					name
+				},
+				{
+					type: "tool-call-delta",
+					id: `id-${String(call)}-0`,
+					argsTextDelta: JSON.stringify(args)
+				},
+				{
+					type: "tool-call-end",
+					id: `id-${String(call)}-0`,
+					args
+				}
+			];
+			const handlesIn = (req) => {
+				const found = [];
+				for (const msg of req.messages) for (const part of msg.parts) if (part.type === "tool-result") {
+					const result = part.result;
+					if (typeof result.handle === "number") found.push(result.handle);
+				}
+				return found;
+			};
+			const adapter = {
+				id: "fake",
+				calls,
+				caps: () => ({
+					contextWindow: 2e5,
+					maxOutputTokens: 4096,
+					structuredOutput: "native",
+					supportsTemperature: true,
+					supportsParallelTools: true,
+					reasoningEfforts: [],
+					pricing: {
+						inputUsdPerMTok: 1,
+						outputUsdPerMTok: 10
+					}
+				}),
+				async *stream(req, signal) {
+					const call = calls.length;
+					calls.push(req);
+					if (agentTypeOfReq(req) !== "") {
+						await Promise.race([childrenGate, new Promise((resolve) => {
+							signal?.addEventListener("abort", () => resolve(), { once: true });
+						})]);
+						if (signal?.aborted === true) return;
+						yield {
+							type: "text-delta",
+							text: "worked"
+						};
+						yield {
+							type: "finish",
+							finish: { reason: "stop" },
+							usage: {
+								inputTokens: 10,
+								outputTokens: 5,
+								cacheReadTokens: 0,
+								cacheWriteTokens: 0
+							}
+						};
+						return;
+					}
+					orchTurn += 1;
+					const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
+						agentType: "worker",
+						prompt: "task A"
+					}, call).concat(toolEvents("spawn_agent", {
+						agentType: "worker",
+						prompt: "task B"
+					}, call + 1e3)) : orchTurn === 2 ? toolEvents("await_all", { handles: handlesIn(req) }, call) : toolEvents("finish", { result: "joined after the wait" }, call);
+					for (const event of turn) yield event;
+					yield {
+						type: "finish",
+						finish: { reason: "tool-calls" },
+						usage: {
+							inputTokens: 10,
+							outputTokens: 5,
+							cacheReadTokens: 0,
+							cacheWriteTokens: 0
+						}
+					};
+				}
+			};
+			const store = new InMemoryStore();
+			const engine = createEngine({
+				adapters: [adapter],
+				stores: { journal: store },
+				defaults: {
+					routing: {
+						loop: "fake:model",
+						orchestrate: "fake:model"
+					},
+					profiles: { worker: {
+						description: "the gated worker",
+						limits: { maxOutputTokensPerTurn: 2500 }
+					} }
+				}
+			});
+			const wf = makeOrchestratorWorkflow("join the gated wave", { limits: { maxOutputTokensPerTurn: 4e3 } });
+			const handle = engine.run(wf, void 0, {
+				runId: "fault-recovery-exposure",
+				budgetUsd: 10,
+				maxInFlightExposureUsd: .08
+			});
+			const waits = [];
+			const errors = [];
+			handle.on("budget:exposure-wait", (event) => {
+				waits.push(event);
+				releaseChildren();
+			});
+			handle.on("agent:error", (event) => errors.push(event));
+			setTimeout(() => releaseChildren(), 4e3).unref?.();
+			const outcome = await handle.result;
+			const entries = await store.load("fault-recovery-exposure");
+			const settle = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
+			const childTerminals = entries.filter((entry) => entry.kind === "agent" && entry.scope.startsWith("agent:") && entry.status !== "running");
+			const invoice = invoiceFromJournal(entries, (servedBy, usage) => {
+				return usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10;
+			});
+			return {
+				observation: {
+					matched: outcome.status === "ok" && outcome.value === "joined after the wait" && waits.length >= 1 && waits[0]?.willWait === true && errors.length === 0 && settle !== void 0 && childTerminals.length === 2 && childTerminals.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests,
+					detail: `run '${outcome.status}' with ${String(waits.length)} exposure wait(s) (first willWait=${String(waits[0]?.willWait)}), ${String(errors.length)} agent:error, ${String(childTerminals.length)} child terminals before settle seq ${String(settle?.seq)}, wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}`
+				},
+				artifacts: [
+					jsonArtifact("outcome.json", {
+						status: outcome.status,
+						value: outcome.value ?? null,
+						envelope: outcome.envelope
+					}),
+					jsonArtifact("events.json", {
+						waits,
+						errors
+					}),
+					jsonArtifact("journal.json", entries)
+				]
+			};
+		}
 	}
 ];
 /** The scenario names in run order. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.195.0",
+  "version": "1.197.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,11 +22,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/anthropic": "1.195.0",
-    "@rulvar/core": "1.195.0",
-    "@rulvar/plan": "1.195.0",
-    "@rulvar/testing": "1.195.0",
-    "@rulvar/openai": "1.195.0"
+    "@rulvar/core": "1.197.0",
+    "@rulvar/anthropic": "1.197.0",
+    "@rulvar/openai": "1.197.0",
+    "@rulvar/plan": "1.197.0",
+    "@rulvar/testing": "1.197.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",