@rulvar/evals 1.209.0 → 1.211.0

package/dist/index.js

@@ -2804,6 +2804,263 @@ const SCENARIOS = [
 				]
 			};
 		}
+	},
+	{
+		name: "parity-quiescence-deadlock",
+		doctrine: "the parity deadlock shape ends in an exhausted terminal, never a silent exit: drained children die typed 'exposure-drained' at zero provider attempts (RV2001/RV2002), the root forced-finishes partial (RV1902), and run_settle seals a one-denominator journal (RV2003); the parity rerun exited mid-run with none of these",
+		async run() {
+			const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
+			let orchTurn = 0;
+			const calls = [];
+			const toolEvents = (name, args, id) => [
+				{
+					type: "tool-call-start",
+					id,
+					name
+				},
+				{
+					type: "tool-call-delta",
+					id,
+					argsTextDelta: JSON.stringify(args)
+				},
+				{
+					type: "tool-call-end",
+					id,
+					args
+				}
+			];
+			const adapter = {
+				id: "fake",
+				calls,
+				caps: () => ({
+					contextWindow: 2e5,
+					maxOutputTokens: 16e3,
+					structuredOutput: "native",
+					supportsTemperature: true,
+					supportsParallelTools: true,
+					reasoningEfforts: [],
+					pricing: {
+						inputUsdPerMTok: 1,
+						outputUsdPerMTok: 10
+					}
+				}),
+				async *stream(req) {
+					const call = calls.length;
+					calls.push(req);
+					await Promise.resolve();
+					if (agentTypeOfReq(req) !== "") {
+						yield {
+							type: "text-delta",
+							text: "unreachable worker"
+						};
+						yield {
+							type: "finish",
+							finish: { reason: "stop" },
+							usage: {
+								inputTokens: 10,
+								outputTokens: 5,
+								cacheReadTokens: 0,
+								cacheWriteTokens: 0
+							}
+						};
+						return;
+					}
+					orchTurn += 1;
+					const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
+						agentType: "worker",
+						prompt: "research A"
+					}, `id-${String(call)}-0`).concat(toolEvents("spawn_agent", {
+						agentType: "worker",
+						prompt: "research B"
+					}, `id-${String(call)}-1`), toolEvents("spawn_agent", {
+						agentType: "worker",
+						prompt: "research C"
+					}, `id-${String(call)}-2`)) : toolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
+					for (const event of turn) yield event;
+					yield {
+						type: "finish",
+						finish: { reason: "tool-calls" },
+						usage: {
+							inputTokens: 10,
+							outputTokens: orchTurn === 1 ? 2e3 : 5,
+							cacheReadTokens: 0,
+							cacheWriteTokens: 0
+						}
+					};
+				}
+			};
+			const store = new InMemoryStore();
+			const engine = createEngine({
+				adapters: [adapter],
+				stores: { journal: store },
+				defaults: {
+					routing: {
+						loop: "fake:model",
+						orchestrate: "fake:model"
+					},
+					profiles: { worker: {
+						description: "the oversized worker",
+						limits: { maxOutputTokensPerTurn: 1e4 }
+					} }
+				}
+			});
+			const wf = makeOrchestratorWorkflow("the parity shape", { limits: { maxOutputTokensPerTurn: 2500 } });
+			const waits = [];
+			const handle = engine.run(wf, void 0, {
+				runId: "fault-parity-quiescence",
+				budgetUsd: 10,
+				maxInFlightExposureUsd: .04
+			});
+			handle.on("budget:exposure-wait", (event) => waits.push(event));
+			const outcome = await handle.result;
+			const entries = await store.load("fault-parity-quiescence");
+			const drained = entries.filter((entry) => entry.kind === "agent" && entry.status === "error" && (entry.error?.data)?.reason === "exposure-drained");
+			const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
+			const settle = [...entries].reverse().find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
+			const agentEntries = entries.filter((entry) => entry.kind === "agent");
+			const rosterClosed = agentEntries.filter((entry) => entry.status === "running").map((entry) => entry.seq).every((seq) => agentEntries.some((entry) => entry.ref === seq && entry.status !== "running"));
+			const invoice = invoiceFromJournal(entries, (servedBy, usage) => usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10);
+			const envelope = outcome.value;
+			return {
+				observation: {
+					matched: outcome.status === "exhausted" && envelope?.forcedFinishFallback === true && envelope.completion === "partial" && drained.length === 3 && workerCalls.length === 0 && waits.some((event) => event.scope === "child" && event.willWait === false) && settle !== void 0 && rosterClosed && agentEntries.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests && invoice.unsettled === void 0,
+					detail: `run '${outcome.status}' (forcedFinishFallback=${String(envelope?.forcedFinishFallback)}, completion=${String(envelope?.completion)}); ${String(drained.length)} drained child terminal(s) at ${String(workerCalls.length)} worker call(s); roster closed=${String(rosterClosed)} before settle seq ${String(settle?.seq)}; wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}, unsettled lane absent=${String(invoice.unsettled === void 0)}`
+				},
+				artifacts: [
+					jsonArtifact("outcome.json", {
+						status: outcome.status,
+						value: outcome.value ?? null,
+						envelope: outcome.envelope
+					}),
+					jsonArtifact("events.json", { waits }),
+					jsonArtifact("journal.json", entries)
+				]
+			};
+		}
+	},
+	{
+		name: "parity-sequential-roster-floor",
+		doctrine: "a seat-by-seat roster under an unreachable acceptance floor refuses its FIRST seat typed 'roster_floor' with the whole-roster arithmetic journaled and zero paid children (RV2005): the parity arm paid three seats the settle verdict was bound to reject",
+		async run() {
+			const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
+			let orchTurn = 0;
+			const calls = [];
+			const toolEvents = (name, args, id) => [
+				{
+					type: "tool-call-start",
+					id,
+					name
+				},
+				{
+					type: "tool-call-delta",
+					id,
+					argsTextDelta: JSON.stringify(args)
+				},
+				{
+					type: "tool-call-end",
+					id,
+					args
+				}
+			];
+			const adapter = {
+				id: "fake",
+				calls,
+				caps: () => ({
+					contextWindow: 2e5,
+					maxOutputTokens: 16e3,
+					structuredOutput: "native",
+					supportsTemperature: true,
+					supportsParallelTools: true,
+					reasoningEfforts: [],
+					pricing: {
+						inputUsdPerMTok: 1,
+						outputUsdPerMTok: 10
+					}
+				}),
+				async *stream(req) {
+					const call = calls.length;
+					calls.push(req);
+					await Promise.resolve();
+					if (agentTypeOfReq(req) !== "") {
+						yield {
+							type: "text-delta",
+							text: "seated worker"
+						};
+						yield {
+							type: "finish",
+							finish: { reason: "stop" },
+							usage: {
+								inputTokens: 10,
+								outputTokens: 5,
+								cacheReadTokens: 0,
+								cacheWriteTokens: 0
+							}
+						};
+						return;
+					}
+					orchTurn += 1;
+					const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
+						agentType: "worker",
+						prompt: "seat 1"
+					}, `id-${String(call)}-0`) : toolEvents("finish", { result: "stopped early" }, `id-${String(call)}-0`);
+					for (const event of turn) yield event;
+					yield {
+						type: "finish",
+						finish: { reason: "tool-calls" },
+						usage: {
+							inputTokens: 10,
+							outputTokens: 5,
+							cacheReadTokens: 0,
+							cacheWriteTokens: 0
+						}
+					};
+				}
+			};
+			const store = new InMemoryStore();
+			const engine = createEngine({
+				adapters: [adapter],
+				stores: { journal: store },
+				defaults: {
+					routing: {
+						loop: "fake:model",
+						orchestrate: "fake:model"
+					},
+					profiles: { worker: {
+						description: "the estimated worker",
+						estCost: .7
+					} }
+				}
+			});
+			const wf = makeOrchestratorWorkflow("the seat-by-seat parity roster", { acceptance: {
+				childPolicy: "all-ok",
+				minSpawnedChildren: 4
+			} });
+			const rejects = [];
+			const handle = engine.run(wf, void 0, {
+				runId: "fault-parity-roster",
+				budgetUsd: 2.5
+			});
+			handle.on("spawn:rejected", (event) => rejects.push(event));
+			const outcome = await handle.result;
+			const entries = await store.load("fault-parity-roster");
+			const decision = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "spawn-admission" && entry.value.decision?.verdict?.reason?.code === "roster_floor");
+			const reason = (decision?.value)?.decision.verdict.reason;
+			const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
+			return {
+				observation: {
+					matched: workerCalls.length === 0 && rejects.some((event) => event.code === "roster_floor") && decision !== void 0 && reason?.floor === 4 && reason.seatsRemaining === 4 && Math.abs((reason.perSeatProjectionUsd ?? 0) - .7) < 1e-9 && outcome.status === "error",
+					detail: `run '${outcome.status}' with ${String(workerCalls.length)} worker call(s); roster_floor rejected=${String(rejects.some((e) => e.code === "roster_floor"))}, journaled arithmetic floor=${String(reason?.floor)} seatsRemaining=${String(reason?.seatsRemaining)} perSeat=${String(reason?.perSeatProjectionUsd)}`
+				},
+				artifacts: [
+					jsonArtifact("outcome.json", {
+						status: outcome.status,
+						error: outcome.error ?? null
+					}),
+					jsonArtifact("events.json", { rejects }),
+					jsonArtifact("journal.json", entries)
+				]
+			};
+		}
 	}
 ];
 /** The scenario names in run order. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/evals",
-  "version": "1.209.0",
+  "version": "1.211.0",
   "description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
   "type": "module",
   "license": "Apache-2.0",
@@ -22,11 +22,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/anthropic": "1.209.0",
-    "@rulvar/core": "1.209.0",
-    "@rulvar/testing": "1.209.0",
-    "@rulvar/openai": "1.209.0",
-    "@rulvar/plan": "1.209.0"
+    "@rulvar/anthropic": "1.211.0",
+    "@rulvar/core": "1.211.0",
+    "@rulvar/plan": "1.211.0",
+    "@rulvar/openai": "1.211.0",
+    "@rulvar/testing": "1.211.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",