@rulvar/evals 1.225.0 → 1.226.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +836 -422
  2. package/package.json +6 -6
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire } from "node:module";
2
- import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
2
+ import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, citedValueValidator, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, evidenceGradeValidator, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
3
3
  import { createHash } from "node:crypto";
4
4
  import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
5
5
  import { tmpdir } from "node:os";
@@ -2531,6 +2531,582 @@ const TIER_CROSSING_TURN = {
2531
2531
  cacheWriteTokens: 0
2532
2532
  }]
2533
2533
  };
2534
+ const tierCrossingLiveParity = {
2535
+ name: "tier-crossing-live-parity",
2536
+ doctrine: "a long-context tier crossed by the sum of one call that no single mid-stream slice reached debits the live budget exactly like the settled fold (RV1101): the per-call marginal meter re-prices the whole call at the crossing slice, live and settled read the same dollars, and a ceiling between the per-slice and tiered readings severs the run instead of settling ok over its own hard cap",
2537
+ async run() {
2538
+ const runTier = async (runId, budgetUsd) => {
2539
+ const handle = createEngine({
2540
+ adapters: [wireAdapter("tier", [TIER_CROSSING_TURN])],
2541
+ stores: { journal: new InMemoryStore() },
2542
+ defaults: { routing: { loop: "tier:model" } },
2543
+ pricing: TIER_PRICING
2544
+ }).run(echoWorkflow, void 0, {
2545
+ runId,
2546
+ budgetUsd
2547
+ });
2548
+ let maxLiveSpentUsd = 0;
2549
+ const ladder = [];
2550
+ handle.on("budget:update", (event) => {
2551
+ if (event.spentUsd > maxLiveSpentUsd) maxLiveSpentUsd = event.spentUsd;
2552
+ if (event.spentUsd > 0 && ladder[ladder.length - 1] !== event.spentUsd) ladder.push(event.spentUsd);
2553
+ });
2554
+ return {
2555
+ outcome: await handle.result,
2556
+ maxLiveSpentUsd,
2557
+ ladder
2558
+ };
2559
+ };
2560
+ const parity = await runTier("fault-tier-parity", 100);
2561
+ const capped = await runTier("fault-tier-capped", 4);
2562
+ const expectedLadder = [
2563
+ 1.5,
2564
+ 5,
2565
+ 5.75
2566
+ ];
2567
+ let cursor = 0;
2568
+ for (const reading of parity.ladder) if (reading === expectedLadder[cursor]) cursor += 1;
2569
+ const ladderDriven = cursor === expectedLadder.length;
2570
+ return {
2571
+ observation: {
2572
+ matched: parity.outcome.status === "ok" && parity.outcome.cost.totalUsd === 5.75 && parity.maxLiveSpentUsd === 5.75 && ladderDriven && capped.outcome.status !== "ok" && capped.outcome.cost.totalUsd === 5.75 && capped.maxLiveSpentUsd === capped.outcome.cost.totalUsd,
2573
+ detail: `a 250k call arriving as 150k + 100k slices (no slice crossed the 200k tier) debited live=${String(parity.maxLiveSpentUsd)} USD and settled=${String(parity.outcome.cost.totalUsd)} USD on the same run over the live ladder ${parity.ladder.join(" -> ")}; under the $4 ceiling between the per-slice ($3.00) and tiered readings the run settled '${capped.outcome.status}' at ${String(capped.outcome.cost.totalUsd)} USD with live=${String(capped.maxLiveSpentUsd)}`
2574
+ },
2575
+ artifacts: [jsonArtifact("parity-run.json", {
2576
+ status: parity.outcome.status,
2577
+ settledUsd: parity.outcome.cost.totalUsd,
2578
+ liveUsd: parity.maxLiveSpentUsd,
2579
+ ladder: parity.ladder,
2580
+ usage: parity.outcome.usage
2581
+ }), jsonArtifact("capped-run.json", {
2582
+ status: capped.outcome.status,
2583
+ settledUsd: capped.outcome.cost.totalUsd,
2584
+ liveUsd: capped.maxLiveSpentUsd,
2585
+ ladder: capped.ladder,
2586
+ error: capped.outcome.error?.message ?? null
2587
+ })]
2588
+ };
2589
+ }
2590
+ };
2591
+ /**
2592
+ * RV1905, the four-role benchmark's primary arm as a permanent gate:
2593
+ * the exact $6.00 / $4.50 cap / $1.00 synthesis / four 0.62 workers
2594
+ * configuration whose preflight read 5/5 green while the live gate
2595
+ * refused the third worker. Since RV1901 the projection holds the
2596
+ * synthesis reserve like both live gates, seats 2 of 4, exposes the
2597
+ * equation terms, and names the roster shortfall as an error finding.
2598
+ */
2599
+ const benchmarkPrimaryPreflightParity = {
2600
+ name: "benchmark-primary-preflight-parity",
2601
+ doctrine: "the admission projection holds the synthesis reserve exactly like the live gates (RV1901): the benchmark primary configuration projects 2 of 4 seats with the equation terms exposed and the roster shortfall named admission-below-roster-floor, never the 5/5 green wave the live gate is bound to refuse",
2602
+ run() {
2603
+ const report = preflightEstimate({
2604
+ engine: {
2605
+ adapters: [new FakeAdapter({ agents: { "*": "unused" } })],
2606
+ defaults: { routing: {
2607
+ loop: FAKE_MODEL_REF,
2608
+ orchestrate: FAKE_MODEL_REF,
2609
+ synthesize: FAKE_MODEL_REF
2610
+ } }
2611
+ },
2612
+ run: { budgetUsd: 6 },
2613
+ orchestrator: {
2614
+ budget: {
2615
+ capUsd: 4.5,
2616
+ capFraction: 1,
2617
+ synthesisReserveUsd: 1
2618
+ },
2619
+ synthesis: { limits: { maxTurns: 2 } },
2620
+ acceptance: { minSpawnedChildren: 4 }
2621
+ },
2622
+ spawns: [
2623
+ "product",
2624
+ "finops",
2625
+ "durability",
2626
+ "adversarial"
2627
+ ].map((label) => ({
2628
+ label,
2629
+ estCost: .62
2630
+ }))
2631
+ });
2632
+ const denied = report.admission.wave.filter((row) => !row.admitted);
2633
+ const floorFinding = report.findings.find((finding) => finding.code === "admission-below-roster-floor");
2634
+ const matched = report.admission.admitted === 3 && report.admission.denied === 2 && denied.every((row) => row.deniedBy === "budget") && report.admission.synthesisReserveUsd === 1 && Math.abs((report.admission.wave[0]?.reserveUsd ?? 0) - 3.5) < 1e-9 && Math.abs((report.admission.wave[0]?.heldAtEvaluationUsd ?? 0) - 1) < 1e-9 && floorFinding?.severity === "error";
2635
+ return Promise.resolve({
2636
+ observation: {
2637
+ matched,
2638
+ detail: `the wave seats ${String(report.admission.admitted)} of 5 rows (${String(report.admission.denied)} denied by budget), synthesis hold ${String(report.admission.synthesisReserveUsd)} USD, roster finding '${String(floorFinding?.code)}' at severity '${String(floorFinding?.severity)}'`
2639
+ },
2640
+ artifacts: [jsonArtifact("preflight.json", report)]
2641
+ });
2642
+ }
2643
+ };
2644
+ /**
2645
+ * RV1905, the four-role benchmark's recovery arm as a permanent gate:
2646
+ * four admitted children still finalizing, the root's next turn refused
2647
+ * pre-wire by the exposure cap. Since RV1902 the root parks and
2648
+ * retries; since RV1903 every child reaches a journaled terminal before
2649
+ * run_settle; since RV1904 the terminal and the invoice share one wire
2650
+ * denominator. One scenario drives the whole fixed pipeline.
2651
+ */
2652
+ const benchmarkRecoveryRootExposure = {
2653
+ name: "benchmark-recovery-root-exposure",
2654
+ doctrine: "a root turn refused by the in-flight exposure cap beside live children parks and completes after a hold releases (RV1902), every child terminal precedes run_settle (RV1903), and the terminal envelope and the invoice cardinality agree on the wire count (RV1904): the recovery arm terminal-failed on every one of these",
2655
+ async run() {
2656
+ let releaseChildren = () => {};
2657
+ const childrenGate = new Promise((resolve) => {
2658
+ releaseChildren = resolve;
2659
+ });
2660
+ const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
2661
+ let orchTurn = 0;
2662
+ const calls = [];
2663
+ const toolEvents = (name, args, call) => [
2664
+ {
2665
+ type: "tool-call-start",
2666
+ id: `id-${String(call)}-0`,
2667
+ name
2668
+ },
2669
+ {
2670
+ type: "tool-call-delta",
2671
+ id: `id-${String(call)}-0`,
2672
+ argsTextDelta: JSON.stringify(args)
2673
+ },
2674
+ {
2675
+ type: "tool-call-end",
2676
+ id: `id-${String(call)}-0`,
2677
+ args
2678
+ }
2679
+ ];
2680
+ const handlesIn = (req) => {
2681
+ const found = [];
2682
+ for (const msg of req.messages) for (const part of msg.parts) if (part.type === "tool-result") {
2683
+ const result = part.result;
2684
+ if (typeof result.handle === "number") found.push(result.handle);
2685
+ }
2686
+ return found;
2687
+ };
2688
+ const adapter = {
2689
+ id: "fake",
2690
+ calls,
2691
+ caps: () => ({
2692
+ contextWindow: 2e5,
2693
+ maxOutputTokens: 4096,
2694
+ structuredOutput: "native",
2695
+ supportsTemperature: true,
2696
+ supportsParallelTools: true,
2697
+ reasoningEfforts: [],
2698
+ pricing: {
2699
+ inputUsdPerMTok: 1,
2700
+ outputUsdPerMTok: 10
2701
+ }
2702
+ }),
2703
+ async *stream(req, signal) {
2704
+ const call = calls.length;
2705
+ calls.push(req);
2706
+ if (agentTypeOfReq(req) !== "") {
2707
+ await Promise.race([childrenGate, new Promise((resolve) => {
2708
+ signal?.addEventListener("abort", () => resolve(), { once: true });
2709
+ })]);
2710
+ if (signal?.aborted === true) return;
2711
+ yield {
2712
+ type: "text-delta",
2713
+ text: "worked"
2714
+ };
2715
+ yield {
2716
+ type: "finish",
2717
+ finish: { reason: "stop" },
2718
+ usage: {
2719
+ inputTokens: 10,
2720
+ outputTokens: 5,
2721
+ cacheReadTokens: 0,
2722
+ cacheWriteTokens: 0
2723
+ }
2724
+ };
2725
+ return;
2726
+ }
2727
+ orchTurn += 1;
2728
+ const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
2729
+ agentType: "worker",
2730
+ prompt: "task A"
2731
+ }, call).concat(toolEvents("spawn_agent", {
2732
+ agentType: "worker",
2733
+ prompt: "task B"
2734
+ }, call + 1e3)) : orchTurn === 2 ? toolEvents("await_all", { handles: handlesIn(req) }, call) : toolEvents("finish", { result: "joined after the wait" }, call);
2735
+ for (const event of turn) yield event;
2736
+ yield {
2737
+ type: "finish",
2738
+ finish: { reason: "tool-calls" },
2739
+ usage: {
2740
+ inputTokens: 10,
2741
+ outputTokens: 5,
2742
+ cacheReadTokens: 0,
2743
+ cacheWriteTokens: 0
2744
+ }
2745
+ };
2746
+ }
2747
+ };
2748
+ const store = new InMemoryStore();
2749
+ const engine = createEngine({
2750
+ adapters: [adapter],
2751
+ stores: { journal: store },
2752
+ defaults: {
2753
+ routing: {
2754
+ loop: "fake:model",
2755
+ orchestrate: "fake:model"
2756
+ },
2757
+ profiles: { worker: {
2758
+ description: "the gated worker",
2759
+ limits: { maxOutputTokensPerTurn: 2500 }
2760
+ } }
2761
+ }
2762
+ });
2763
+ const wf = makeOrchestratorWorkflow("join the gated wave", { limits: { maxOutputTokensPerTurn: 4e3 } });
2764
+ const handle = engine.run(wf, void 0, {
2765
+ runId: "fault-recovery-exposure",
2766
+ budgetUsd: 10,
2767
+ maxInFlightExposureUsd: .08
2768
+ });
2769
+ const waits = [];
2770
+ const errors = [];
2771
+ handle.on("budget:exposure-wait", (event) => {
2772
+ waits.push(event);
2773
+ releaseChildren();
2774
+ });
2775
+ handle.on("agent:error", (event) => errors.push(event));
2776
+ setTimeout(() => releaseChildren(), 4e3).unref?.();
2777
+ const outcome = await handle.result;
2778
+ const entries = await store.load("fault-recovery-exposure");
2779
+ const settle = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
2780
+ const childTerminals = entries.filter((entry) => entry.kind === "agent" && entry.scope.startsWith("agent:") && entry.status !== "running");
2781
+ const invoice = invoiceFromJournal(entries, (servedBy, usage) => {
2782
+ return usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10;
2783
+ });
2784
+ return {
2785
+ observation: {
2786
+ matched: outcome.status === "ok" && outcome.value === "joined after the wait" && waits.length >= 1 && waits[0]?.willWait === true && errors.length === 0 && settle !== void 0 && childTerminals.length === 2 && childTerminals.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests,
2787
+ detail: `run '${outcome.status}' with ${String(waits.length)} exposure wait(s) (first willWait=${String(waits[0]?.willWait)}), ${String(errors.length)} agent:error, ${String(childTerminals.length)} child terminals before settle seq ${String(settle?.seq)}, wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}`
2788
+ },
2789
+ artifacts: [
2790
+ jsonArtifact("outcome.json", {
2791
+ status: outcome.status,
2792
+ value: outcome.value ?? null,
2793
+ envelope: outcome.envelope
2794
+ }),
2795
+ jsonArtifact("events.json", {
2796
+ waits,
2797
+ errors
2798
+ }),
2799
+ jsonArtifact("journal.json", entries)
2800
+ ]
2801
+ };
2802
+ }
2803
+ };
2804
+ /**
2805
+ * The third parity rerun's terminal shape as a permanent gate (RV2009):
2806
+ * the coordination turn eats most of the exposure cap, every spawned
2807
+ * worker is refused DRAINED (typed 'exposure-drained', zero provider
2808
+ * attempts, the RV2002 seat instead of the parity mid-research death),
2809
+ * the next coordination turn drains too, and the run forced-finishes
2810
+ * partial (RV1902) into an exhausted terminal with a sealed journal:
2811
+ * run_settle recorded after every agent terminal, the settled fold and
2812
+ * the invoice in one denominator (the RV2003 no-silent-exit invariant,
2813
+ * where the parity process left a forever-running root and no settle).
2814
+ */
2815
+ const parityQuiescenceDeadlock = {
2816
+ name: "parity-quiescence-deadlock",
2817
+ doctrine: "the parity deadlock shape ends in an exhausted terminal, never a silent exit: drained children die typed 'exposure-drained' at zero provider attempts (RV2001/RV2002), the root forced-finishes partial (RV1902), and run_settle seals a one-denominator journal (RV2003); the parity rerun exited mid-run with none of these",
2818
+ async run() {
2819
+ const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
2820
+ let orchTurn = 0;
2821
+ const calls = [];
2822
+ const toolEvents = (name, args, id) => [
2823
+ {
2824
+ type: "tool-call-start",
2825
+ id,
2826
+ name
2827
+ },
2828
+ {
2829
+ type: "tool-call-delta",
2830
+ id,
2831
+ argsTextDelta: JSON.stringify(args)
2832
+ },
2833
+ {
2834
+ type: "tool-call-end",
2835
+ id,
2836
+ args
2837
+ }
2838
+ ];
2839
+ const adapter = {
2840
+ id: "fake",
2841
+ calls,
2842
+ caps: () => ({
2843
+ contextWindow: 2e5,
2844
+ maxOutputTokens: 16e3,
2845
+ structuredOutput: "native",
2846
+ supportsTemperature: true,
2847
+ supportsParallelTools: true,
2848
+ reasoningEfforts: [],
2849
+ pricing: {
2850
+ inputUsdPerMTok: 1,
2851
+ outputUsdPerMTok: 10
2852
+ }
2853
+ }),
2854
+ async *stream(req) {
2855
+ const call = calls.length;
2856
+ calls.push(req);
2857
+ await Promise.resolve();
2858
+ if (agentTypeOfReq(req) !== "") {
2859
+ yield {
2860
+ type: "text-delta",
2861
+ text: "unreachable worker"
2862
+ };
2863
+ yield {
2864
+ type: "finish",
2865
+ finish: { reason: "stop" },
2866
+ usage: {
2867
+ inputTokens: 10,
2868
+ outputTokens: 5,
2869
+ cacheReadTokens: 0,
2870
+ cacheWriteTokens: 0
2871
+ }
2872
+ };
2873
+ return;
2874
+ }
2875
+ orchTurn += 1;
2876
+ const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
2877
+ agentType: "worker",
2878
+ prompt: "research A"
2879
+ }, `id-${String(call)}-0`).concat(toolEvents("spawn_agent", {
2880
+ agentType: "worker",
2881
+ prompt: "research B"
2882
+ }, `id-${String(call)}-1`), toolEvents("spawn_agent", {
2883
+ agentType: "worker",
2884
+ prompt: "research C"
2885
+ }, `id-${String(call)}-2`)) : toolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
2886
+ for (const event of turn) yield event;
2887
+ yield {
2888
+ type: "finish",
2889
+ finish: { reason: "tool-calls" },
2890
+ usage: {
2891
+ inputTokens: 10,
2892
+ outputTokens: orchTurn === 1 ? 2e3 : 5,
2893
+ cacheReadTokens: 0,
2894
+ cacheWriteTokens: 0
2895
+ }
2896
+ };
2897
+ }
2898
+ };
2899
+ const store = new InMemoryStore();
2900
+ const engine = createEngine({
2901
+ adapters: [adapter],
2902
+ stores: { journal: store },
2903
+ defaults: {
2904
+ routing: {
2905
+ loop: "fake:model",
2906
+ orchestrate: "fake:model"
2907
+ },
2908
+ profiles: { worker: {
2909
+ description: "the oversized worker",
2910
+ limits: { maxOutputTokensPerTurn: 1e4 }
2911
+ } }
2912
+ }
2913
+ });
2914
+ const wf = makeOrchestratorWorkflow("the parity shape", { limits: { maxOutputTokensPerTurn: 2500 } });
2915
+ const waits = [];
2916
+ const handle = engine.run(wf, void 0, {
2917
+ runId: "fault-parity-quiescence",
2918
+ budgetUsd: 10,
2919
+ maxInFlightExposureUsd: .04
2920
+ });
2921
+ handle.on("budget:exposure-wait", (event) => waits.push(event));
2922
+ const outcome = await handle.result;
2923
+ const entries = await store.load("fault-parity-quiescence");
2924
+ const drained = entries.filter((entry) => entry.kind === "agent" && entry.status === "error" && (entry.error?.data)?.reason === "exposure-drained");
2925
+ const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
2926
+ const settle = [...entries].reverse().find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
2927
+ const agentEntries = entries.filter((entry) => entry.kind === "agent");
2928
+ const rosterClosed = agentEntries.filter((entry) => entry.status === "running").map((entry) => entry.seq).every((seq) => agentEntries.some((entry) => entry.ref === seq && entry.status !== "running"));
2929
+ const invoice = invoiceFromJournal(entries, (servedBy, usage) => usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10);
2930
+ const envelope = outcome.value;
2931
+ return {
2932
+ observation: {
2933
+ matched: outcome.status === "exhausted" && envelope?.forcedFinishFallback === true && envelope.completion === "partial" && drained.length === 3 && workerCalls.length === 0 && waits.some((event) => event.scope === "child" && event.willWait === false) && settle !== void 0 && rosterClosed && agentEntries.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests && invoice.unsettled === void 0,
2934
+ detail: `run '${outcome.status}' (forcedFinishFallback=${String(envelope?.forcedFinishFallback)}, completion=${String(envelope?.completion)}); ${String(drained.length)} drained child terminal(s) at ${String(workerCalls.length)} worker call(s); roster closed=${String(rosterClosed)} before settle seq ${String(settle?.seq)}; wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}, unsettled lane absent=${String(invoice.unsettled === void 0)}`
2935
+ },
2936
+ artifacts: [
2937
+ jsonArtifact("outcome.json", {
2938
+ status: outcome.status,
2939
+ value: outcome.value ?? null,
2940
+ envelope: outcome.envelope
2941
+ }),
2942
+ jsonArtifact("events.json", { waits }),
2943
+ jsonArtifact("journal.json", entries)
2944
+ ]
2945
+ };
2946
+ }
2947
+ };
2948
+ /**
2949
+ * The parity roster paid seat by seat under an unreachable floor, as a
2950
+ * permanent gate (RV2009): every SINGLE spawn admission projects the
2951
+ * whole remaining roster (RV2005) and the FIRST seat refuses typed
2952
+ * 'roster_floor' with the arithmetic journaled, zero paid children;
2953
+ * the parity arm paid three of four.
2954
+ */
2955
+ const paritySequentialRosterFloor = {
2956
+ name: "parity-sequential-roster-floor",
2957
+ doctrine: "a seat-by-seat roster under an unreachable acceptance floor refuses its FIRST seat typed 'roster_floor' with the whole-roster arithmetic journaled and zero paid children (RV2005): the parity arm paid three seats the settle verdict was bound to reject",
2958
+ async run() {
2959
+ const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
2960
+ let orchTurn = 0;
2961
+ const calls = [];
2962
+ const toolEvents = (name, args, id) => [
2963
+ {
2964
+ type: "tool-call-start",
2965
+ id,
2966
+ name
2967
+ },
2968
+ {
2969
+ type: "tool-call-delta",
2970
+ id,
2971
+ argsTextDelta: JSON.stringify(args)
2972
+ },
2973
+ {
2974
+ type: "tool-call-end",
2975
+ id,
2976
+ args
2977
+ }
2978
+ ];
2979
+ const adapter = {
2980
+ id: "fake",
2981
+ calls,
2982
+ caps: () => ({
2983
+ contextWindow: 2e5,
2984
+ maxOutputTokens: 16e3,
2985
+ structuredOutput: "native",
2986
+ supportsTemperature: true,
2987
+ supportsParallelTools: true,
2988
+ reasoningEfforts: [],
2989
+ pricing: {
2990
+ inputUsdPerMTok: 1,
2991
+ outputUsdPerMTok: 10
2992
+ }
2993
+ }),
2994
+ async *stream(req) {
2995
+ const call = calls.length;
2996
+ calls.push(req);
2997
+ await Promise.resolve();
2998
+ if (agentTypeOfReq(req) !== "") {
2999
+ yield {
3000
+ type: "text-delta",
3001
+ text: "seated worker"
3002
+ };
3003
+ yield {
3004
+ type: "finish",
3005
+ finish: { reason: "stop" },
3006
+ usage: {
3007
+ inputTokens: 10,
3008
+ outputTokens: 5,
3009
+ cacheReadTokens: 0,
3010
+ cacheWriteTokens: 0
3011
+ }
3012
+ };
3013
+ return;
3014
+ }
3015
+ orchTurn += 1;
3016
+ const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
3017
+ agentType: "worker",
3018
+ prompt: "seat 1"
3019
+ }, `id-${String(call)}-0`) : toolEvents("finish", { result: "stopped early" }, `id-${String(call)}-0`);
3020
+ for (const event of turn) yield event;
3021
+ yield {
3022
+ type: "finish",
3023
+ finish: { reason: "tool-calls" },
3024
+ usage: {
3025
+ inputTokens: 10,
3026
+ outputTokens: 5,
3027
+ cacheReadTokens: 0,
3028
+ cacheWriteTokens: 0
3029
+ }
3030
+ };
3031
+ }
3032
+ };
3033
+ const store = new InMemoryStore();
3034
+ const engine = createEngine({
3035
+ adapters: [adapter],
3036
+ stores: { journal: store },
3037
+ defaults: {
3038
+ routing: {
3039
+ loop: "fake:model",
3040
+ orchestrate: "fake:model"
3041
+ },
3042
+ profiles: { worker: {
3043
+ description: "the estimated worker",
3044
+ estCost: .7
3045
+ } }
3046
+ }
3047
+ });
3048
+ const wf = makeOrchestratorWorkflow("the seat-by-seat parity roster", { acceptance: {
3049
+ childPolicy: "all-ok",
3050
+ minSpawnedChildren: 4
3051
+ } });
3052
+ const rejects = [];
3053
+ const handle = engine.run(wf, void 0, {
3054
+ runId: "fault-parity-roster",
3055
+ budgetUsd: 2.5
3056
+ });
3057
+ handle.on("spawn:rejected", (event) => rejects.push(event));
3058
+ const outcome = await handle.result;
3059
+ const entries = await store.load("fault-parity-roster");
3060
+ const decision = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "spawn-admission" && entry.value.decision?.verdict?.reason?.code === "roster_floor");
3061
+ const reason = (decision?.value)?.decision.verdict.reason;
3062
+ const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
3063
+ return {
3064
+ observation: {
3065
+ matched: workerCalls.length === 0 && rejects.some((event) => event.code === "roster_floor") && decision !== void 0 && reason?.floor === 4 && reason.seatsRemaining === 4 && Math.abs((reason.perSeatProjectionUsd ?? 0) - .7) < 1e-9 && outcome.status === "error",
3066
+ detail: `run '${outcome.status}' with ${String(workerCalls.length)} worker call(s); roster_floor rejected=${String(rejects.some((e) => e.code === "roster_floor"))}, journaled arithmetic floor=${String(reason?.floor)} seatsRemaining=${String(reason?.seatsRemaining)} perSeat=${String(reason?.perSeatProjectionUsd)}`
3067
+ },
3068
+ artifacts: [
3069
+ jsonArtifact("outcome.json", {
3070
+ status: outcome.status,
3071
+ error: outcome.error ?? null
3072
+ }),
3073
+ jsonArtifact("events.json", { rejects }),
3074
+ jsonArtifact("journal.json", entries)
3075
+ ]
3076
+ };
3077
+ }
3078
+ };
3079
+ /** Shared tool-call event triple for scripted orchestrator streams. */
3080
+ const scriptedToolEvents = (name, args, id) => [
3081
+ {
3082
+ type: "tool-call-start",
3083
+ id,
3084
+ name
3085
+ },
3086
+ {
3087
+ type: "tool-call-delta",
3088
+ id,
3089
+ argsTextDelta: JSON.stringify(args)
3090
+ },
3091
+ {
3092
+ type: "tool-call-end",
3093
+ id,
3094
+ args
3095
+ }
3096
+ ];
3097
+ const scriptedAgentType = (req) => req.providerOptions?.rulvar?.agentType ?? "";
3098
+ const scriptedCaps = () => ({
3099
+ contextWindow: 2e5,
3100
+ maxOutputTokens: 16e3,
3101
+ structuredOutput: "native",
3102
+ supportsTemperature: true,
3103
+ supportsParallelTools: true,
3104
+ reasoningEfforts: [],
3105
+ pricing: {
3106
+ inputUsdPerMTok: 1,
3107
+ outputUsdPerMTok: 10
3108
+ }
3109
+ });
2534
3110
  const SCENARIOS = [
2535
3111
  inFlightExposure,
2536
3112
  duplicateQuotaRule,
@@ -2551,311 +3127,209 @@ const SCENARIOS = [
2551
3127
  pauseTurnRealAdapter,
2552
3128
  statementSettleableGuard,
2553
3129
  supersededTerminalHonesty,
3130
+ tierCrossingLiveParity,
3131
+ benchmarkPrimaryPreflightParity,
3132
+ benchmarkRecoveryRootExposure,
3133
+ parityQuiescenceDeadlock,
3134
+ paritySequentialRosterFloor,
2554
3135
  {
2555
- name: "tier-crossing-live-parity",
2556
- doctrine: "a long-context tier crossed by the sum of one call that no single mid-stream slice reached debits the live budget exactly like the settled fold (RV1101): the per-call marginal meter re-prices the whole call at the crossing slice, live and settled read the same dollars, and a ceiling between the per-slice and tiered readings severs the run instead of settling ok over its own hard cap",
2557
- async run() {
2558
- const runTier = async (runId, budgetUsd) => {
2559
- const handle = createEngine({
2560
- adapters: [wireAdapter("tier", [TIER_CROSSING_TURN])],
2561
- stores: { journal: new InMemoryStore() },
2562
- defaults: { routing: { loop: "tier:model" } },
2563
- pricing: TIER_PRICING
2564
- }).run(echoWorkflow, void 0, {
2565
- runId,
2566
- budgetUsd
2567
- });
2568
- let maxLiveSpentUsd = 0;
2569
- const ladder = [];
2570
- handle.on("budget:update", (event) => {
2571
- if (event.spentUsd > maxLiveSpentUsd) maxLiveSpentUsd = event.spentUsd;
2572
- if (event.spentUsd > 0 && ladder[ladder.length - 1] !== event.spentUsd) ladder.push(event.spentUsd);
2573
- });
2574
- return {
2575
- outcome: await handle.result,
2576
- maxLiveSpentUsd,
2577
- ladder
2578
- };
2579
- };
2580
- const parity = await runTier("fault-tier-parity", 100);
2581
- const capped = await runTier("fault-tier-capped", 4);
2582
- const expectedLadder = [
2583
- 1.5,
2584
- 5,
2585
- 5.75
2586
- ];
2587
- let cursor = 0;
2588
- for (const reading of parity.ladder) if (reading === expectedLadder[cursor]) cursor += 1;
2589
- const ladderDriven = cursor === expectedLadder.length;
2590
- return {
2591
- observation: {
2592
- matched: parity.outcome.status === "ok" && parity.outcome.cost.totalUsd === 5.75 && parity.maxLiveSpentUsd === 5.75 && ladderDriven && capped.outcome.status !== "ok" && capped.outcome.cost.totalUsd === 5.75 && capped.maxLiveSpentUsd === capped.outcome.cost.totalUsd,
2593
- detail: `a 250k call arriving as 150k + 100k slices (no slice crossed the 200k tier) debited live=${String(parity.maxLiveSpentUsd)} USD and settled=${String(parity.outcome.cost.totalUsd)} USD on the same run over the live ladder ${parity.ladder.join(" -> ")}; under the $4 ceiling between the per-slice ($3.00) and tiered readings the run settled '${capped.outcome.status}' at ${String(capped.outcome.cost.totalUsd)} USD with live=${String(capped.maxLiveSpentUsd)}`
2594
- },
2595
- artifacts: [jsonArtifact("parity-run.json", {
2596
- status: parity.outcome.status,
2597
- settledUsd: parity.outcome.cost.totalUsd,
2598
- liveUsd: parity.maxLiveSpentUsd,
2599
- ladder: parity.ladder,
2600
- usage: parity.outcome.usage
2601
- }), jsonArtifact("capped-run.json", {
2602
- status: capped.outcome.status,
2603
- settledUsd: capped.outcome.cost.totalUsd,
2604
- liveUsd: capped.maxLiveSpentUsd,
2605
- ladder: capped.ladder,
2606
- error: capped.outcome.error?.message ?? null
2607
- })]
2608
- };
2609
- }
2610
- },
2611
- {
2612
- name: "benchmark-primary-preflight-parity",
2613
- doctrine: "the admission projection holds the synthesis reserve exactly like the live gates (RV1901): the benchmark primary configuration projects 2 of 4 seats with the equation terms exposed and the roster shortfall named admission-below-roster-floor, never the 5/5 green wave the live gate is bound to refuse",
2614
- run() {
2615
- const report = preflightEstimate({
2616
- engine: {
2617
- adapters: [new FakeAdapter({ agents: { "*": "unused" } })],
2618
- defaults: { routing: {
2619
- loop: FAKE_MODEL_REF,
2620
- orchestrate: FAKE_MODEL_REF,
2621
- synthesize: FAKE_MODEL_REF
2622
- } }
2623
- },
2624
- run: { budgetUsd: 6 },
2625
- orchestrator: {
2626
- budget: {
2627
- capUsd: 4.5,
2628
- capFraction: 1,
2629
- synthesisReserveUsd: 1
2630
- },
2631
- synthesis: { limits: { maxTurns: 2 } },
2632
- acceptance: { minSpawnedChildren: 4 }
2633
- },
2634
- spawns: [
2635
- "product",
2636
- "finops",
2637
- "durability",
2638
- "adversarial"
2639
- ].map((label) => ({
2640
- label,
2641
- estCost: .62
2642
- }))
2643
- });
2644
- const denied = report.admission.wave.filter((row) => !row.admitted);
2645
- const floorFinding = report.findings.find((finding) => finding.code === "admission-below-roster-floor");
2646
- const matched = report.admission.admitted === 3 && report.admission.denied === 2 && denied.every((row) => row.deniedBy === "budget") && report.admission.synthesisReserveUsd === 1 && Math.abs((report.admission.wave[0]?.reserveUsd ?? 0) - 3.5) < 1e-9 && Math.abs((report.admission.wave[0]?.heldAtEvaluationUsd ?? 0) - 1) < 1e-9 && floorFinding?.severity === "error";
2647
- return Promise.resolve({
2648
- observation: {
2649
- matched,
2650
- detail: `the wave seats ${String(report.admission.admitted)} of 5 rows (${String(report.admission.denied)} denied by budget), synthesis hold ${String(report.admission.synthesisReserveUsd)} USD, roster finding '${String(floorFinding?.code)}' at severity '${String(floorFinding?.severity)}'`
2651
- },
2652
- artifacts: [jsonArtifact("preflight.json", report)]
2653
- });
2654
- }
2655
- },
2656
- {
2657
- name: "benchmark-recovery-root-exposure",
2658
- doctrine: "a root turn refused by the in-flight exposure cap beside live children parks and completes after a hold releases (RV1902), every child terminal precedes run_settle (RV1903), and the terminal envelope and the invoice cardinality agree on the wire count (RV1904): the recovery arm terminal-failed on every one of these",
3136
+ name: "parity-reserve-line-redemption",
3137
+ doctrine: "a coordination turn refused at the reserve line folds typed 'budget-floor' and the held synthesis reserve funds the redemption whose result rides the partial envelope (RV2101); the PlanRunner extension arm produces the same fold and the same redeemed result (DEF-7)",
2659
3138
  async run() {
2660
- let releaseChildren = () => {};
2661
- const childrenGate = new Promise((resolve) => {
2662
- releaseChildren = resolve;
2663
- });
2664
- const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
2665
- let orchTurn = 0;
2666
- const calls = [];
2667
- const toolEvents = (name, args, call) => [
2668
- {
2669
- type: "tool-call-start",
2670
- id: `id-${String(call)}-0`,
2671
- name
2672
- },
2673
- {
2674
- type: "tool-call-delta",
2675
- id: `id-${String(call)}-0`,
2676
- argsTextDelta: JSON.stringify(args)
2677
- },
2678
- {
2679
- type: "tool-call-end",
2680
- id: `id-${String(call)}-0`,
2681
- args
2682
- }
2683
- ];
2684
- const handlesIn = (req) => {
2685
- const found = [];
2686
- for (const msg of req.messages) for (const part of msg.parts) if (part.type === "tool-result") {
2687
- const result = part.result;
2688
- if (typeof result.handle === "number") found.push(result.handle);
2689
- }
2690
- return found;
2691
- };
2692
- const adapter = {
2693
- id: "fake",
2694
- calls,
2695
- caps: () => ({
2696
- contextWindow: 2e5,
2697
- maxOutputTokens: 4096,
2698
- structuredOutput: "native",
2699
- supportsTemperature: true,
2700
- supportsParallelTools: true,
2701
- reasoningEfforts: [],
2702
- pricing: {
2703
- inputUsdPerMTok: 1,
2704
- outputUsdPerMTok: 10
2705
- }
2706
- }),
2707
- async *stream(req, signal) {
2708
- const call = calls.length;
2709
- calls.push(req);
2710
- if (agentTypeOfReq(req) !== "") {
2711
- await Promise.race([childrenGate, new Promise((resolve) => {
2712
- signal?.addEventListener("abort", () => resolve(), { once: true });
2713
- })]);
2714
- if (signal?.aborted === true) return;
2715
- yield {
2716
- type: "text-delta",
2717
- text: "worked"
2718
- };
3139
+ const runArm = async (mode) => {
3140
+ const calls = [];
3141
+ const adapter = {
3142
+ id: "fake",
3143
+ calls,
3144
+ caps: scriptedCaps,
3145
+ async *stream(req) {
3146
+ const call = calls.length;
3147
+ calls.push(req);
3148
+ await Promise.resolve();
3149
+ if (scriptedAgentType(req) !== "") {
3150
+ yield {
3151
+ type: "text-delta",
3152
+ text: "settled evidence"
3153
+ };
3154
+ yield {
3155
+ type: "finish",
3156
+ finish: { reason: "stop" },
3157
+ usage: {
3158
+ inputTokens: 10,
3159
+ outputTokens: 5,
3160
+ cacheReadTokens: 0,
3161
+ cacheWriteTokens: 0
3162
+ }
3163
+ };
3164
+ return;
3165
+ }
3166
+ if (!(req.tools ?? []).some((tool) => tool.name === "spawn_agent")) {
3167
+ for (const event of scriptedToolEvents("finish", { result: "the redeemed synthesis" }, `id-${String(call)}-0`)) yield event;
3168
+ yield {
3169
+ type: "finish",
3170
+ finish: { reason: "tool-calls" },
3171
+ usage: {
3172
+ inputTokens: 12,
3173
+ outputTokens: 8,
3174
+ cacheReadTokens: 0,
3175
+ cacheWriteTokens: 0
3176
+ }
3177
+ };
3178
+ return;
3179
+ }
3180
+ const turn = calls.filter((prior) => scriptedAgentType(prior) === "" && (prior.tools ?? []).some((tool) => tool.name === "spawn_agent")).length === 1 ? scriptedToolEvents("spawn_agent", {
3181
+ agentType: "worker",
3182
+ prompt: "gather A"
3183
+ }, `id-${String(call)}-0`).concat(scriptedToolEvents("spawn_agent", {
3184
+ agentType: "worker",
3185
+ prompt: "gather B"
3186
+ }, `id-${String(call)}-1`)) : scriptedToolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
3187
+ for (const event of turn) yield event;
2719
3188
  yield {
2720
3189
  type: "finish",
2721
- finish: { reason: "stop" },
3190
+ finish: { reason: "tool-calls" },
2722
3191
  usage: {
2723
3192
  inputTokens: 10,
2724
- outputTokens: 5,
3193
+ outputTokens: 1e3,
2725
3194
  cacheReadTokens: 0,
2726
3195
  cacheWriteTokens: 0
2727
3196
  }
2728
3197
  };
2729
- return;
2730
3198
  }
2731
- orchTurn += 1;
2732
- const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
2733
- agentType: "worker",
2734
- prompt: "task A"
2735
- }, call).concat(toolEvents("spawn_agent", {
2736
- agentType: "worker",
2737
- prompt: "task B"
2738
- }, call + 1e3)) : orchTurn === 2 ? toolEvents("await_all", { handles: handlesIn(req) }, call) : toolEvents("finish", { result: "joined after the wait" }, call);
2739
- for (const event of turn) yield event;
2740
- yield {
2741
- type: "finish",
2742
- finish: { reason: "tool-calls" },
2743
- usage: {
2744
- inputTokens: 10,
2745
- outputTokens: 5,
2746
- cacheReadTokens: 0,
2747
- cacheWriteTokens: 0
2748
- }
2749
- };
2750
- }
2751
- };
2752
- const store = new InMemoryStore();
2753
- const engine = createEngine({
2754
- adapters: [adapter],
2755
- stores: { journal: store },
2756
- defaults: {
2757
- routing: {
2758
- loop: "fake:model",
2759
- orchestrate: "fake:model"
3199
+ };
3200
+ const store = new InMemoryStore();
3201
+ const engine = createEngine({
3202
+ adapters: [adapter],
3203
+ stores: { journal: store },
3204
+ defaults: {
3205
+ routing: {
3206
+ loop: "fake:model",
3207
+ orchestrate: "fake:model",
3208
+ synthesize: "fake:model"
3209
+ },
3210
+ profiles: { worker: { description: "the cheap gatherer" } }
3211
+ }
3212
+ });
3213
+ const opts = {
3214
+ limits: { maxOutputTokensPerTurn: 1e3 },
3215
+ synthesis: {
3216
+ estCost: .006,
3217
+ limits: { maxOutputTokensPerTurn: 500 }
2760
3218
  },
2761
- profiles: { worker: {
2762
- description: "the gated worker",
2763
- limits: { maxOutputTokensPerTurn: 2500 }
2764
- } }
2765
- }
2766
- });
2767
- const wf = makeOrchestratorWorkflow("join the gated wave", { limits: { maxOutputTokensPerTurn: 4e3 } });
2768
- const handle = engine.run(wf, void 0, {
2769
- runId: "fault-recovery-exposure",
2770
- budgetUsd: 10,
2771
- maxInFlightExposureUsd: .08
2772
- });
2773
- const waits = [];
2774
- const errors = [];
2775
- handle.on("budget:exposure-wait", (event) => {
2776
- waits.push(event);
2777
- releaseChildren();
2778
- });
2779
- handle.on("agent:error", (event) => errors.push(event));
2780
- setTimeout(() => releaseChildren(), 4e3).unref?.();
2781
- const outcome = await handle.result;
2782
- const entries = await store.load("fault-recovery-exposure");
2783
- const settle = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
2784
- const childTerminals = entries.filter((entry) => entry.kind === "agent" && entry.scope.startsWith("agent:") && entry.status !== "running");
2785
- const invoice = invoiceFromJournal(entries, (servedBy, usage) => {
2786
- return usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10;
2787
- });
3219
+ budget: {
3220
+ capUsd: .03,
3221
+ synthesisReserveUsd: .02,
3222
+ finalizeReserveUsd: 0
3223
+ }
3224
+ };
3225
+ const runId = `fault-reserve-line-${mode}`;
3226
+ const outcome = mode === "workflow" ? await engine.run(makeOrchestratorWorkflow("the reserve line shape", opts), void 0, {
3227
+ runId,
3228
+ budgetUsd: 10
3229
+ }).result : await orchestratePlanned(engine, "the reserve line shape", opts, { runId }).result;
3230
+ const entries = await store.load(runId);
3231
+ const fallback = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finalize_fallback");
3232
+ const settle = entries.some((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
3233
+ const value = outcome.value;
3234
+ return {
3235
+ status: outcome.status,
3236
+ error: outcome.error?.message,
3237
+ forcedFinishFallback: value?.forcedFinishFallback,
3238
+ completion: value?.completion,
3239
+ result: value?.result,
3240
+ fallbackReason: (fallback?.value)?.reason,
3241
+ rootClassCalls: calls.filter((req) => scriptedAgentType(req) === "").length,
3242
+ workerOkTerminals: entries.filter((entry) => entry.kind === "agent" && entry.status === "ok").length,
3243
+ settleRecorded: settle,
3244
+ entries
3245
+ };
3246
+ };
3247
+ const arms = {
3248
+ workflow: await runArm("workflow"),
3249
+ extension: await runArm("extension")
3250
+ };
3251
+ const armMatched = (arm) => arm.status === "exhausted" && arm.forcedFinishFallback === true && arm.completion === "partial" && arm.result === "the redeemed synthesis" && arm.fallbackReason === "budget-floor" && arm.rootClassCalls === 2 && arm.settleRecorded;
2788
3252
  return {
2789
3253
  observation: {
2790
- matched: outcome.status === "ok" && outcome.value === "joined after the wait" && waits.length >= 1 && waits[0]?.willWait === true && errors.length === 0 && settle !== void 0 && childTerminals.length === 2 && childTerminals.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests,
2791
- detail: `run '${outcome.status}' with ${String(waits.length)} exposure wait(s) (first willWait=${String(waits[0]?.willWait)}), ${String(errors.length)} agent:error, ${String(childTerminals.length)} child terminals before settle seq ${String(settle?.seq)}, wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}`
3254
+ matched: armMatched(arms.workflow) && armMatched(arms.extension),
3255
+ detail: `workflow arm '${arms.workflow.status}' reason=${String(arms.workflow.fallbackReason)} result=${JSON.stringify(arms.workflow.result)} rootCalls=${String(arms.workflow.rootClassCalls)}; extension arm '${arms.extension.status}' reason=${String(arms.extension.fallbackReason)} result=${JSON.stringify(arms.extension.result)} rootCalls=${String(arms.extension.rootClassCalls)} (DEF-7 parity: both arms fold 'budget-floor' and both redeem the synthesis from the held reserve)`
2792
3256
  },
2793
3257
  artifacts: [
2794
- jsonArtifact("outcome.json", {
2795
- status: outcome.status,
2796
- value: outcome.value ?? null,
2797
- envelope: outcome.envelope
2798
- }),
2799
- jsonArtifact("events.json", {
2800
- waits,
2801
- errors
3258
+ jsonArtifact("arms.json", {
3259
+ workflow: {
3260
+ ...arms.workflow,
3261
+ entries: void 0
3262
+ },
3263
+ extension: {
3264
+ ...arms.extension,
3265
+ entries: void 0
3266
+ }
2802
3267
  }),
2803
- jsonArtifact("journal.json", entries)
3268
+ jsonArtifact("journal-workflow.json", arms.workflow.entries),
3269
+ jsonArtifact("journal-extension.json", arms.extension.entries)
2804
3270
  ]
2805
3271
  };
2806
3272
  }
2807
3273
  },
2808
3274
  {
2809
- name: "parity-quiescence-deadlock",
2810
- doctrine: "the parity deadlock shape ends in an exhausted terminal, never a silent exit: drained children die typed 'exposure-drained' at zero provider attempts (RV2001/RV2002), the root forced-finishes partial (RV1902), and run_settle seals a one-denominator journal (RV2003); the parity rerun exited mid-run with none of these",
3275
+ name: "resume-spawn-famine",
3276
+ doctrine: "a resume re-admits recovered agents without re-counting them against lifetimeSpawnCap (RV2201): the kill-mid-fan-out journal resumes to the finished dossier at the EXACT cap, with no 'lifetime spawn cap' decline journaled and only the unsettled workers re-paid; the c7 resume starved its judge and synthesis on re-counted admissions",
2811
3277
  async run() {
2812
- const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
2813
- let orchTurn = 0;
2814
- const calls = [];
2815
- const toolEvents = (name, args, id) => [
2816
- {
2817
- type: "tool-call-start",
2818
- id,
2819
- name
2820
- },
2821
- {
2822
- type: "tool-call-delta",
2823
- id,
2824
- argsTextDelta: JSON.stringify(args)
2825
- },
2826
- {
2827
- type: "tool-call-end",
2828
- id,
2829
- args
2830
- }
2831
- ];
2832
- const adapter = {
2833
- id: "fake",
2834
- calls,
2835
- caps: () => ({
2836
- contextWindow: 2e5,
2837
- maxOutputTokens: 16e3,
2838
- structuredOutput: "native",
2839
- supportsTemperature: true,
2840
- supportsParallelTools: true,
2841
- reasoningEfforts: [],
2842
- pricing: {
2843
- inputUsdPerMTok: 1,
2844
- outputUsdPerMTok: 10
2845
- }
2846
- }),
2847
- async *stream(req) {
2848
- const call = calls.length;
2849
- calls.push(req);
2850
- await Promise.resolve();
2851
- if (agentTypeOfReq(req) !== "") {
2852
- yield {
2853
- type: "text-delta",
2854
- text: "unreachable worker"
2855
- };
3278
+ const makeAdapter = () => {
3279
+ const calls = [];
3280
+ return {
3281
+ id: "fake",
3282
+ calls,
3283
+ caps: scriptedCaps,
3284
+ async *stream(req) {
3285
+ const call = calls.length;
3286
+ calls.push(req);
3287
+ await Promise.resolve();
3288
+ if (scriptedAgentType(req) !== "") {
3289
+ yield {
3290
+ type: "text-delta",
3291
+ text: "gathered evidence"
3292
+ };
3293
+ yield {
3294
+ type: "finish",
3295
+ finish: { reason: "stop" },
3296
+ usage: {
3297
+ inputTokens: 10,
3298
+ outputTokens: 5,
3299
+ cacheReadTokens: 0,
3300
+ cacheWriteTokens: 0
3301
+ }
3302
+ };
3303
+ return;
3304
+ }
3305
+ if (!(req.tools ?? []).some((tool) => tool.name === "spawn_agent")) {
3306
+ for (const event of scriptedToolEvents("finish", { result: "the resumed dossier" }, `id-${String(call)}-0`)) yield event;
3307
+ yield {
3308
+ type: "finish",
3309
+ finish: { reason: "tool-calls" },
3310
+ usage: {
3311
+ inputTokens: 12,
3312
+ outputTokens: 8,
3313
+ cacheReadTokens: 0,
3314
+ cacheWriteTokens: 0
3315
+ }
3316
+ };
3317
+ return;
3318
+ }
3319
+ const turn = calls.filter((prior) => scriptedAgentType(prior) === "" && (prior.tools ?? []).some((tool) => tool.name === "spawn_agent")).length === 1 ? scriptedToolEvents("spawn_agent", {
3320
+ agentType: "worker",
3321
+ prompt: "gather A"
3322
+ }, `id-${String(call)}-0`).concat(scriptedToolEvents("spawn_agent", {
3323
+ agentType: "worker",
3324
+ prompt: "gather B"
3325
+ }, `id-${String(call)}-1`), scriptedToolEvents("spawn_agent", {
3326
+ agentType: "worker",
3327
+ prompt: "gather C"
3328
+ }, `id-${String(call)}-2`)) : scriptedToolEvents("finish", { result: "the coordination draft" }, `id-${String(call)}-0`);
3329
+ for (const event of turn) yield event;
2856
3330
  yield {
2857
3331
  type: "finish",
2858
- finish: { reason: "stop" },
3332
+ finish: { reason: "tool-calls" },
2859
3333
  usage: {
2860
3334
  inputTokens: 10,
2861
3335
  outputTokens: 5,
@@ -2863,153 +3337,91 @@ const SCENARIOS = [
2863
3337
  cacheWriteTokens: 0
2864
3338
  }
2865
3339
  };
2866
- return;
2867
3340
  }
2868
- orchTurn += 1;
2869
- const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
2870
- agentType: "worker",
2871
- prompt: "research A"
2872
- }, `id-${String(call)}-0`).concat(toolEvents("spawn_agent", {
2873
- agentType: "worker",
2874
- prompt: "research B"
2875
- }, `id-${String(call)}-1`), toolEvents("spawn_agent", {
2876
- agentType: "worker",
2877
- prompt: "research C"
2878
- }, `id-${String(call)}-2`)) : toolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
2879
- for (const event of turn) yield event;
2880
- yield {
2881
- type: "finish",
2882
- finish: { reason: "tool-calls" },
2883
- usage: {
2884
- inputTokens: 10,
2885
- outputTokens: orchTurn === 1 ? 2e3 : 5,
2886
- cacheReadTokens: 0,
2887
- cacheWriteTokens: 0
2888
- }
2889
- };
2890
- }
3341
+ };
2891
3342
  };
2892
- const store = new InMemoryStore();
2893
- const engine = createEngine({
3343
+ const LIFETIME_SPAWNS = 5;
3344
+ const wf = makeOrchestratorWorkflow("the resumable fan-out", {
3345
+ synthesis: {
3346
+ estCost: .006,
3347
+ limits: { maxOutputTokensPerTurn: 500 }
3348
+ },
3349
+ budget: { finalizeReserveUsd: 0 }
3350
+ });
3351
+ const makeEngine = (adapter, store) => createEngine({
2894
3352
  adapters: [adapter],
2895
3353
  stores: { journal: store },
3354
+ budgetDefaults: { lifetimeSpawnCap: LIFETIME_SPAWNS },
2896
3355
  defaults: {
2897
3356
  routing: {
2898
3357
  loop: "fake:model",
2899
- orchestrate: "fake:model"
3358
+ orchestrate: "fake:model",
3359
+ synthesize: "fake:model"
2900
3360
  },
2901
- profiles: { worker: {
2902
- description: "the oversized worker",
2903
- limits: { maxOutputTokensPerTurn: 1e4 }
2904
- } }
3361
+ profiles: { worker: { description: "the resumable gatherer" } }
2905
3362
  }
2906
3363
  });
2907
- const wf = makeOrchestratorWorkflow("the parity shape", { limits: { maxOutputTokensPerTurn: 2500 } });
2908
- const waits = [];
2909
- const handle = engine.run(wf, void 0, {
2910
- runId: "fault-parity-quiescence",
2911
- budgetUsd: 10,
2912
- maxInFlightExposureUsd: .04
2913
- });
2914
- handle.on("budget:exposure-wait", (event) => waits.push(event));
2915
- const outcome = await handle.result;
2916
- const entries = await store.load("fault-parity-quiescence");
2917
- const drained = entries.filter((entry) => entry.kind === "agent" && entry.status === "error" && (entry.error?.data)?.reason === "exposure-drained");
2918
- const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
2919
- const settle = [...entries].reverse().find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
2920
- const agentEntries = entries.filter((entry) => entry.kind === "agent");
2921
- const rosterClosed = agentEntries.filter((entry) => entry.status === "running").map((entry) => entry.seq).every((seq) => agentEntries.some((entry) => entry.ref === seq && entry.status !== "running"));
2922
- const invoice = invoiceFromJournal(entries, (servedBy, usage) => usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10);
2923
- const envelope = outcome.value;
3364
+ const storeA = new InMemoryStore();
3365
+ const seeded = await makeEngine(makeAdapter(), storeA).run(wf, void 0, {
3366
+ runId: "fault-resume-famine",
3367
+ budgetUsd: 10
3368
+ }).result;
3369
+ if (seeded.status !== "ok") throw new Error(`fault kit: the seeding run settled '${seeded.status}' instead of ok`);
3370
+ const entriesA = await storeA.load("fault-resume-famine");
3371
+ const firstWorkerOk = entriesA.findIndex((entry) => entry.kind === "agent" && entry.status === "ok");
3372
+ if (firstWorkerOk < 0) throw new Error("fault kit: the seeding journal carries no worker terminal to cut at");
3373
+ const cut = entriesA.slice(0, firstWorkerOk + 1);
3374
+ const storeB = new InMemoryStore();
3375
+ for (const entry of cut) await storeB.append("fault-resume-famine", entry);
3376
+ const resumeAdapter = makeAdapter();
3377
+ const resumed = await makeEngine(resumeAdapter, storeB).resume("fault-resume-famine", wf).result;
3378
+ const entriesB = await storeB.load("fault-resume-famine");
3379
+ const capDeclines = entriesB.filter((entry) => entry.kind === "decision" && JSON.stringify(entry.value ?? {}).includes("lifetime spawn cap"));
3380
+ const redemptionDeclines = entriesB.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_synthesis_redemption_declined");
3381
+ const liveWorkerCalls = resumeAdapter.calls.filter((req) => scriptedAgentType(req) !== "").length;
3382
+ const distinctAgentScopes = new Set(entriesB.filter((entry) => entry.kind === "agent" && entry.status === "running").map((entry) => `${entry.scope}#${String(entry.key)}`)).size;
2924
3383
  return {
2925
3384
  observation: {
2926
- matched: outcome.status === "exhausted" && envelope?.forcedFinishFallback === true && envelope.completion === "partial" && drained.length === 3 && workerCalls.length === 0 && waits.some((event) => event.scope === "child" && event.willWait === false) && settle !== void 0 && rosterClosed && agentEntries.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests && invoice.unsettled === void 0,
2927
- detail: `run '${outcome.status}' (forcedFinishFallback=${String(envelope?.forcedFinishFallback)}, completion=${String(envelope?.completion)}); ${String(drained.length)} drained child terminal(s) at ${String(workerCalls.length)} worker call(s); roster closed=${String(rosterClosed)} before settle seq ${String(settle?.seq)}; wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}, unsettled lane absent=${String(invoice.unsettled === void 0)}`
3385
+ matched: resumed.status === "ok" && resumed.value === "the resumed dossier" && capDeclines.length === 0 && redemptionDeclines.length === 0 && liveWorkerCalls === 2 && distinctAgentScopes === LIFETIME_SPAWNS,
3386
+ detail: `resumed '${resumed.status}' result=${JSON.stringify(resumed.value)} at lifetimeSpawnCap ${String(LIFETIME_SPAWNS)} (${String(distinctAgentScopes)} distinct lifetime agent scopes); cap declines journaled=${String(capDeclines.length)}, redemption declines=${String(redemptionDeclines.length)}, live worker calls in the resumed segment=${String(liveWorkerCalls)} (the settled worker replayed free)`
2928
3387
  },
2929
3388
  artifacts: [
2930
- jsonArtifact("outcome.json", {
2931
- status: outcome.status,
2932
- value: outcome.value ?? null,
2933
- envelope: outcome.envelope
3389
+ jsonArtifact("journal-cut.json", cut),
3390
+ jsonArtifact("resume-outcome.json", {
3391
+ status: resumed.status,
3392
+ value: resumed.value ?? null,
3393
+ liveWorkerCalls
2934
3394
  }),
2935
- jsonArtifact("events.json", { waits }),
2936
- jsonArtifact("journal.json", entries)
3395
+ jsonArtifact("journal-resumed.json", entriesB)
2937
3396
  ]
2938
3397
  };
2939
3398
  }
2940
3399
  },
2941
3400
  {
2942
- name: "parity-sequential-roster-floor",
2943
- doctrine: "a seat-by-seat roster under an unreachable acceptance floor refuses its FIRST seat typed 'roster_floor' with the whole-roster arithmetic journaled and zero paid children (RV2005): the parity arm paid three seats the settle verdict was bound to reject",
3401
+ name: "validator-guidance-conflict",
3402
+ doctrine: "the evidence-grade reason names the SAFE composition against its cited-value sibling (RV2202): the c3 trap finish repairs in ONE round by moving the run id into its own sentence, with cited-value never rejecting; the third subscription run burned both repairs between the two verdicts",
2944
3403
  async run() {
2945
- const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
2946
- let orchTurn = 0;
3404
+ const TRAP_FINISH = "The reserve fold is live-observed under sustained load. The engine seals the journal at settle (README.md:3).";
3405
+ const FIXED_FINISH = "The reserve fold is live-observed under sustained load in run 01ARZ3NDEKTSV4RRFFQ69G5FAV. The engine seals the journal at settle (README.md:3).";
2947
3406
  const calls = [];
2948
- const toolEvents = (name, args, id) => [
2949
- {
2950
- type: "tool-call-start",
2951
- id,
2952
- name
2953
- },
2954
- {
2955
- type: "tool-call-delta",
2956
- id,
2957
- argsTextDelta: JSON.stringify(args)
2958
- },
2959
- {
2960
- type: "tool-call-end",
2961
- id,
2962
- args
2963
- }
2964
- ];
3407
+ let finishAttempts = 0;
2965
3408
  const adapter = {
2966
3409
  id: "fake",
2967
3410
  calls,
2968
- caps: () => ({
2969
- contextWindow: 2e5,
2970
- maxOutputTokens: 16e3,
2971
- structuredOutput: "native",
2972
- supportsTemperature: true,
2973
- supportsParallelTools: true,
2974
- reasoningEfforts: [],
2975
- pricing: {
2976
- inputUsdPerMTok: 1,
2977
- outputUsdPerMTok: 10
2978
- }
2979
- }),
3411
+ caps: scriptedCaps,
2980
3412
  async *stream(req) {
2981
3413
  const call = calls.length;
2982
3414
  calls.push(req);
2983
3415
  await Promise.resolve();
2984
- if (agentTypeOfReq(req) !== "") {
2985
- yield {
2986
- type: "text-delta",
2987
- text: "seated worker"
2988
- };
2989
- yield {
2990
- type: "finish",
2991
- finish: { reason: "stop" },
2992
- usage: {
2993
- inputTokens: 10,
2994
- outputTokens: 5,
2995
- cacheReadTokens: 0,
2996
- cacheWriteTokens: 0
2997
- }
2998
- };
2999
- return;
3000
- }
3001
- orchTurn += 1;
3002
- const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
3003
- agentType: "worker",
3004
- prompt: "seat 1"
3005
- }, `id-${String(call)}-0`) : toolEvents("finish", { result: "stopped early" }, `id-${String(call)}-0`);
3006
- for (const event of turn) yield event;
3416
+ finishAttempts += 1;
3417
+ const result = finishAttempts === 1 ? TRAP_FINISH : FIXED_FINISH;
3418
+ for (const event of scriptedToolEvents("finish", { result }, `id-${String(call)}-0`)) yield event;
3007
3419
  yield {
3008
3420
  type: "finish",
3009
3421
  finish: { reason: "tool-calls" },
3010
3422
  usage: {
3011
3423
  inputTokens: 10,
3012
- outputTokens: 5,
3424
+ outputTokens: 40,
3013
3425
  cacheReadTokens: 0,
3014
3426
  cacheWriteTokens: 0
3015
3427
  }
@@ -3020,43 +3432,45 @@ const SCENARIOS = [
3020
3432
  const engine = createEngine({
3021
3433
  adapters: [adapter],
3022
3434
  stores: { journal: store },
3023
- defaults: {
3024
- routing: {
3025
- loop: "fake:model",
3026
- orchestrate: "fake:model"
3027
- },
3028
- profiles: { worker: {
3029
- description: "the estimated worker",
3030
- estCost: .7
3031
- } }
3032
- }
3435
+ defaults: { routing: {
3436
+ loop: "fake:model",
3437
+ orchestrate: "fake:model"
3438
+ } }
3033
3439
  });
3034
- const wf = makeOrchestratorWorkflow("the seat-by-seat parity roster", { acceptance: {
3035
- childPolicy: "all-ok",
3036
- minSpawnedChildren: 4
3440
+ const resolveSource = (target) => target.path === "README.md" ? "the rulvar engine\nholds one denominator\nthe engine seals the journal at settle\n" : void 0;
3441
+ const wf = makeOrchestratorWorkflow("the guidance trap", { finishValidation: {
3442
+ validators: [evidenceGradeValidator(), citedValueValidator({
3443
+ resolve: resolveSource,
3444
+ window: 2
3445
+ })],
3446
+ maxRepairs: 2,
3447
+ repairTurnReserve: 2
3037
3448
  } });
3038
- const rejects = [];
3039
- const handle = engine.run(wf, void 0, {
3040
- runId: "fault-parity-roster",
3041
- budgetUsd: 2.5
3042
- });
3043
- handle.on("spawn:rejected", (event) => rejects.push(event));
3044
- const outcome = await handle.result;
3045
- const entries = await store.load("fault-parity-roster");
3046
- const decision = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "spawn-admission" && entry.value.decision?.verdict?.reason?.code === "roster_floor");
3047
- const reason = (decision?.value)?.decision.verdict.reason;
3048
- const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
3449
+ const outcome = await engine.run(wf, void 0, {
3450
+ runId: "fault-guidance-conflict",
3451
+ budgetUsd: 10
3452
+ }).result;
3453
+ const entries = await store.load("fault-guidance-conflict");
3454
+ const repairRequest = calls[1];
3455
+ const repairBytes = JSON.stringify(repairRequest?.messages ?? []);
3456
+ const guidanceQuoted = repairBytes.includes("SEPARATE sentence") && repairBytes.includes("run id");
3457
+ const citedValueNamed = repairBytes.includes("cited-value one");
3458
+ const decisionsText = JSON.stringify(entries.filter((entry) => entry.kind === "decision").map((entry) => entry.value ?? null));
3459
+ const citedValueRejected = decisionsText.includes("\"cited-value\"") ? decisionsText.includes("not present inside the cited window") : false;
3049
3460
  return {
3050
3461
  observation: {
3051
- matched: workerCalls.length === 0 && rejects.some((event) => event.code === "roster_floor") && decision !== void 0 && reason?.floor === 4 && reason.seatsRemaining === 4 && Math.abs((reason.perSeatProjectionUsd ?? 0) - .7) < 1e-9 && outcome.status === "error",
3052
- detail: `run '${outcome.status}' with ${String(workerCalls.length)} worker call(s); roster_floor rejected=${String(rejects.some((e) => e.code === "roster_floor"))}, journaled arithmetic floor=${String(reason?.floor)} seatsRemaining=${String(reason?.seatsRemaining)} perSeat=${String(reason?.perSeatProjectionUsd)}`
3462
+ matched: outcome.status === "ok" && outcome.value === FIXED_FINISH && finishAttempts === 2 && guidanceQuoted && citedValueNamed && !citedValueRejected,
3463
+ detail: `run '${outcome.status}' after ${String(finishAttempts)} finish attempt(s); the repair exchange quoted the safe composition (SEPARATE sentence + run id: ${String(guidanceQuoted)}) and named the cited-value trade (${String(citedValueNamed)}); cited-value rejected=${String(citedValueRejected)}; final result carries the separate-sentence run id: ${String(outcome.value === FIXED_FINISH)}`
3053
3464
  },
3054
3465
  artifacts: [
3055
3466
  jsonArtifact("outcome.json", {
3056
3467
  status: outcome.status,
3057
- error: outcome.error ?? null
3468
+ value: outcome.value ?? null
3058
3469
  }),
3059
- jsonArtifact("events.json", { rejects }),
3470
+ {
3471
+ name: "repair-request.json",
3472
+ content: repairBytes
3473
+ },
3060
3474
  jsonArtifact("journal.json", entries)
3061
3475
  ]
3062
3476
  };