@rulvar/evals 1.225.0 → 1.227.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +836 -422
- package/package.json +6 -6
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
|
-
import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
|
|
2
|
+
import { ConfigError, InMemoryStore, JsonlFileStore, KnowledgeCasError, LeaseHeldError, SettlementError, SupersededError, citedValueValidator, claimCoverageOf, claimExpiry, compareRates, compileVerifiedLayer, createEngine, defineWorkflow, evidenceGradeValidator, hashRunOutput, invoiceFromJournal, journalPricingSnapshot, lastRunSettle, makeOrchestratorWorkflow, memoryQuotaLimiter, pairDraftClaims, pairRunFactClaims, preflightEstimate, priceComponentsOf } from "@rulvar/core";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { tmpdir } from "node:os";
|
|
@@ -2531,6 +2531,582 @@ const TIER_CROSSING_TURN = {
|
|
|
2531
2531
|
cacheWriteTokens: 0
|
|
2532
2532
|
}]
|
|
2533
2533
|
};
|
|
2534
|
+
const tierCrossingLiveParity = {
|
|
2535
|
+
name: "tier-crossing-live-parity",
|
|
2536
|
+
doctrine: "a long-context tier crossed by the sum of one call that no single mid-stream slice reached debits the live budget exactly like the settled fold (RV1101): the per-call marginal meter re-prices the whole call at the crossing slice, live and settled read the same dollars, and a ceiling between the per-slice and tiered readings severs the run instead of settling ok over its own hard cap",
|
|
2537
|
+
async run() {
|
|
2538
|
+
const runTier = async (runId, budgetUsd) => {
|
|
2539
|
+
const handle = createEngine({
|
|
2540
|
+
adapters: [wireAdapter("tier", [TIER_CROSSING_TURN])],
|
|
2541
|
+
stores: { journal: new InMemoryStore() },
|
|
2542
|
+
defaults: { routing: { loop: "tier:model" } },
|
|
2543
|
+
pricing: TIER_PRICING
|
|
2544
|
+
}).run(echoWorkflow, void 0, {
|
|
2545
|
+
runId,
|
|
2546
|
+
budgetUsd
|
|
2547
|
+
});
|
|
2548
|
+
let maxLiveSpentUsd = 0;
|
|
2549
|
+
const ladder = [];
|
|
2550
|
+
handle.on("budget:update", (event) => {
|
|
2551
|
+
if (event.spentUsd > maxLiveSpentUsd) maxLiveSpentUsd = event.spentUsd;
|
|
2552
|
+
if (event.spentUsd > 0 && ladder[ladder.length - 1] !== event.spentUsd) ladder.push(event.spentUsd);
|
|
2553
|
+
});
|
|
2554
|
+
return {
|
|
2555
|
+
outcome: await handle.result,
|
|
2556
|
+
maxLiveSpentUsd,
|
|
2557
|
+
ladder
|
|
2558
|
+
};
|
|
2559
|
+
};
|
|
2560
|
+
const parity = await runTier("fault-tier-parity", 100);
|
|
2561
|
+
const capped = await runTier("fault-tier-capped", 4);
|
|
2562
|
+
const expectedLadder = [
|
|
2563
|
+
1.5,
|
|
2564
|
+
5,
|
|
2565
|
+
5.75
|
|
2566
|
+
];
|
|
2567
|
+
let cursor = 0;
|
|
2568
|
+
for (const reading of parity.ladder) if (reading === expectedLadder[cursor]) cursor += 1;
|
|
2569
|
+
const ladderDriven = cursor === expectedLadder.length;
|
|
2570
|
+
return {
|
|
2571
|
+
observation: {
|
|
2572
|
+
matched: parity.outcome.status === "ok" && parity.outcome.cost.totalUsd === 5.75 && parity.maxLiveSpentUsd === 5.75 && ladderDriven && capped.outcome.status !== "ok" && capped.outcome.cost.totalUsd === 5.75 && capped.maxLiveSpentUsd === capped.outcome.cost.totalUsd,
|
|
2573
|
+
detail: `a 250k call arriving as 150k + 100k slices (no slice crossed the 200k tier) debited live=${String(parity.maxLiveSpentUsd)} USD and settled=${String(parity.outcome.cost.totalUsd)} USD on the same run over the live ladder ${parity.ladder.join(" -> ")}; under the $4 ceiling between the per-slice ($3.00) and tiered readings the run settled '${capped.outcome.status}' at ${String(capped.outcome.cost.totalUsd)} USD with live=${String(capped.maxLiveSpentUsd)}`
|
|
2574
|
+
},
|
|
2575
|
+
artifacts: [jsonArtifact("parity-run.json", {
|
|
2576
|
+
status: parity.outcome.status,
|
|
2577
|
+
settledUsd: parity.outcome.cost.totalUsd,
|
|
2578
|
+
liveUsd: parity.maxLiveSpentUsd,
|
|
2579
|
+
ladder: parity.ladder,
|
|
2580
|
+
usage: parity.outcome.usage
|
|
2581
|
+
}), jsonArtifact("capped-run.json", {
|
|
2582
|
+
status: capped.outcome.status,
|
|
2583
|
+
settledUsd: capped.outcome.cost.totalUsd,
|
|
2584
|
+
liveUsd: capped.maxLiveSpentUsd,
|
|
2585
|
+
ladder: capped.ladder,
|
|
2586
|
+
error: capped.outcome.error?.message ?? null
|
|
2587
|
+
})]
|
|
2588
|
+
};
|
|
2589
|
+
}
|
|
2590
|
+
};
|
|
2591
|
+
/**
|
|
2592
|
+
* RV1905, the four-role benchmark's primary arm as a permanent gate:
|
|
2593
|
+
* the exact $6.00 / $4.50 cap / $1.00 synthesis / four 0.62 workers
|
|
2594
|
+
* configuration whose preflight read 5/5 green while the live gate
|
|
2595
|
+
* refused the third worker. Since RV1901 the projection holds the
|
|
2596
|
+
* synthesis reserve like both live gates, seats 2 of 4, exposes the
|
|
2597
|
+
* equation terms, and names the roster shortfall as an error finding.
|
|
2598
|
+
*/
|
|
2599
|
+
const benchmarkPrimaryPreflightParity = {
|
|
2600
|
+
name: "benchmark-primary-preflight-parity",
|
|
2601
|
+
doctrine: "the admission projection holds the synthesis reserve exactly like the live gates (RV1901): the benchmark primary configuration projects 2 of 4 seats with the equation terms exposed and the roster shortfall named admission-below-roster-floor, never the 5/5 green wave the live gate is bound to refuse",
|
|
2602
|
+
run() {
|
|
2603
|
+
const report = preflightEstimate({
|
|
2604
|
+
engine: {
|
|
2605
|
+
adapters: [new FakeAdapter({ agents: { "*": "unused" } })],
|
|
2606
|
+
defaults: { routing: {
|
|
2607
|
+
loop: FAKE_MODEL_REF,
|
|
2608
|
+
orchestrate: FAKE_MODEL_REF,
|
|
2609
|
+
synthesize: FAKE_MODEL_REF
|
|
2610
|
+
} }
|
|
2611
|
+
},
|
|
2612
|
+
run: { budgetUsd: 6 },
|
|
2613
|
+
orchestrator: {
|
|
2614
|
+
budget: {
|
|
2615
|
+
capUsd: 4.5,
|
|
2616
|
+
capFraction: 1,
|
|
2617
|
+
synthesisReserveUsd: 1
|
|
2618
|
+
},
|
|
2619
|
+
synthesis: { limits: { maxTurns: 2 } },
|
|
2620
|
+
acceptance: { minSpawnedChildren: 4 }
|
|
2621
|
+
},
|
|
2622
|
+
spawns: [
|
|
2623
|
+
"product",
|
|
2624
|
+
"finops",
|
|
2625
|
+
"durability",
|
|
2626
|
+
"adversarial"
|
|
2627
|
+
].map((label) => ({
|
|
2628
|
+
label,
|
|
2629
|
+
estCost: .62
|
|
2630
|
+
}))
|
|
2631
|
+
});
|
|
2632
|
+
const denied = report.admission.wave.filter((row) => !row.admitted);
|
|
2633
|
+
const floorFinding = report.findings.find((finding) => finding.code === "admission-below-roster-floor");
|
|
2634
|
+
const matched = report.admission.admitted === 3 && report.admission.denied === 2 && denied.every((row) => row.deniedBy === "budget") && report.admission.synthesisReserveUsd === 1 && Math.abs((report.admission.wave[0]?.reserveUsd ?? 0) - 3.5) < 1e-9 && Math.abs((report.admission.wave[0]?.heldAtEvaluationUsd ?? 0) - 1) < 1e-9 && floorFinding?.severity === "error";
|
|
2635
|
+
return Promise.resolve({
|
|
2636
|
+
observation: {
|
|
2637
|
+
matched,
|
|
2638
|
+
detail: `the wave seats ${String(report.admission.admitted)} of 5 rows (${String(report.admission.denied)} denied by budget), synthesis hold ${String(report.admission.synthesisReserveUsd)} USD, roster finding '${String(floorFinding?.code)}' at severity '${String(floorFinding?.severity)}'`
|
|
2639
|
+
},
|
|
2640
|
+
artifacts: [jsonArtifact("preflight.json", report)]
|
|
2641
|
+
});
|
|
2642
|
+
}
|
|
2643
|
+
};
|
|
2644
|
+
/**
|
|
2645
|
+
* RV1905, the four-role benchmark's recovery arm as a permanent gate:
|
|
2646
|
+
* four admitted children still finalizing, the root's next turn refused
|
|
2647
|
+
* pre-wire by the exposure cap. Since RV1902 the root parks and
|
|
2648
|
+
* retries; since RV1903 every child reaches a journaled terminal before
|
|
2649
|
+
* run_settle; since RV1904 the terminal and the invoice share one wire
|
|
2650
|
+
* denominator. One scenario drives the whole fixed pipeline.
|
|
2651
|
+
*/
|
|
2652
|
+
const benchmarkRecoveryRootExposure = {
|
|
2653
|
+
name: "benchmark-recovery-root-exposure",
|
|
2654
|
+
doctrine: "a root turn refused by the in-flight exposure cap beside live children parks and completes after a hold releases (RV1902), every child terminal precedes run_settle (RV1903), and the terminal envelope and the invoice cardinality agree on the wire count (RV1904): the recovery arm terminal-failed on every one of these",
|
|
2655
|
+
async run() {
|
|
2656
|
+
let releaseChildren = () => {};
|
|
2657
|
+
const childrenGate = new Promise((resolve) => {
|
|
2658
|
+
releaseChildren = resolve;
|
|
2659
|
+
});
|
|
2660
|
+
const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
|
|
2661
|
+
let orchTurn = 0;
|
|
2662
|
+
const calls = [];
|
|
2663
|
+
const toolEvents = (name, args, call) => [
|
|
2664
|
+
{
|
|
2665
|
+
type: "tool-call-start",
|
|
2666
|
+
id: `id-${String(call)}-0`,
|
|
2667
|
+
name
|
|
2668
|
+
},
|
|
2669
|
+
{
|
|
2670
|
+
type: "tool-call-delta",
|
|
2671
|
+
id: `id-${String(call)}-0`,
|
|
2672
|
+
argsTextDelta: JSON.stringify(args)
|
|
2673
|
+
},
|
|
2674
|
+
{
|
|
2675
|
+
type: "tool-call-end",
|
|
2676
|
+
id: `id-${String(call)}-0`,
|
|
2677
|
+
args
|
|
2678
|
+
}
|
|
2679
|
+
];
|
|
2680
|
+
const handlesIn = (req) => {
|
|
2681
|
+
const found = [];
|
|
2682
|
+
for (const msg of req.messages) for (const part of msg.parts) if (part.type === "tool-result") {
|
|
2683
|
+
const result = part.result;
|
|
2684
|
+
if (typeof result.handle === "number") found.push(result.handle);
|
|
2685
|
+
}
|
|
2686
|
+
return found;
|
|
2687
|
+
};
|
|
2688
|
+
const adapter = {
|
|
2689
|
+
id: "fake",
|
|
2690
|
+
calls,
|
|
2691
|
+
caps: () => ({
|
|
2692
|
+
contextWindow: 2e5,
|
|
2693
|
+
maxOutputTokens: 4096,
|
|
2694
|
+
structuredOutput: "native",
|
|
2695
|
+
supportsTemperature: true,
|
|
2696
|
+
supportsParallelTools: true,
|
|
2697
|
+
reasoningEfforts: [],
|
|
2698
|
+
pricing: {
|
|
2699
|
+
inputUsdPerMTok: 1,
|
|
2700
|
+
outputUsdPerMTok: 10
|
|
2701
|
+
}
|
|
2702
|
+
}),
|
|
2703
|
+
async *stream(req, signal) {
|
|
2704
|
+
const call = calls.length;
|
|
2705
|
+
calls.push(req);
|
|
2706
|
+
if (agentTypeOfReq(req) !== "") {
|
|
2707
|
+
await Promise.race([childrenGate, new Promise((resolve) => {
|
|
2708
|
+
signal?.addEventListener("abort", () => resolve(), { once: true });
|
|
2709
|
+
})]);
|
|
2710
|
+
if (signal?.aborted === true) return;
|
|
2711
|
+
yield {
|
|
2712
|
+
type: "text-delta",
|
|
2713
|
+
text: "worked"
|
|
2714
|
+
};
|
|
2715
|
+
yield {
|
|
2716
|
+
type: "finish",
|
|
2717
|
+
finish: { reason: "stop" },
|
|
2718
|
+
usage: {
|
|
2719
|
+
inputTokens: 10,
|
|
2720
|
+
outputTokens: 5,
|
|
2721
|
+
cacheReadTokens: 0,
|
|
2722
|
+
cacheWriteTokens: 0
|
|
2723
|
+
}
|
|
2724
|
+
};
|
|
2725
|
+
return;
|
|
2726
|
+
}
|
|
2727
|
+
orchTurn += 1;
|
|
2728
|
+
const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
|
|
2729
|
+
agentType: "worker",
|
|
2730
|
+
prompt: "task A"
|
|
2731
|
+
}, call).concat(toolEvents("spawn_agent", {
|
|
2732
|
+
agentType: "worker",
|
|
2733
|
+
prompt: "task B"
|
|
2734
|
+
}, call + 1e3)) : orchTurn === 2 ? toolEvents("await_all", { handles: handlesIn(req) }, call) : toolEvents("finish", { result: "joined after the wait" }, call);
|
|
2735
|
+
for (const event of turn) yield event;
|
|
2736
|
+
yield {
|
|
2737
|
+
type: "finish",
|
|
2738
|
+
finish: { reason: "tool-calls" },
|
|
2739
|
+
usage: {
|
|
2740
|
+
inputTokens: 10,
|
|
2741
|
+
outputTokens: 5,
|
|
2742
|
+
cacheReadTokens: 0,
|
|
2743
|
+
cacheWriteTokens: 0
|
|
2744
|
+
}
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
};
|
|
2748
|
+
const store = new InMemoryStore();
|
|
2749
|
+
const engine = createEngine({
|
|
2750
|
+
adapters: [adapter],
|
|
2751
|
+
stores: { journal: store },
|
|
2752
|
+
defaults: {
|
|
2753
|
+
routing: {
|
|
2754
|
+
loop: "fake:model",
|
|
2755
|
+
orchestrate: "fake:model"
|
|
2756
|
+
},
|
|
2757
|
+
profiles: { worker: {
|
|
2758
|
+
description: "the gated worker",
|
|
2759
|
+
limits: { maxOutputTokensPerTurn: 2500 }
|
|
2760
|
+
} }
|
|
2761
|
+
}
|
|
2762
|
+
});
|
|
2763
|
+
const wf = makeOrchestratorWorkflow("join the gated wave", { limits: { maxOutputTokensPerTurn: 4e3 } });
|
|
2764
|
+
const handle = engine.run(wf, void 0, {
|
|
2765
|
+
runId: "fault-recovery-exposure",
|
|
2766
|
+
budgetUsd: 10,
|
|
2767
|
+
maxInFlightExposureUsd: .08
|
|
2768
|
+
});
|
|
2769
|
+
const waits = [];
|
|
2770
|
+
const errors = [];
|
|
2771
|
+
handle.on("budget:exposure-wait", (event) => {
|
|
2772
|
+
waits.push(event);
|
|
2773
|
+
releaseChildren();
|
|
2774
|
+
});
|
|
2775
|
+
handle.on("agent:error", (event) => errors.push(event));
|
|
2776
|
+
setTimeout(() => releaseChildren(), 4e3).unref?.();
|
|
2777
|
+
const outcome = await handle.result;
|
|
2778
|
+
const entries = await store.load("fault-recovery-exposure");
|
|
2779
|
+
const settle = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
|
|
2780
|
+
const childTerminals = entries.filter((entry) => entry.kind === "agent" && entry.scope.startsWith("agent:") && entry.status !== "running");
|
|
2781
|
+
const invoice = invoiceFromJournal(entries, (servedBy, usage) => {
|
|
2782
|
+
return usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10;
|
|
2783
|
+
});
|
|
2784
|
+
return {
|
|
2785
|
+
observation: {
|
|
2786
|
+
matched: outcome.status === "ok" && outcome.value === "joined after the wait" && waits.length >= 1 && waits[0]?.willWait === true && errors.length === 0 && settle !== void 0 && childTerminals.length === 2 && childTerminals.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests,
|
|
2787
|
+
detail: `run '${outcome.status}' with ${String(waits.length)} exposure wait(s) (first willWait=${String(waits[0]?.willWait)}), ${String(errors.length)} agent:error, ${String(childTerminals.length)} child terminals before settle seq ${String(settle?.seq)}, wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}`
|
|
2788
|
+
},
|
|
2789
|
+
artifacts: [
|
|
2790
|
+
jsonArtifact("outcome.json", {
|
|
2791
|
+
status: outcome.status,
|
|
2792
|
+
value: outcome.value ?? null,
|
|
2793
|
+
envelope: outcome.envelope
|
|
2794
|
+
}),
|
|
2795
|
+
jsonArtifact("events.json", {
|
|
2796
|
+
waits,
|
|
2797
|
+
errors
|
|
2798
|
+
}),
|
|
2799
|
+
jsonArtifact("journal.json", entries)
|
|
2800
|
+
]
|
|
2801
|
+
};
|
|
2802
|
+
}
|
|
2803
|
+
};
|
|
2804
|
+
/**
|
|
2805
|
+
* The third parity rerun's terminal shape as a permanent gate (RV2009):
|
|
2806
|
+
* the coordination turn eats most of the exposure cap, every spawned
|
|
2807
|
+
* worker is refused DRAINED (typed 'exposure-drained', zero provider
|
|
2808
|
+
* attempts, the RV2002 seat instead of the parity mid-research death),
|
|
2809
|
+
* the next coordination turn drains too, and the run forced-finishes
|
|
2810
|
+
* partial (RV1902) into an exhausted terminal with a sealed journal:
|
|
2811
|
+
* run_settle recorded after every agent terminal, the settled fold and
|
|
2812
|
+
* the invoice in one denominator (the RV2003 no-silent-exit invariant,
|
|
2813
|
+
* where the parity process left a forever-running root and no settle).
|
|
2814
|
+
*/
|
|
2815
|
+
const parityQuiescenceDeadlock = {
|
|
2816
|
+
name: "parity-quiescence-deadlock",
|
|
2817
|
+
doctrine: "the parity deadlock shape ends in an exhausted terminal, never a silent exit: drained children die typed 'exposure-drained' at zero provider attempts (RV2001/RV2002), the root forced-finishes partial (RV1902), and run_settle seals a one-denominator journal (RV2003); the parity rerun exited mid-run with none of these",
|
|
2818
|
+
async run() {
|
|
2819
|
+
const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
|
|
2820
|
+
let orchTurn = 0;
|
|
2821
|
+
const calls = [];
|
|
2822
|
+
const toolEvents = (name, args, id) => [
|
|
2823
|
+
{
|
|
2824
|
+
type: "tool-call-start",
|
|
2825
|
+
id,
|
|
2826
|
+
name
|
|
2827
|
+
},
|
|
2828
|
+
{
|
|
2829
|
+
type: "tool-call-delta",
|
|
2830
|
+
id,
|
|
2831
|
+
argsTextDelta: JSON.stringify(args)
|
|
2832
|
+
},
|
|
2833
|
+
{
|
|
2834
|
+
type: "tool-call-end",
|
|
2835
|
+
id,
|
|
2836
|
+
args
|
|
2837
|
+
}
|
|
2838
|
+
];
|
|
2839
|
+
const adapter = {
|
|
2840
|
+
id: "fake",
|
|
2841
|
+
calls,
|
|
2842
|
+
caps: () => ({
|
|
2843
|
+
contextWindow: 2e5,
|
|
2844
|
+
maxOutputTokens: 16e3,
|
|
2845
|
+
structuredOutput: "native",
|
|
2846
|
+
supportsTemperature: true,
|
|
2847
|
+
supportsParallelTools: true,
|
|
2848
|
+
reasoningEfforts: [],
|
|
2849
|
+
pricing: {
|
|
2850
|
+
inputUsdPerMTok: 1,
|
|
2851
|
+
outputUsdPerMTok: 10
|
|
2852
|
+
}
|
|
2853
|
+
}),
|
|
2854
|
+
async *stream(req) {
|
|
2855
|
+
const call = calls.length;
|
|
2856
|
+
calls.push(req);
|
|
2857
|
+
await Promise.resolve();
|
|
2858
|
+
if (agentTypeOfReq(req) !== "") {
|
|
2859
|
+
yield {
|
|
2860
|
+
type: "text-delta",
|
|
2861
|
+
text: "unreachable worker"
|
|
2862
|
+
};
|
|
2863
|
+
yield {
|
|
2864
|
+
type: "finish",
|
|
2865
|
+
finish: { reason: "stop" },
|
|
2866
|
+
usage: {
|
|
2867
|
+
inputTokens: 10,
|
|
2868
|
+
outputTokens: 5,
|
|
2869
|
+
cacheReadTokens: 0,
|
|
2870
|
+
cacheWriteTokens: 0
|
|
2871
|
+
}
|
|
2872
|
+
};
|
|
2873
|
+
return;
|
|
2874
|
+
}
|
|
2875
|
+
orchTurn += 1;
|
|
2876
|
+
const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
|
|
2877
|
+
agentType: "worker",
|
|
2878
|
+
prompt: "research A"
|
|
2879
|
+
}, `id-${String(call)}-0`).concat(toolEvents("spawn_agent", {
|
|
2880
|
+
agentType: "worker",
|
|
2881
|
+
prompt: "research B"
|
|
2882
|
+
}, `id-${String(call)}-1`), toolEvents("spawn_agent", {
|
|
2883
|
+
agentType: "worker",
|
|
2884
|
+
prompt: "research C"
|
|
2885
|
+
}, `id-${String(call)}-2`)) : toolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
|
|
2886
|
+
for (const event of turn) yield event;
|
|
2887
|
+
yield {
|
|
2888
|
+
type: "finish",
|
|
2889
|
+
finish: { reason: "tool-calls" },
|
|
2890
|
+
usage: {
|
|
2891
|
+
inputTokens: 10,
|
|
2892
|
+
outputTokens: orchTurn === 1 ? 2e3 : 5,
|
|
2893
|
+
cacheReadTokens: 0,
|
|
2894
|
+
cacheWriteTokens: 0
|
|
2895
|
+
}
|
|
2896
|
+
};
|
|
2897
|
+
}
|
|
2898
|
+
};
|
|
2899
|
+
const store = new InMemoryStore();
|
|
2900
|
+
const engine = createEngine({
|
|
2901
|
+
adapters: [adapter],
|
|
2902
|
+
stores: { journal: store },
|
|
2903
|
+
defaults: {
|
|
2904
|
+
routing: {
|
|
2905
|
+
loop: "fake:model",
|
|
2906
|
+
orchestrate: "fake:model"
|
|
2907
|
+
},
|
|
2908
|
+
profiles: { worker: {
|
|
2909
|
+
description: "the oversized worker",
|
|
2910
|
+
limits: { maxOutputTokensPerTurn: 1e4 }
|
|
2911
|
+
} }
|
|
2912
|
+
}
|
|
2913
|
+
});
|
|
2914
|
+
const wf = makeOrchestratorWorkflow("the parity shape", { limits: { maxOutputTokensPerTurn: 2500 } });
|
|
2915
|
+
const waits = [];
|
|
2916
|
+
const handle = engine.run(wf, void 0, {
|
|
2917
|
+
runId: "fault-parity-quiescence",
|
|
2918
|
+
budgetUsd: 10,
|
|
2919
|
+
maxInFlightExposureUsd: .04
|
|
2920
|
+
});
|
|
2921
|
+
handle.on("budget:exposure-wait", (event) => waits.push(event));
|
|
2922
|
+
const outcome = await handle.result;
|
|
2923
|
+
const entries = await store.load("fault-parity-quiescence");
|
|
2924
|
+
const drained = entries.filter((entry) => entry.kind === "agent" && entry.status === "error" && (entry.error?.data)?.reason === "exposure-drained");
|
|
2925
|
+
const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
|
|
2926
|
+
const settle = [...entries].reverse().find((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
|
|
2927
|
+
const agentEntries = entries.filter((entry) => entry.kind === "agent");
|
|
2928
|
+
const rosterClosed = agentEntries.filter((entry) => entry.status === "running").map((entry) => entry.seq).every((seq) => agentEntries.some((entry) => entry.ref === seq && entry.status !== "running"));
|
|
2929
|
+
const invoice = invoiceFromJournal(entries, (servedBy, usage) => usage.inputTokens / 1e6 * 1 + usage.outputTokens / 1e6 * 10);
|
|
2930
|
+
const envelope = outcome.value;
|
|
2931
|
+
return {
|
|
2932
|
+
observation: {
|
|
2933
|
+
matched: outcome.status === "exhausted" && envelope?.forcedFinishFallback === true && envelope.completion === "partial" && drained.length === 3 && workerCalls.length === 0 && waits.some((event) => event.scope === "child" && event.willWait === false) && settle !== void 0 && rosterClosed && agentEntries.every((entry) => entry.seq < settle.seq) && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests && invoice.unsettled === void 0,
|
|
2934
|
+
detail: `run '${outcome.status}' (forcedFinishFallback=${String(envelope?.forcedFinishFallback)}, completion=${String(envelope?.completion)}); ${String(drained.length)} drained child terminal(s) at ${String(workerCalls.length)} worker call(s); roster closed=${String(rosterClosed)} before settle seq ${String(settle?.seq)}; wires ${String(outcome.envelope.wireRequests)} == invoice ${String(invoice.cardinality.wireRequests)}, unsettled lane absent=${String(invoice.unsettled === void 0)}`
|
|
2935
|
+
},
|
|
2936
|
+
artifacts: [
|
|
2937
|
+
jsonArtifact("outcome.json", {
|
|
2938
|
+
status: outcome.status,
|
|
2939
|
+
value: outcome.value ?? null,
|
|
2940
|
+
envelope: outcome.envelope
|
|
2941
|
+
}),
|
|
2942
|
+
jsonArtifact("events.json", { waits }),
|
|
2943
|
+
jsonArtifact("journal.json", entries)
|
|
2944
|
+
]
|
|
2945
|
+
};
|
|
2946
|
+
}
|
|
2947
|
+
};
|
|
2948
|
+
/**
|
|
2949
|
+
* The parity roster paid seat by seat under an unreachable floor, as a
|
|
2950
|
+
* permanent gate (RV2009): every SINGLE spawn admission projects the
|
|
2951
|
+
* whole remaining roster (RV2005) and the FIRST seat refuses typed
|
|
2952
|
+
* 'roster_floor' with the arithmetic journaled, zero paid children;
|
|
2953
|
+
* the parity arm paid three of four.
|
|
2954
|
+
*/
|
|
2955
|
+
const paritySequentialRosterFloor = {
|
|
2956
|
+
name: "parity-sequential-roster-floor",
|
|
2957
|
+
doctrine: "a seat-by-seat roster under an unreachable acceptance floor refuses its FIRST seat typed 'roster_floor' with the whole-roster arithmetic journaled and zero paid children (RV2005): the parity arm paid three seats the settle verdict was bound to reject",
|
|
2958
|
+
async run() {
|
|
2959
|
+
const agentTypeOfReq = (req) => req.providerOptions?.rulvar?.agentType ?? "";
|
|
2960
|
+
let orchTurn = 0;
|
|
2961
|
+
const calls = [];
|
|
2962
|
+
const toolEvents = (name, args, id) => [
|
|
2963
|
+
{
|
|
2964
|
+
type: "tool-call-start",
|
|
2965
|
+
id,
|
|
2966
|
+
name
|
|
2967
|
+
},
|
|
2968
|
+
{
|
|
2969
|
+
type: "tool-call-delta",
|
|
2970
|
+
id,
|
|
2971
|
+
argsTextDelta: JSON.stringify(args)
|
|
2972
|
+
},
|
|
2973
|
+
{
|
|
2974
|
+
type: "tool-call-end",
|
|
2975
|
+
id,
|
|
2976
|
+
args
|
|
2977
|
+
}
|
|
2978
|
+
];
|
|
2979
|
+
const adapter = {
|
|
2980
|
+
id: "fake",
|
|
2981
|
+
calls,
|
|
2982
|
+
caps: () => ({
|
|
2983
|
+
contextWindow: 2e5,
|
|
2984
|
+
maxOutputTokens: 16e3,
|
|
2985
|
+
structuredOutput: "native",
|
|
2986
|
+
supportsTemperature: true,
|
|
2987
|
+
supportsParallelTools: true,
|
|
2988
|
+
reasoningEfforts: [],
|
|
2989
|
+
pricing: {
|
|
2990
|
+
inputUsdPerMTok: 1,
|
|
2991
|
+
outputUsdPerMTok: 10
|
|
2992
|
+
}
|
|
2993
|
+
}),
|
|
2994
|
+
async *stream(req) {
|
|
2995
|
+
const call = calls.length;
|
|
2996
|
+
calls.push(req);
|
|
2997
|
+
await Promise.resolve();
|
|
2998
|
+
if (agentTypeOfReq(req) !== "") {
|
|
2999
|
+
yield {
|
|
3000
|
+
type: "text-delta",
|
|
3001
|
+
text: "seated worker"
|
|
3002
|
+
};
|
|
3003
|
+
yield {
|
|
3004
|
+
type: "finish",
|
|
3005
|
+
finish: { reason: "stop" },
|
|
3006
|
+
usage: {
|
|
3007
|
+
inputTokens: 10,
|
|
3008
|
+
outputTokens: 5,
|
|
3009
|
+
cacheReadTokens: 0,
|
|
3010
|
+
cacheWriteTokens: 0
|
|
3011
|
+
}
|
|
3012
|
+
};
|
|
3013
|
+
return;
|
|
3014
|
+
}
|
|
3015
|
+
orchTurn += 1;
|
|
3016
|
+
const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
|
|
3017
|
+
agentType: "worker",
|
|
3018
|
+
prompt: "seat 1"
|
|
3019
|
+
}, `id-${String(call)}-0`) : toolEvents("finish", { result: "stopped early" }, `id-${String(call)}-0`);
|
|
3020
|
+
for (const event of turn) yield event;
|
|
3021
|
+
yield {
|
|
3022
|
+
type: "finish",
|
|
3023
|
+
finish: { reason: "tool-calls" },
|
|
3024
|
+
usage: {
|
|
3025
|
+
inputTokens: 10,
|
|
3026
|
+
outputTokens: 5,
|
|
3027
|
+
cacheReadTokens: 0,
|
|
3028
|
+
cacheWriteTokens: 0
|
|
3029
|
+
}
|
|
3030
|
+
};
|
|
3031
|
+
}
|
|
3032
|
+
};
|
|
3033
|
+
const store = new InMemoryStore();
|
|
3034
|
+
const engine = createEngine({
|
|
3035
|
+
adapters: [adapter],
|
|
3036
|
+
stores: { journal: store },
|
|
3037
|
+
defaults: {
|
|
3038
|
+
routing: {
|
|
3039
|
+
loop: "fake:model",
|
|
3040
|
+
orchestrate: "fake:model"
|
|
3041
|
+
},
|
|
3042
|
+
profiles: { worker: {
|
|
3043
|
+
description: "the estimated worker",
|
|
3044
|
+
estCost: .7
|
|
3045
|
+
} }
|
|
3046
|
+
}
|
|
3047
|
+
});
|
|
3048
|
+
const wf = makeOrchestratorWorkflow("the seat-by-seat parity roster", { acceptance: {
|
|
3049
|
+
childPolicy: "all-ok",
|
|
3050
|
+
minSpawnedChildren: 4
|
|
3051
|
+
} });
|
|
3052
|
+
const rejects = [];
|
|
3053
|
+
const handle = engine.run(wf, void 0, {
|
|
3054
|
+
runId: "fault-parity-roster",
|
|
3055
|
+
budgetUsd: 2.5
|
|
3056
|
+
});
|
|
3057
|
+
handle.on("spawn:rejected", (event) => rejects.push(event));
|
|
3058
|
+
const outcome = await handle.result;
|
|
3059
|
+
const entries = await store.load("fault-parity-roster");
|
|
3060
|
+
const decision = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "spawn-admission" && entry.value.decision?.verdict?.reason?.code === "roster_floor");
|
|
3061
|
+
const reason = (decision?.value)?.decision.verdict.reason;
|
|
3062
|
+
const workerCalls = calls.filter((req) => agentTypeOfReq(req) !== "");
|
|
3063
|
+
return {
|
|
3064
|
+
observation: {
|
|
3065
|
+
matched: workerCalls.length === 0 && rejects.some((event) => event.code === "roster_floor") && decision !== void 0 && reason?.floor === 4 && reason.seatsRemaining === 4 && Math.abs((reason.perSeatProjectionUsd ?? 0) - .7) < 1e-9 && outcome.status === "error",
|
|
3066
|
+
detail: `run '${outcome.status}' with ${String(workerCalls.length)} worker call(s); roster_floor rejected=${String(rejects.some((e) => e.code === "roster_floor"))}, journaled arithmetic floor=${String(reason?.floor)} seatsRemaining=${String(reason?.seatsRemaining)} perSeat=${String(reason?.perSeatProjectionUsd)}`
|
|
3067
|
+
},
|
|
3068
|
+
artifacts: [
|
|
3069
|
+
jsonArtifact("outcome.json", {
|
|
3070
|
+
status: outcome.status,
|
|
3071
|
+
error: outcome.error ?? null
|
|
3072
|
+
}),
|
|
3073
|
+
jsonArtifact("events.json", { rejects }),
|
|
3074
|
+
jsonArtifact("journal.json", entries)
|
|
3075
|
+
]
|
|
3076
|
+
};
|
|
3077
|
+
}
|
|
3078
|
+
};
|
|
3079
|
+
/** Shared tool-call event triple for scripted orchestrator streams. */
|
|
3080
|
+
const scriptedToolEvents = (name, args, id) => [
|
|
3081
|
+
{
|
|
3082
|
+
type: "tool-call-start",
|
|
3083
|
+
id,
|
|
3084
|
+
name
|
|
3085
|
+
},
|
|
3086
|
+
{
|
|
3087
|
+
type: "tool-call-delta",
|
|
3088
|
+
id,
|
|
3089
|
+
argsTextDelta: JSON.stringify(args)
|
|
3090
|
+
},
|
|
3091
|
+
{
|
|
3092
|
+
type: "tool-call-end",
|
|
3093
|
+
id,
|
|
3094
|
+
args
|
|
3095
|
+
}
|
|
3096
|
+
];
|
|
3097
|
+
const scriptedAgentType = (req) => req.providerOptions?.rulvar?.agentType ?? "";
|
|
3098
|
+
const scriptedCaps = () => ({
|
|
3099
|
+
contextWindow: 2e5,
|
|
3100
|
+
maxOutputTokens: 16e3,
|
|
3101
|
+
structuredOutput: "native",
|
|
3102
|
+
supportsTemperature: true,
|
|
3103
|
+
supportsParallelTools: true,
|
|
3104
|
+
reasoningEfforts: [],
|
|
3105
|
+
pricing: {
|
|
3106
|
+
inputUsdPerMTok: 1,
|
|
3107
|
+
outputUsdPerMTok: 10
|
|
3108
|
+
}
|
|
3109
|
+
});
|
|
2534
3110
|
const SCENARIOS = [
|
|
2535
3111
|
inFlightExposure,
|
|
2536
3112
|
duplicateQuotaRule,
|
|
@@ -2551,311 +3127,209 @@ const SCENARIOS = [
|
|
|
2551
3127
|
pauseTurnRealAdapter,
|
|
2552
3128
|
statementSettleableGuard,
|
|
2553
3129
|
supersededTerminalHonesty,
|
|
3130
|
+
tierCrossingLiveParity,
|
|
3131
|
+
benchmarkPrimaryPreflightParity,
|
|
3132
|
+
benchmarkRecoveryRootExposure,
|
|
3133
|
+
parityQuiescenceDeadlock,
|
|
3134
|
+
paritySequentialRosterFloor,
|
|
2554
3135
|
{
|
|
2555
|
-
name: "
|
|
2556
|
-
doctrine: "a
|
|
2557
|
-
async run() {
|
|
2558
|
-
const runTier = async (runId, budgetUsd) => {
|
|
2559
|
-
const handle = createEngine({
|
|
2560
|
-
adapters: [wireAdapter("tier", [TIER_CROSSING_TURN])],
|
|
2561
|
-
stores: { journal: new InMemoryStore() },
|
|
2562
|
-
defaults: { routing: { loop: "tier:model" } },
|
|
2563
|
-
pricing: TIER_PRICING
|
|
2564
|
-
}).run(echoWorkflow, void 0, {
|
|
2565
|
-
runId,
|
|
2566
|
-
budgetUsd
|
|
2567
|
-
});
|
|
2568
|
-
let maxLiveSpentUsd = 0;
|
|
2569
|
-
const ladder = [];
|
|
2570
|
-
handle.on("budget:update", (event) => {
|
|
2571
|
-
if (event.spentUsd > maxLiveSpentUsd) maxLiveSpentUsd = event.spentUsd;
|
|
2572
|
-
if (event.spentUsd > 0 && ladder[ladder.length - 1] !== event.spentUsd) ladder.push(event.spentUsd);
|
|
2573
|
-
});
|
|
2574
|
-
return {
|
|
2575
|
-
outcome: await handle.result,
|
|
2576
|
-
maxLiveSpentUsd,
|
|
2577
|
-
ladder
|
|
2578
|
-
};
|
|
2579
|
-
};
|
|
2580
|
-
const parity = await runTier("fault-tier-parity", 100);
|
|
2581
|
-
const capped = await runTier("fault-tier-capped", 4);
|
|
2582
|
-
const expectedLadder = [
|
|
2583
|
-
1.5,
|
|
2584
|
-
5,
|
|
2585
|
-
5.75
|
|
2586
|
-
];
|
|
2587
|
-
let cursor = 0;
|
|
2588
|
-
for (const reading of parity.ladder) if (reading === expectedLadder[cursor]) cursor += 1;
|
|
2589
|
-
const ladderDriven = cursor === expectedLadder.length;
|
|
2590
|
-
return {
|
|
2591
|
-
observation: {
|
|
2592
|
-
matched: parity.outcome.status === "ok" && parity.outcome.cost.totalUsd === 5.75 && parity.maxLiveSpentUsd === 5.75 && ladderDriven && capped.outcome.status !== "ok" && capped.outcome.cost.totalUsd === 5.75 && capped.maxLiveSpentUsd === capped.outcome.cost.totalUsd,
|
|
2593
|
-
detail: `a 250k call arriving as 150k + 100k slices (no slice crossed the 200k tier) debited live=${String(parity.maxLiveSpentUsd)} USD and settled=${String(parity.outcome.cost.totalUsd)} USD on the same run over the live ladder ${parity.ladder.join(" -> ")}; under the $4 ceiling between the per-slice ($3.00) and tiered readings the run settled '${capped.outcome.status}' at ${String(capped.outcome.cost.totalUsd)} USD with live=${String(capped.maxLiveSpentUsd)}`
|
|
2594
|
-
},
|
|
2595
|
-
artifacts: [jsonArtifact("parity-run.json", {
|
|
2596
|
-
status: parity.outcome.status,
|
|
2597
|
-
settledUsd: parity.outcome.cost.totalUsd,
|
|
2598
|
-
liveUsd: parity.maxLiveSpentUsd,
|
|
2599
|
-
ladder: parity.ladder,
|
|
2600
|
-
usage: parity.outcome.usage
|
|
2601
|
-
}), jsonArtifact("capped-run.json", {
|
|
2602
|
-
status: capped.outcome.status,
|
|
2603
|
-
settledUsd: capped.outcome.cost.totalUsd,
|
|
2604
|
-
liveUsd: capped.maxLiveSpentUsd,
|
|
2605
|
-
ladder: capped.ladder,
|
|
2606
|
-
error: capped.outcome.error?.message ?? null
|
|
2607
|
-
})]
|
|
2608
|
-
};
|
|
2609
|
-
}
|
|
2610
|
-
},
|
|
2611
|
-
{
|
|
2612
|
-
name: "benchmark-primary-preflight-parity",
|
|
2613
|
-
doctrine: "the admission projection holds the synthesis reserve exactly like the live gates (RV1901): the benchmark primary configuration projects 2 of 4 seats with the equation terms exposed and the roster shortfall named admission-below-roster-floor, never the 5/5 green wave the live gate is bound to refuse",
|
|
2614
|
-
run() {
|
|
2615
|
-
const report = preflightEstimate({
|
|
2616
|
-
engine: {
|
|
2617
|
-
adapters: [new FakeAdapter({ agents: { "*": "unused" } })],
|
|
2618
|
-
defaults: { routing: {
|
|
2619
|
-
loop: FAKE_MODEL_REF,
|
|
2620
|
-
orchestrate: FAKE_MODEL_REF,
|
|
2621
|
-
synthesize: FAKE_MODEL_REF
|
|
2622
|
-
} }
|
|
2623
|
-
},
|
|
2624
|
-
run: { budgetUsd: 6 },
|
|
2625
|
-
orchestrator: {
|
|
2626
|
-
budget: {
|
|
2627
|
-
capUsd: 4.5,
|
|
2628
|
-
capFraction: 1,
|
|
2629
|
-
synthesisReserveUsd: 1
|
|
2630
|
-
},
|
|
2631
|
-
synthesis: { limits: { maxTurns: 2 } },
|
|
2632
|
-
acceptance: { minSpawnedChildren: 4 }
|
|
2633
|
-
},
|
|
2634
|
-
spawns: [
|
|
2635
|
-
"product",
|
|
2636
|
-
"finops",
|
|
2637
|
-
"durability",
|
|
2638
|
-
"adversarial"
|
|
2639
|
-
].map((label) => ({
|
|
2640
|
-
label,
|
|
2641
|
-
estCost: .62
|
|
2642
|
-
}))
|
|
2643
|
-
});
|
|
2644
|
-
const denied = report.admission.wave.filter((row) => !row.admitted);
|
|
2645
|
-
const floorFinding = report.findings.find((finding) => finding.code === "admission-below-roster-floor");
|
|
2646
|
-
const matched = report.admission.admitted === 3 && report.admission.denied === 2 && denied.every((row) => row.deniedBy === "budget") && report.admission.synthesisReserveUsd === 1 && Math.abs((report.admission.wave[0]?.reserveUsd ?? 0) - 3.5) < 1e-9 && Math.abs((report.admission.wave[0]?.heldAtEvaluationUsd ?? 0) - 1) < 1e-9 && floorFinding?.severity === "error";
|
|
2647
|
-
return Promise.resolve({
|
|
2648
|
-
observation: {
|
|
2649
|
-
matched,
|
|
2650
|
-
detail: `the wave seats ${String(report.admission.admitted)} of 5 rows (${String(report.admission.denied)} denied by budget), synthesis hold ${String(report.admission.synthesisReserveUsd)} USD, roster finding '${String(floorFinding?.code)}' at severity '${String(floorFinding?.severity)}'`
|
|
2651
|
-
},
|
|
2652
|
-
artifacts: [jsonArtifact("preflight.json", report)]
|
|
2653
|
-
});
|
|
2654
|
-
}
|
|
2655
|
-
},
|
|
2656
|
-
{
|
|
2657
|
-
name: "benchmark-recovery-root-exposure",
|
|
2658
|
-
doctrine: "a root turn refused by the in-flight exposure cap beside live children parks and completes after a hold releases (RV1902), every child terminal precedes run_settle (RV1903), and the terminal envelope and the invoice cardinality agree on the wire count (RV1904): the recovery arm terminal-failed on every one of these",
|
|
3136
|
+
name: "parity-reserve-line-redemption",
|
|
3137
|
+
doctrine: "a coordination turn refused at the reserve line folds typed 'budget-floor' and the held synthesis reserve funds the redemption whose result rides the partial envelope (RV2101); the PlanRunner extension arm produces the same fold and the same redeemed result (DEF-7)",
|
|
2659
3138
|
async run() {
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
calls.push(req);
|
|
2710
|
-
if (agentTypeOfReq(req) !== "") {
|
|
2711
|
-
await Promise.race([childrenGate, new Promise((resolve) => {
|
|
2712
|
-
signal?.addEventListener("abort", () => resolve(), { once: true });
|
|
2713
|
-
})]);
|
|
2714
|
-
if (signal?.aborted === true) return;
|
|
2715
|
-
yield {
|
|
2716
|
-
type: "text-delta",
|
|
2717
|
-
text: "worked"
|
|
2718
|
-
};
|
|
3139
|
+
const runArm = async (mode) => {
|
|
3140
|
+
const calls = [];
|
|
3141
|
+
const adapter = {
|
|
3142
|
+
id: "fake",
|
|
3143
|
+
calls,
|
|
3144
|
+
caps: scriptedCaps,
|
|
3145
|
+
async *stream(req) {
|
|
3146
|
+
const call = calls.length;
|
|
3147
|
+
calls.push(req);
|
|
3148
|
+
await Promise.resolve();
|
|
3149
|
+
if (scriptedAgentType(req) !== "") {
|
|
3150
|
+
yield {
|
|
3151
|
+
type: "text-delta",
|
|
3152
|
+
text: "settled evidence"
|
|
3153
|
+
};
|
|
3154
|
+
yield {
|
|
3155
|
+
type: "finish",
|
|
3156
|
+
finish: { reason: "stop" },
|
|
3157
|
+
usage: {
|
|
3158
|
+
inputTokens: 10,
|
|
3159
|
+
outputTokens: 5,
|
|
3160
|
+
cacheReadTokens: 0,
|
|
3161
|
+
cacheWriteTokens: 0
|
|
3162
|
+
}
|
|
3163
|
+
};
|
|
3164
|
+
return;
|
|
3165
|
+
}
|
|
3166
|
+
if (!(req.tools ?? []).some((tool) => tool.name === "spawn_agent")) {
|
|
3167
|
+
for (const event of scriptedToolEvents("finish", { result: "the redeemed synthesis" }, `id-${String(call)}-0`)) yield event;
|
|
3168
|
+
yield {
|
|
3169
|
+
type: "finish",
|
|
3170
|
+
finish: { reason: "tool-calls" },
|
|
3171
|
+
usage: {
|
|
3172
|
+
inputTokens: 12,
|
|
3173
|
+
outputTokens: 8,
|
|
3174
|
+
cacheReadTokens: 0,
|
|
3175
|
+
cacheWriteTokens: 0
|
|
3176
|
+
}
|
|
3177
|
+
};
|
|
3178
|
+
return;
|
|
3179
|
+
}
|
|
3180
|
+
const turn = calls.filter((prior) => scriptedAgentType(prior) === "" && (prior.tools ?? []).some((tool) => tool.name === "spawn_agent")).length === 1 ? scriptedToolEvents("spawn_agent", {
|
|
3181
|
+
agentType: "worker",
|
|
3182
|
+
prompt: "gather A"
|
|
3183
|
+
}, `id-${String(call)}-0`).concat(scriptedToolEvents("spawn_agent", {
|
|
3184
|
+
agentType: "worker",
|
|
3185
|
+
prompt: "gather B"
|
|
3186
|
+
}, `id-${String(call)}-1`)) : scriptedToolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
|
|
3187
|
+
for (const event of turn) yield event;
|
|
2719
3188
|
yield {
|
|
2720
3189
|
type: "finish",
|
|
2721
|
-
finish: { reason: "
|
|
3190
|
+
finish: { reason: "tool-calls" },
|
|
2722
3191
|
usage: {
|
|
2723
3192
|
inputTokens: 10,
|
|
2724
|
-
outputTokens:
|
|
3193
|
+
outputTokens: 1e3,
|
|
2725
3194
|
cacheReadTokens: 0,
|
|
2726
3195
|
cacheWriteTokens: 0
|
|
2727
3196
|
}
|
|
2728
3197
|
};
|
|
2729
|
-
return;
|
|
2730
3198
|
}
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
},
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
}
|
|
2751
|
-
};
|
|
2752
|
-
const store = new InMemoryStore();
|
|
2753
|
-
const engine = createEngine({
|
|
2754
|
-
adapters: [adapter],
|
|
2755
|
-
stores: { journal: store },
|
|
2756
|
-
defaults: {
|
|
2757
|
-
routing: {
|
|
2758
|
-
loop: "fake:model",
|
|
2759
|
-
orchestrate: "fake:model"
|
|
3199
|
+
};
|
|
3200
|
+
const store = new InMemoryStore();
|
|
3201
|
+
const engine = createEngine({
|
|
3202
|
+
adapters: [adapter],
|
|
3203
|
+
stores: { journal: store },
|
|
3204
|
+
defaults: {
|
|
3205
|
+
routing: {
|
|
3206
|
+
loop: "fake:model",
|
|
3207
|
+
orchestrate: "fake:model",
|
|
3208
|
+
synthesize: "fake:model"
|
|
3209
|
+
},
|
|
3210
|
+
profiles: { worker: { description: "the cheap gatherer" } }
|
|
3211
|
+
}
|
|
3212
|
+
});
|
|
3213
|
+
const opts = {
|
|
3214
|
+
limits: { maxOutputTokensPerTurn: 1e3 },
|
|
3215
|
+
synthesis: {
|
|
3216
|
+
estCost: .006,
|
|
3217
|
+
limits: { maxOutputTokensPerTurn: 500 }
|
|
2760
3218
|
},
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
3219
|
+
budget: {
|
|
3220
|
+
capUsd: .03,
|
|
3221
|
+
synthesisReserveUsd: .02,
|
|
3222
|
+
finalizeReserveUsd: 0
|
|
3223
|
+
}
|
|
3224
|
+
};
|
|
3225
|
+
const runId = `fault-reserve-line-${mode}`;
|
|
3226
|
+
const outcome = mode === "workflow" ? await engine.run(makeOrchestratorWorkflow("the reserve line shape", opts), void 0, {
|
|
3227
|
+
runId,
|
|
3228
|
+
budgetUsd: 10
|
|
3229
|
+
}).result : await orchestratePlanned(engine, "the reserve line shape", opts, { runId }).result;
|
|
3230
|
+
const entries = await store.load(runId);
|
|
3231
|
+
const fallback = entries.find((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finalize_fallback");
|
|
3232
|
+
const settle = entries.some((entry) => entry.kind === "decision" && entry.value?.decisionType === "run_settle");
|
|
3233
|
+
const value = outcome.value;
|
|
3234
|
+
return {
|
|
3235
|
+
status: outcome.status,
|
|
3236
|
+
error: outcome.error?.message,
|
|
3237
|
+
forcedFinishFallback: value?.forcedFinishFallback,
|
|
3238
|
+
completion: value?.completion,
|
|
3239
|
+
result: value?.result,
|
|
3240
|
+
fallbackReason: (fallback?.value)?.reason,
|
|
3241
|
+
rootClassCalls: calls.filter((req) => scriptedAgentType(req) === "").length,
|
|
3242
|
+
workerOkTerminals: entries.filter((entry) => entry.kind === "agent" && entry.status === "ok").length,
|
|
3243
|
+
settleRecorded: settle,
|
|
3244
|
+
entries
|
|
3245
|
+
};
|
|
3246
|
+
};
|
|
3247
|
+
const arms = {
|
|
3248
|
+
workflow: await runArm("workflow"),
|
|
3249
|
+
extension: await runArm("extension")
|
|
3250
|
+
};
|
|
3251
|
+
const armMatched = (arm) => arm.status === "exhausted" && arm.forcedFinishFallback === true && arm.completion === "partial" && arm.result === "the redeemed synthesis" && arm.fallbackReason === "budget-floor" && arm.rootClassCalls === 2 && arm.settleRecorded;
|
|
2788
3252
|
return {
|
|
2789
3253
|
observation: {
|
|
2790
|
-
matched:
|
|
2791
|
-
detail: `
|
|
3254
|
+
matched: armMatched(arms.workflow) && armMatched(arms.extension),
|
|
3255
|
+
detail: `workflow arm '${arms.workflow.status}' reason=${String(arms.workflow.fallbackReason)} result=${JSON.stringify(arms.workflow.result)} rootCalls=${String(arms.workflow.rootClassCalls)}; extension arm '${arms.extension.status}' reason=${String(arms.extension.fallbackReason)} result=${JSON.stringify(arms.extension.result)} rootCalls=${String(arms.extension.rootClassCalls)} (DEF-7 parity: both arms fold 'budget-floor' and both redeem the synthesis from the held reserve)`
|
|
2792
3256
|
},
|
|
2793
3257
|
artifacts: [
|
|
2794
|
-
jsonArtifact("
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
3258
|
+
jsonArtifact("arms.json", {
|
|
3259
|
+
workflow: {
|
|
3260
|
+
...arms.workflow,
|
|
3261
|
+
entries: void 0
|
|
3262
|
+
},
|
|
3263
|
+
extension: {
|
|
3264
|
+
...arms.extension,
|
|
3265
|
+
entries: void 0
|
|
3266
|
+
}
|
|
2802
3267
|
}),
|
|
2803
|
-
jsonArtifact("journal.json", entries)
|
|
3268
|
+
jsonArtifact("journal-workflow.json", arms.workflow.entries),
|
|
3269
|
+
jsonArtifact("journal-extension.json", arms.extension.entries)
|
|
2804
3270
|
]
|
|
2805
3271
|
};
|
|
2806
3272
|
}
|
|
2807
3273
|
},
|
|
2808
3274
|
{
|
|
2809
|
-
name: "
|
|
2810
|
-
doctrine: "
|
|
3275
|
+
name: "resume-spawn-famine",
|
|
3276
|
+
doctrine: "a resume re-admits recovered agents without re-counting them against lifetimeSpawnCap (RV2201): the kill-mid-fan-out journal resumes to the finished dossier at the EXACT cap, with no 'lifetime spawn cap' decline journaled and only the unsettled workers re-paid; the c7 resume starved its judge and synthesis on re-counted admissions",
|
|
2811
3277
|
async run() {
|
|
2812
|
-
const
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
3278
|
+
const makeAdapter = () => {
|
|
3279
|
+
const calls = [];
|
|
3280
|
+
return {
|
|
3281
|
+
id: "fake",
|
|
3282
|
+
calls,
|
|
3283
|
+
caps: scriptedCaps,
|
|
3284
|
+
async *stream(req) {
|
|
3285
|
+
const call = calls.length;
|
|
3286
|
+
calls.push(req);
|
|
3287
|
+
await Promise.resolve();
|
|
3288
|
+
if (scriptedAgentType(req) !== "") {
|
|
3289
|
+
yield {
|
|
3290
|
+
type: "text-delta",
|
|
3291
|
+
text: "gathered evidence"
|
|
3292
|
+
};
|
|
3293
|
+
yield {
|
|
3294
|
+
type: "finish",
|
|
3295
|
+
finish: { reason: "stop" },
|
|
3296
|
+
usage: {
|
|
3297
|
+
inputTokens: 10,
|
|
3298
|
+
outputTokens: 5,
|
|
3299
|
+
cacheReadTokens: 0,
|
|
3300
|
+
cacheWriteTokens: 0
|
|
3301
|
+
}
|
|
3302
|
+
};
|
|
3303
|
+
return;
|
|
3304
|
+
}
|
|
3305
|
+
if (!(req.tools ?? []).some((tool) => tool.name === "spawn_agent")) {
|
|
3306
|
+
for (const event of scriptedToolEvents("finish", { result: "the resumed dossier" }, `id-${String(call)}-0`)) yield event;
|
|
3307
|
+
yield {
|
|
3308
|
+
type: "finish",
|
|
3309
|
+
finish: { reason: "tool-calls" },
|
|
3310
|
+
usage: {
|
|
3311
|
+
inputTokens: 12,
|
|
3312
|
+
outputTokens: 8,
|
|
3313
|
+
cacheReadTokens: 0,
|
|
3314
|
+
cacheWriteTokens: 0
|
|
3315
|
+
}
|
|
3316
|
+
};
|
|
3317
|
+
return;
|
|
3318
|
+
}
|
|
3319
|
+
const turn = calls.filter((prior) => scriptedAgentType(prior) === "" && (prior.tools ?? []).some((tool) => tool.name === "spawn_agent")).length === 1 ? scriptedToolEvents("spawn_agent", {
|
|
3320
|
+
agentType: "worker",
|
|
3321
|
+
prompt: "gather A"
|
|
3322
|
+
}, `id-${String(call)}-0`).concat(scriptedToolEvents("spawn_agent", {
|
|
3323
|
+
agentType: "worker",
|
|
3324
|
+
prompt: "gather B"
|
|
3325
|
+
}, `id-${String(call)}-1`), scriptedToolEvents("spawn_agent", {
|
|
3326
|
+
agentType: "worker",
|
|
3327
|
+
prompt: "gather C"
|
|
3328
|
+
}, `id-${String(call)}-2`)) : scriptedToolEvents("finish", { result: "the coordination draft" }, `id-${String(call)}-0`);
|
|
3329
|
+
for (const event of turn) yield event;
|
|
2856
3330
|
yield {
|
|
2857
3331
|
type: "finish",
|
|
2858
|
-
finish: { reason: "
|
|
3332
|
+
finish: { reason: "tool-calls" },
|
|
2859
3333
|
usage: {
|
|
2860
3334
|
inputTokens: 10,
|
|
2861
3335
|
outputTokens: 5,
|
|
@@ -2863,153 +3337,91 @@ const SCENARIOS = [
|
|
|
2863
3337
|
cacheWriteTokens: 0
|
|
2864
3338
|
}
|
|
2865
3339
|
};
|
|
2866
|
-
return;
|
|
2867
3340
|
}
|
|
2868
|
-
|
|
2869
|
-
const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
|
|
2870
|
-
agentType: "worker",
|
|
2871
|
-
prompt: "research A"
|
|
2872
|
-
}, `id-${String(call)}-0`).concat(toolEvents("spawn_agent", {
|
|
2873
|
-
agentType: "worker",
|
|
2874
|
-
prompt: "research B"
|
|
2875
|
-
}, `id-${String(call)}-1`), toolEvents("spawn_agent", {
|
|
2876
|
-
agentType: "worker",
|
|
2877
|
-
prompt: "research C"
|
|
2878
|
-
}, `id-${String(call)}-2`)) : toolEvents("finish", { result: "unreachable" }, `id-${String(call)}-0`);
|
|
2879
|
-
for (const event of turn) yield event;
|
|
2880
|
-
yield {
|
|
2881
|
-
type: "finish",
|
|
2882
|
-
finish: { reason: "tool-calls" },
|
|
2883
|
-
usage: {
|
|
2884
|
-
inputTokens: 10,
|
|
2885
|
-
outputTokens: orchTurn === 1 ? 2e3 : 5,
|
|
2886
|
-
cacheReadTokens: 0,
|
|
2887
|
-
cacheWriteTokens: 0
|
|
2888
|
-
}
|
|
2889
|
-
};
|
|
2890
|
-
}
|
|
3341
|
+
};
|
|
2891
3342
|
};
|
|
2892
|
-
const
|
|
2893
|
-
const
|
|
3343
|
+
const LIFETIME_SPAWNS = 5;
|
|
3344
|
+
const wf = makeOrchestratorWorkflow("the resumable fan-out", {
|
|
3345
|
+
synthesis: {
|
|
3346
|
+
estCost: .006,
|
|
3347
|
+
limits: { maxOutputTokensPerTurn: 500 }
|
|
3348
|
+
},
|
|
3349
|
+
budget: { finalizeReserveUsd: 0 }
|
|
3350
|
+
});
|
|
3351
|
+
const makeEngine = (adapter, store) => createEngine({
|
|
2894
3352
|
adapters: [adapter],
|
|
2895
3353
|
stores: { journal: store },
|
|
3354
|
+
budgetDefaults: { lifetimeSpawnCap: LIFETIME_SPAWNS },
|
|
2896
3355
|
defaults: {
|
|
2897
3356
|
routing: {
|
|
2898
3357
|
loop: "fake:model",
|
|
2899
|
-
orchestrate: "fake:model"
|
|
3358
|
+
orchestrate: "fake:model",
|
|
3359
|
+
synthesize: "fake:model"
|
|
2900
3360
|
},
|
|
2901
|
-
profiles: { worker: {
|
|
2902
|
-
description: "the oversized worker",
|
|
2903
|
-
limits: { maxOutputTokensPerTurn: 1e4 }
|
|
2904
|
-
} }
|
|
3361
|
+
profiles: { worker: { description: "the resumable gatherer" } }
|
|
2905
3362
|
}
|
|
2906
3363
|
});
|
|
2907
|
-
const
|
|
2908
|
-
const
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
const
|
|
2917
|
-
const
|
|
2918
|
-
const
|
|
2919
|
-
const
|
|
2920
|
-
const
|
|
2921
|
-
const
|
|
2922
|
-
const
|
|
2923
|
-
const
|
|
3364
|
+
const storeA = new InMemoryStore();
|
|
3365
|
+
const seeded = await makeEngine(makeAdapter(), storeA).run(wf, void 0, {
|
|
3366
|
+
runId: "fault-resume-famine",
|
|
3367
|
+
budgetUsd: 10
|
|
3368
|
+
}).result;
|
|
3369
|
+
if (seeded.status !== "ok") throw new Error(`fault kit: the seeding run settled '${seeded.status}' instead of ok`);
|
|
3370
|
+
const entriesA = await storeA.load("fault-resume-famine");
|
|
3371
|
+
const firstWorkerOk = entriesA.findIndex((entry) => entry.kind === "agent" && entry.status === "ok");
|
|
3372
|
+
if (firstWorkerOk < 0) throw new Error("fault kit: the seeding journal carries no worker terminal to cut at");
|
|
3373
|
+
const cut = entriesA.slice(0, firstWorkerOk + 1);
|
|
3374
|
+
const storeB = new InMemoryStore();
|
|
3375
|
+
for (const entry of cut) await storeB.append("fault-resume-famine", entry);
|
|
3376
|
+
const resumeAdapter = makeAdapter();
|
|
3377
|
+
const resumed = await makeEngine(resumeAdapter, storeB).resume("fault-resume-famine", wf).result;
|
|
3378
|
+
const entriesB = await storeB.load("fault-resume-famine");
|
|
3379
|
+
const capDeclines = entriesB.filter((entry) => entry.kind === "decision" && JSON.stringify(entry.value ?? {}).includes("lifetime spawn cap"));
|
|
3380
|
+
const redemptionDeclines = entriesB.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_synthesis_redemption_declined");
|
|
3381
|
+
const liveWorkerCalls = resumeAdapter.calls.filter((req) => scriptedAgentType(req) !== "").length;
|
|
3382
|
+
const distinctAgentScopes = new Set(entriesB.filter((entry) => entry.kind === "agent" && entry.status === "running").map((entry) => `${entry.scope}#${String(entry.key)}`)).size;
|
|
2924
3383
|
return {
|
|
2925
3384
|
observation: {
|
|
2926
|
-
matched:
|
|
2927
|
-
detail: `
|
|
3385
|
+
matched: resumed.status === "ok" && resumed.value === "the resumed dossier" && capDeclines.length === 0 && redemptionDeclines.length === 0 && liveWorkerCalls === 2 && distinctAgentScopes === LIFETIME_SPAWNS,
|
|
3386
|
+
detail: `resumed '${resumed.status}' result=${JSON.stringify(resumed.value)} at lifetimeSpawnCap ${String(LIFETIME_SPAWNS)} (${String(distinctAgentScopes)} distinct lifetime agent scopes); cap declines journaled=${String(capDeclines.length)}, redemption declines=${String(redemptionDeclines.length)}, live worker calls in the resumed segment=${String(liveWorkerCalls)} (the settled worker replayed free)`
|
|
2928
3387
|
},
|
|
2929
3388
|
artifacts: [
|
|
2930
|
-
jsonArtifact("
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
3389
|
+
jsonArtifact("journal-cut.json", cut),
|
|
3390
|
+
jsonArtifact("resume-outcome.json", {
|
|
3391
|
+
status: resumed.status,
|
|
3392
|
+
value: resumed.value ?? null,
|
|
3393
|
+
liveWorkerCalls
|
|
2934
3394
|
}),
|
|
2935
|
-
jsonArtifact("
|
|
2936
|
-
jsonArtifact("journal.json", entries)
|
|
3395
|
+
jsonArtifact("journal-resumed.json", entriesB)
|
|
2937
3396
|
]
|
|
2938
3397
|
};
|
|
2939
3398
|
}
|
|
2940
3399
|
},
|
|
2941
3400
|
{
|
|
2942
|
-
name: "
|
|
2943
|
-
doctrine: "
|
|
3401
|
+
name: "validator-guidance-conflict",
|
|
3402
|
+
doctrine: "the evidence-grade reason names the SAFE composition against its cited-value sibling and names the run id it wants written (RV2202, RV2501, RV2502): the c3 trap finish repairs in ONE round by carrying THIS run's own id in the graded sentence, beside a source citation, with cited-value reading that id as identity and never rejecting; the third subscription run burned both repairs between the two verdicts, and the 1.226.0 comparison run burned both again on an id no artifact pattern could match",
|
|
2944
3403
|
async run() {
|
|
2945
|
-
const
|
|
2946
|
-
|
|
3404
|
+
const TRAP_FINISH = "The reserve fold is live-observed under sustained load. The engine seals the journal at settle (`README.md:3`).";
|
|
3405
|
+
const FIXED_FINISH = "The reserve fold is live-observed under sustained load in run `fault-guidance-conflict`, where the engine seals the journal at settle (`README.md:3`).";
|
|
2947
3406
|
const calls = [];
|
|
2948
|
-
|
|
2949
|
-
{
|
|
2950
|
-
type: "tool-call-start",
|
|
2951
|
-
id,
|
|
2952
|
-
name
|
|
2953
|
-
},
|
|
2954
|
-
{
|
|
2955
|
-
type: "tool-call-delta",
|
|
2956
|
-
id,
|
|
2957
|
-
argsTextDelta: JSON.stringify(args)
|
|
2958
|
-
},
|
|
2959
|
-
{
|
|
2960
|
-
type: "tool-call-end",
|
|
2961
|
-
id,
|
|
2962
|
-
args
|
|
2963
|
-
}
|
|
2964
|
-
];
|
|
3407
|
+
let finishAttempts = 0;
|
|
2965
3408
|
const adapter = {
|
|
2966
3409
|
id: "fake",
|
|
2967
3410
|
calls,
|
|
2968
|
-
caps:
|
|
2969
|
-
contextWindow: 2e5,
|
|
2970
|
-
maxOutputTokens: 16e3,
|
|
2971
|
-
structuredOutput: "native",
|
|
2972
|
-
supportsTemperature: true,
|
|
2973
|
-
supportsParallelTools: true,
|
|
2974
|
-
reasoningEfforts: [],
|
|
2975
|
-
pricing: {
|
|
2976
|
-
inputUsdPerMTok: 1,
|
|
2977
|
-
outputUsdPerMTok: 10
|
|
2978
|
-
}
|
|
2979
|
-
}),
|
|
3411
|
+
caps: scriptedCaps,
|
|
2980
3412
|
async *stream(req) {
|
|
2981
3413
|
const call = calls.length;
|
|
2982
3414
|
calls.push(req);
|
|
2983
3415
|
await Promise.resolve();
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
text: "seated worker"
|
|
2988
|
-
};
|
|
2989
|
-
yield {
|
|
2990
|
-
type: "finish",
|
|
2991
|
-
finish: { reason: "stop" },
|
|
2992
|
-
usage: {
|
|
2993
|
-
inputTokens: 10,
|
|
2994
|
-
outputTokens: 5,
|
|
2995
|
-
cacheReadTokens: 0,
|
|
2996
|
-
cacheWriteTokens: 0
|
|
2997
|
-
}
|
|
2998
|
-
};
|
|
2999
|
-
return;
|
|
3000
|
-
}
|
|
3001
|
-
orchTurn += 1;
|
|
3002
|
-
const turn = orchTurn === 1 ? toolEvents("spawn_agent", {
|
|
3003
|
-
agentType: "worker",
|
|
3004
|
-
prompt: "seat 1"
|
|
3005
|
-
}, `id-${String(call)}-0`) : toolEvents("finish", { result: "stopped early" }, `id-${String(call)}-0`);
|
|
3006
|
-
for (const event of turn) yield event;
|
|
3416
|
+
finishAttempts += 1;
|
|
3417
|
+
const result = finishAttempts === 1 ? TRAP_FINISH : FIXED_FINISH;
|
|
3418
|
+
for (const event of scriptedToolEvents("finish", { result }, `id-${String(call)}-0`)) yield event;
|
|
3007
3419
|
yield {
|
|
3008
3420
|
type: "finish",
|
|
3009
3421
|
finish: { reason: "tool-calls" },
|
|
3010
3422
|
usage: {
|
|
3011
3423
|
inputTokens: 10,
|
|
3012
|
-
outputTokens:
|
|
3424
|
+
outputTokens: 40,
|
|
3013
3425
|
cacheReadTokens: 0,
|
|
3014
3426
|
cacheWriteTokens: 0
|
|
3015
3427
|
}
|
|
@@ -3020,43 +3432,45 @@ const SCENARIOS = [
|
|
|
3020
3432
|
const engine = createEngine({
|
|
3021
3433
|
adapters: [adapter],
|
|
3022
3434
|
stores: { journal: store },
|
|
3023
|
-
defaults: {
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
},
|
|
3028
|
-
profiles: { worker: {
|
|
3029
|
-
description: "the estimated worker",
|
|
3030
|
-
estCost: .7
|
|
3031
|
-
} }
|
|
3032
|
-
}
|
|
3435
|
+
defaults: { routing: {
|
|
3436
|
+
loop: "fake:model",
|
|
3437
|
+
orchestrate: "fake:model"
|
|
3438
|
+
} }
|
|
3033
3439
|
});
|
|
3034
|
-
const
|
|
3035
|
-
|
|
3036
|
-
|
|
3440
|
+
const resolveSource = (target) => target.path === "README.md" ? "the rulvar engine\nholds one denominator\nthe engine seals the journal at settle\n" : void 0;
|
|
3441
|
+
const wf = makeOrchestratorWorkflow("the guidance trap", { finishValidation: {
|
|
3442
|
+
validators: [evidenceGradeValidator(), citedValueValidator({
|
|
3443
|
+
resolve: resolveSource,
|
|
3444
|
+
window: 2
|
|
3445
|
+
})],
|
|
3446
|
+
maxRepairs: 2,
|
|
3447
|
+
repairTurnReserve: 2
|
|
3037
3448
|
} });
|
|
3038
|
-
const
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
const
|
|
3045
|
-
const
|
|
3046
|
-
const
|
|
3047
|
-
const
|
|
3048
|
-
const
|
|
3449
|
+
const outcome = await engine.run(wf, void 0, {
|
|
3450
|
+
runId: "fault-guidance-conflict",
|
|
3451
|
+
budgetUsd: 10
|
|
3452
|
+
}).result;
|
|
3453
|
+
const entries = await store.load("fault-guidance-conflict");
|
|
3454
|
+
const repairRequest = calls[1];
|
|
3455
|
+
const repairBytes = JSON.stringify(repairRequest?.messages ?? []);
|
|
3456
|
+
const guidanceQuoted = repairBytes.includes("write this run's id fault-guidance-conflict") && repairBytes.includes("may share a sentence with a source citation");
|
|
3457
|
+
const citedValueNamed = repairBytes.includes("cited-value reads a run id as identity");
|
|
3458
|
+
const decisionsText = JSON.stringify(entries.filter((entry) => entry.kind === "decision").map((entry) => entry.value ?? null));
|
|
3459
|
+
const citedValueRejected = decisionsText.includes("\"cited-value\"") ? decisionsText.includes("not present inside the cited window") : false;
|
|
3049
3460
|
return {
|
|
3050
3461
|
observation: {
|
|
3051
|
-
matched:
|
|
3052
|
-
detail: `run '${outcome.status}'
|
|
3462
|
+
matched: outcome.status === "ok" && outcome.value === FIXED_FINISH && finishAttempts === 2 && guidanceQuoted && citedValueNamed && !citedValueRejected,
|
|
3463
|
+
detail: `run '${outcome.status}' after ${String(finishAttempts)} finish attempt(s); the repair exchange named the run id and the shared-sentence composition (${String(guidanceQuoted)}) and named the identity reading (${String(citedValueNamed)}); cited-value rejected=${String(citedValueRejected)}; final result carries this run's own id beside the citation in the graded sentence: ${String(outcome.value === FIXED_FINISH)}`
|
|
3053
3464
|
},
|
|
3054
3465
|
artifacts: [
|
|
3055
3466
|
jsonArtifact("outcome.json", {
|
|
3056
3467
|
status: outcome.status,
|
|
3057
|
-
|
|
3468
|
+
value: outcome.value ?? null
|
|
3058
3469
|
}),
|
|
3059
|
-
|
|
3470
|
+
{
|
|
3471
|
+
name: "repair-request.json",
|
|
3472
|
+
content: repairBytes
|
|
3473
|
+
},
|
|
3060
3474
|
jsonArtifact("journal.json", entries)
|
|
3061
3475
|
]
|
|
3062
3476
|
};
|