@rulvar/evals 1.242.0 → 1.243.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +8 -2
- package/dist/index.js +593 -256
- package/package.json +6 -6
package/dist/index.d.ts
CHANGED
|
@@ -843,9 +843,15 @@ declare function runFaultInjection(options?: RunFaultInjectionOptions): Promise<
|
|
|
843
843
|
* classes, plus the nineteenth benchmark's pair (RV1809):
|
|
844
844
|
* 'modality-overclaim' is a mitigation stated as an unconditional
|
|
845
845
|
* guarantee, and 'scope-ambiguity' is a child-only total printed as a
|
|
846
|
-
* whole-workflow figure.
|
|
846
|
+
* whole-workflow figure. The third comparison experiment validated
|
|
847
|
+
* three more (RV3804): 'bound-conflation' lists opt-in caps and
|
|
848
|
+
* unconditional guards as one mode, 'derived-premise' is a derived
|
|
849
|
+
* figure whose premise contradicts the declared input (2,000 slots
|
|
850
|
+
* computed from a 30 minute window where the input declares a 20
|
|
851
|
+
* minute burst), and 'cost-basis' prints a locally estimated total as
|
|
852
|
+
* the provider's bill.
|
|
847
853
|
*/
|
|
848
|
-
type ClaimCorpusClass = "live-fact" | "package-identity" | "inverted-default" | "numeric-range" | "negation" | "bounded-coverage" | "modality-overclaim" | "scope-ambiguity";
|
|
854
|
+
type ClaimCorpusClass = "live-fact" | "package-identity" | "inverted-default" | "numeric-range" | "negation" | "bounded-coverage" | "modality-overclaim" | "scope-ambiguity" | "bound-conflation" | "derived-premise" | "cost-basis";
|
|
849
855
|
/** One adversarial case: a draft, its contradicting evidence, and the mechanical expectations. */
|
|
850
856
|
interface ClaimCorpusCase {
|
|
851
857
|
id: string;
|
package/dist/index.js
CHANGED
|
@@ -3397,23 +3397,23 @@ const resumeSpawnFamine = {
|
|
|
3397
3397
|
* beside the live-observed claim, the model wove the id into a cited
|
|
3398
3398
|
* sentence, cited-value lawfully rejected the id as a value absent from
|
|
3399
3399
|
* the cited window, and both repairs burned without an exit. The
|
|
3400
|
-
* evidence-grade reason
|
|
3401
|
-
*
|
|
3402
|
-
*
|
|
3403
|
-
*
|
|
3404
|
-
*
|
|
3405
|
-
*
|
|
3406
|
-
*
|
|
3407
|
-
*
|
|
3408
|
-
*
|
|
3409
|
-
* the old bundle could not accept from either side.
|
|
3400
|
+
* evidence-grade reason then learned to NAME the safe composition
|
|
3401
|
+
* (RV2202) and the id itself (RV2501), with cited-value reading that id
|
|
3402
|
+
* as identity (RV2502), so the trap converged in one repair round. The
|
|
3403
|
+
* deterministic patch (RV3801) closes the arc: the host now PERFORMS
|
|
3404
|
+
* the reason's own prescription, so the same trap converges in ZERO
|
|
3405
|
+
* model repairs, the third subscription run's two burned wires cost
|
|
3406
|
+
* nothing at all, cited-value stays silent over the patched document,
|
|
3407
|
+
* and the guidance bytes remain journaled on the healed verdict for
|
|
3408
|
+
* the failure shapes a patch cannot reach.
|
|
3410
3409
|
*/
|
|
3411
3410
|
const validatorGuidanceConflict = {
|
|
3412
3411
|
name: "validator-guidance-conflict",
|
|
3413
|
-
doctrine: "the evidence-grade reason names the SAFE composition against its cited-value sibling and names the run id it wants written (RV2202, RV2501, RV2502): the c3 trap
|
|
3412
|
+
doctrine: "the evidence-grade reason names the SAFE composition against its cited-value sibling and names the run id it wants written (RV2202, RV2501, RV2502), and the finish loop now performs that prescription itself (RV3801): the c3 trap heals in ZERO model repairs, the accepted document carries THIS run's own id inside the graded sentence with cited-value reading it as identity and never rejecting, and the composition-safe guidance bytes ride the journaled healed verdict; the third subscription run burned both repairs between the two verdicts, and the 1.226.0 comparison run burned both again on an id no artifact pattern could match",
|
|
3414
3413
|
async run() {
|
|
3415
3414
|
const TRAP_FINISH = "The reserve fold is live-observed under sustained load. The engine seals the journal at settle (`README.md:3`).";
|
|
3416
3415
|
const FIXED_FINISH = "The reserve fold is live-observed under sustained load in run `fault-guidance-conflict`, where the engine seals the journal at settle (`README.md:3`).";
|
|
3416
|
+
const PATCHED_TRAP = "The reserve fold is live-observed under sustained load (run fault-guidance-conflict). The engine seals the journal at settle (`README.md:3`).";
|
|
3417
3417
|
const calls = [];
|
|
3418
3418
|
let finishAttempts = 0;
|
|
3419
3419
|
const adapter = {
|
|
@@ -3462,28 +3462,20 @@ const validatorGuidanceConflict = {
|
|
|
3462
3462
|
budgetUsd: 10
|
|
3463
3463
|
}).result;
|
|
3464
3464
|
const entries = await store.load("fault-guidance-conflict");
|
|
3465
|
-
const
|
|
3466
|
-
const
|
|
3467
|
-
const guidanceQuoted =
|
|
3468
|
-
const citedValueNamed =
|
|
3469
|
-
const
|
|
3470
|
-
const citedValueRejected = decisionsText.includes("\"cited-value\"") ? decisionsText.includes("not present inside the cited window") : false;
|
|
3465
|
+
const decision = entries.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finish_validation").map((entry) => entry.value)[0];
|
|
3466
|
+
const healedReasons = (decision?.deterministicRepair?.healed ?? []).flatMap((failure) => failure.reasons ?? []).join(" ");
|
|
3467
|
+
const guidanceQuoted = healedReasons.includes("write this run's id fault-guidance-conflict") && healedReasons.includes("may share a sentence with a source citation");
|
|
3468
|
+
const citedValueNamed = healedReasons.includes("cited-value reads a run id as identity");
|
|
3469
|
+
const citedValueRejected = JSON.stringify(entries.filter((entry) => entry.kind === "decision").map((entry) => entry.value ?? null)).includes("not present inside the cited window");
|
|
3471
3470
|
return {
|
|
3472
3471
|
observation: {
|
|
3473
|
-
matched: outcome.status === "ok" && outcome.value ===
|
|
3474
|
-
detail: `run '${outcome.status}' after ${String(finishAttempts)} finish attempt(s); the
|
|
3472
|
+
matched: outcome.status === "ok" && outcome.value === PATCHED_TRAP && finishAttempts === 1 && decision?.verdict === "accepted" && decision.deterministicRepair?.outcome === "accepted" && guidanceQuoted && citedValueNamed && !citedValueRejected,
|
|
3473
|
+
detail: `run '${outcome.status}' after ${String(finishAttempts)} finish attempt(s), healed host side (patch outcome='${String(decision?.deterministicRepair?.outcome)}'); the journaled healed verdict named the run id and the shared-sentence composition (${String(guidanceQuoted)}) and named the identity reading (${String(citedValueNamed)}); cited-value rejected=${String(citedValueRejected)}; final result carries this run's own id inside the graded sentence: ` + String(outcome.value === PATCHED_TRAP)
|
|
3475
3474
|
},
|
|
3476
|
-
artifacts: [
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
|
-
|
|
3480
|
-
}),
|
|
3481
|
-
{
|
|
3482
|
-
name: "repair-request.json",
|
|
3483
|
-
content: repairBytes
|
|
3484
|
-
},
|
|
3485
|
-
jsonArtifact("journal.json", entries)
|
|
3486
|
-
]
|
|
3475
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3476
|
+
status: outcome.status,
|
|
3477
|
+
value: outcome.value ?? null
|
|
3478
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3487
3479
|
};
|
|
3488
3480
|
}
|
|
3489
3481
|
};
|
|
@@ -3577,6 +3569,513 @@ function tailSpans(entries) {
|
|
|
3577
3569
|
judges: settled.filter((entry) => entry.costAttribution?.label === "claim-consistency-judge-final")
|
|
3578
3570
|
};
|
|
3579
3571
|
}
|
|
3572
|
+
/**
|
|
3573
|
+
* RV3307 as the arc the losing config wanted: the final judge names
|
|
3574
|
+
* the contradiction, the findings ride one more composition, the
|
|
3575
|
+
* re-judge clears the repaired document, and the settled envelope
|
|
3576
|
+
* reports THAT document as the judged one.
|
|
3577
|
+
*/
|
|
3578
|
+
const repairRoundHonesty = {
|
|
3579
|
+
name: "repair-round-honesty",
|
|
3580
|
+
doctrine: "the bounded post judge repair consumes the named finding (RV3307): one more composition carries the findings, the re-judge clears it, the run settles ok with the meta describing the repaired document (judgedStage 'final', findings 0, judgedHash equal to the shipped finalHash), two compositions and two final judge passes in the journal, and the invoice in the same denominator as the envelope",
|
|
3581
|
+
async run() {
|
|
3582
|
+
const { engine, store } = tailEngine(tailAdapter({
|
|
3583
|
+
judge: (call) => call === 1 ? TAIL_FINDS : TAIL_AGREES,
|
|
3584
|
+
finals: [TAIL_FINAL_INVERTED, TAIL_FINAL_CLEAN]
|
|
3585
|
+
}));
|
|
3586
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3587
|
+
...TAIL_OPTS,
|
|
3588
|
+
claimConsistency: {
|
|
3589
|
+
stage: "final",
|
|
3590
|
+
onFound: "repair"
|
|
3591
|
+
}
|
|
3592
|
+
}), void 0, {
|
|
3593
|
+
runId: "fault-repair-honesty",
|
|
3594
|
+
budgetUsd: 10
|
|
3595
|
+
}).result;
|
|
3596
|
+
const value = outcome.value;
|
|
3597
|
+
const entries = await store.load("fault-repair-honesty");
|
|
3598
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3599
|
+
const invoice = invoiceFromJournal(entries, () => 0);
|
|
3600
|
+
return {
|
|
3601
|
+
observation: {
|
|
3602
|
+
matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && Array.isArray(value.claimContradictions) && value.claimContradictions.length === 0 && value.claimConsistencyMeta?.judgedStage === "final" && value.claimConsistencyMeta.findings === 0 && typeof value.claimConsistencyMeta.judgedHash === "string" && value.claimConsistencyMeta.judgedHash === value.draftToFinal?.finalHash && compositions.length === 2 && judges.length === 2 && [...compositions, ...judges].every((entry) => entry.status === "ok") && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests,
|
|
3603
|
+
detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the repaired document" : "an unexpected document"}; meta judgedStage='${String(value?.claimConsistencyMeta?.judgedStage)}' findings=${String(value?.claimConsistencyMeta?.findings)} judgedHash==finalHash=${String(value?.claimConsistencyMeta?.judgedHash === value?.draftToFinal?.finalHash)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es); wires ${String(outcome.envelope.wireRequests)} == invoice ` + String(invoice.cardinality.wireRequests)
|
|
3604
|
+
},
|
|
3605
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3606
|
+
status: outcome.status,
|
|
3607
|
+
value: outcome.value ?? null,
|
|
3608
|
+
envelope: outcome.envelope
|
|
3609
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3610
|
+
};
|
|
3611
|
+
}
|
|
3612
|
+
};
|
|
3613
|
+
/**
|
|
3614
|
+
* RV3307's refusal half: the repair round runs, the re-judge still
|
|
3615
|
+
* finds, and the run fails typed instead of settling over the
|
|
3616
|
+
* surviving contradiction, with the payload distinguishing the two
|
|
3617
|
+
* documents by hash.
|
|
3618
|
+
*/
|
|
3619
|
+
const repairSurvivorRefusal = {
|
|
3620
|
+
name: "repair-survivor-refusal",
|
|
3621
|
+
doctrine: "findings that survive the bounded repair round fail the run typed (RV3307): source 'orchestrator_claim_consistency', repairsUsed 1, preRepairHash distinct from repairedHash (the round demonstrably produced a different document and the judge still refused it), two compositions and two final judge passes paid, never a silent ok over the surviving contradiction",
|
|
3622
|
+
async run() {
|
|
3623
|
+
const { engine, store } = tailEngine(tailAdapter({
|
|
3624
|
+
judge: () => TAIL_FINDS,
|
|
3625
|
+
finals: [TAIL_FINAL_INVERTED, TAIL_FINAL_STILL_INVERTED]
|
|
3626
|
+
}));
|
|
3627
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3628
|
+
...TAIL_OPTS,
|
|
3629
|
+
claimConsistency: {
|
|
3630
|
+
stage: "final",
|
|
3631
|
+
onFound: "repair"
|
|
3632
|
+
}
|
|
3633
|
+
}), void 0, {
|
|
3634
|
+
runId: "fault-repair-survivor",
|
|
3635
|
+
budgetUsd: 10
|
|
3636
|
+
}).result;
|
|
3637
|
+
const data = outcome.error?.data ?? {};
|
|
3638
|
+
const entries = await store.load("fault-repair-survivor");
|
|
3639
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3640
|
+
return {
|
|
3641
|
+
observation: {
|
|
3642
|
+
matched: outcome.status === "error" && (outcome.error?.message ?? "").includes("after the bounded repair round") && data.source === "orchestrator_claim_consistency" && data.repairsUsed === 1 && typeof data.preRepairHash === "string" && typeof data.repairedHash === "string" && data.preRepairHash !== data.repairedHash && compositions.length === 2 && judges.length === 2,
|
|
3643
|
+
detail: `run '${outcome.status}': ${outcome.error?.message ?? ""}; repairsUsed=${String(data.repairsUsed)}, hashes distinct=${String(data.preRepairHash !== data.repairedHash)}, ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3644
|
+
},
|
|
3645
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3646
|
+
status: outcome.status,
|
|
3647
|
+
error: outcome.error ?? null
|
|
3648
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3649
|
+
};
|
|
3650
|
+
}
|
|
3651
|
+
};
|
|
3652
|
+
/**
|
|
3653
|
+
* The round's second death told straight (RV3601): the third
|
|
3654
|
+
* comparison run's repair round DISPATCHED, paid two wires, produced
|
|
3655
|
+
* a candidate its own finish contract rejected, and the terminal read
|
|
3656
|
+
* 'could not dispatch' with repairsUsed 0 beside a null judge meta
|
|
3657
|
+
* and null findings. The class now names the host rejection, counts
|
|
3658
|
+
* the bounded round as spent, mirrors the finish verdict facts, and
|
|
3659
|
+
* the engine lifts the findings beside the meta onto the outcome.
|
|
3660
|
+
*/
|
|
3661
|
+
const repairRoundHostRejection = {
|
|
3662
|
+
name: "repair-round-host-rejection",
|
|
3663
|
+
doctrine: "a repair round that dispatched and lost its candidate to the finish contract fails typed as a host rejection (RV3601): the message names the dispatch, never 'could not dispatch'; data carries roundDispatched true, repairsUsed 1, the judge meta beside the findings, and the finish verdict verbatim (failed validators, candidateHash, candidateChars); the outcome lifts the findings beside the meta",
|
|
3664
|
+
async run() {
|
|
3665
|
+
const { engine, store } = tailEngine(tailAdapter({
|
|
3666
|
+
judge: () => TAIL_FINDS,
|
|
3667
|
+
finals: [
|
|
3668
|
+
TAIL_FINAL_INVERTED,
|
|
3669
|
+
TAIL_FINAL_UNGROUNDED,
|
|
3670
|
+
TAIL_FINAL_UNGROUNDED
|
|
3671
|
+
]
|
|
3672
|
+
}));
|
|
3673
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3674
|
+
...TAIL_OPTS,
|
|
3675
|
+
claimConsistency: {
|
|
3676
|
+
stage: "final",
|
|
3677
|
+
onFound: "repair"
|
|
3678
|
+
},
|
|
3679
|
+
finishValidation: {
|
|
3680
|
+
validators: [minMatchesValidator({
|
|
3681
|
+
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3682
|
+
min: 1,
|
|
3683
|
+
name: "provenance-anchor"
|
|
3684
|
+
})],
|
|
3685
|
+
maxRepairs: 1
|
|
3686
|
+
}
|
|
3687
|
+
}), void 0, {
|
|
3688
|
+
runId: "fault-repair-host-rejection",
|
|
3689
|
+
budgetUsd: 10
|
|
3690
|
+
}).result;
|
|
3691
|
+
const data = outcome.error?.data ?? {};
|
|
3692
|
+
const entries = await store.load("fault-repair-host-rejection");
|
|
3693
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3694
|
+
const liftedFindings = outcome.claimContradictions;
|
|
3695
|
+
const message = outcome.error?.message ?? "";
|
|
3696
|
+
return {
|
|
3697
|
+
observation: {
|
|
3698
|
+
matched: outcome.status === "error" && message.includes("dispatched and its repaired candidate failed host validation") && !message.includes("could not dispatch") && data.source === "orchestrator_claim_consistency" && data.roundDispatched === true && data.repairsUsed === 1 && data.claimConsistencyMeta?.findings === 1 && Array.isArray(data.claimContradictions) && data.claimContradictions.length === 1 && typeof data.finishValidation?.callId === "string" && data.finishValidation.repairsUsed === 1 && data.finishValidation.maxRepairs === 1 && typeof data.finishValidation.candidateHash === "string" && typeof data.finishValidation.candidateChars === "number" && JSON.stringify(data.finishValidation.failed ?? null).includes("provenance-anchor") && Array.isArray(liftedFindings) && liftedFindings.length === 1 && compositions.length === 2 && judges.length === 1,
|
|
3699
|
+
detail: `run '${outcome.status}': roundDispatched=${String(data.roundDispatched)}, repairsUsed=${String(data.repairsUsed)}, meta findings=${String(data.claimConsistencyMeta?.findings)}, finish verdict carries hash=${String(typeof data.finishValidation?.candidateHash === "string")} chars=${String(data.finishValidation?.candidateChars)}, outcome lifts findings=${String(Array.isArray(liftedFindings))}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3700
|
+
},
|
|
3701
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3702
|
+
status: outcome.status,
|
|
3703
|
+
error: outcome.error ?? null
|
|
3704
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3705
|
+
};
|
|
3706
|
+
}
|
|
3707
|
+
};
|
|
3708
|
+
/**
|
|
3709
|
+
* The mechanical repair pool belongs to the invocation (RV3602): the
|
|
3710
|
+
* third comparison run's initial composition spent the single run
|
|
3711
|
+
* wide repair, so the bounded round entered with zero retries by
|
|
3712
|
+
* construction and its first regression was final. This scenario is
|
|
3713
|
+
* that exact frozen sequence carried one step further: the initial
|
|
3714
|
+
* composition trips the contract and repairs mechanically, the final
|
|
3715
|
+
* judge finds the real contradiction, the round's candidate trips the
|
|
3716
|
+
* contract AGAIN, and the round's OWN fresh pool grants the retry
|
|
3717
|
+
* that lands the clean document; the run settles ok over it.
|
|
3718
|
+
*/
|
|
3719
|
+
const repairRoundOwnPool = {
|
|
3720
|
+
name: "repair-round-own-pool",
|
|
3721
|
+
doctrine: "the mechanical repair pool belongs to one composition invocation (RV3602): the bounded claim repair round enters with the full maxRepairs even after the initial composition spent its own, so the frozen third comparison sequence converges to ok/complete with verdicts repair/accepted twice and repairsUsed restarting at the invocation boundary, never a host rejection born of an inherited spent pool; the round's prompt also carries the HOST VALIDATION LESSONS block naming the failure the run already paid for (RV3603), while the initial composition carries none",
|
|
3722
|
+
async run() {
|
|
3723
|
+
let judgeCalls = 0;
|
|
3724
|
+
const adapter = tailAdapter({
|
|
3725
|
+
judge: () => (judgeCalls += 1) === 1 ? TAIL_FINDS : TAIL_AGREES,
|
|
3726
|
+
finals: [
|
|
3727
|
+
TAIL_FINAL_UNGROUNDED_INITIAL,
|
|
3728
|
+
TAIL_FINAL_INVERTED,
|
|
3729
|
+
TAIL_FINAL_UNGROUNDED,
|
|
3730
|
+
TAIL_FINAL_CLEAN
|
|
3731
|
+
]
|
|
3732
|
+
});
|
|
3733
|
+
const { engine, store } = tailEngine(adapter);
|
|
3734
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3735
|
+
...TAIL_OPTS,
|
|
3736
|
+
claimConsistency: {
|
|
3737
|
+
stage: "final",
|
|
3738
|
+
onFound: "repair"
|
|
3739
|
+
},
|
|
3740
|
+
finishValidation: {
|
|
3741
|
+
validators: [minMatchesValidator({
|
|
3742
|
+
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3743
|
+
min: 1,
|
|
3744
|
+
name: "provenance-anchor"
|
|
3745
|
+
})],
|
|
3746
|
+
maxRepairs: 1
|
|
3747
|
+
}
|
|
3748
|
+
}), void 0, {
|
|
3749
|
+
runId: "fault-repair-own-pool",
|
|
3750
|
+
budgetUsd: 10
|
|
3751
|
+
}).result;
|
|
3752
|
+
const value = outcome.value;
|
|
3753
|
+
const entries = await store.load("fault-repair-own-pool");
|
|
3754
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3755
|
+
const verdictRows = entries.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finish_validation").map((entry) => entry.value);
|
|
3756
|
+
const verdicts = verdictRows.map((row) => row.verdict).join(",");
|
|
3757
|
+
const pools = verdictRows.map((row) => String(row.repairsUsed)).join(",");
|
|
3758
|
+
const compositionPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
|
|
3759
|
+
const lessonCarried = compositionPrompts.length === 4 && compositionPrompts[0]?.includes("HOST VALIDATION LESSONS") === false && compositionPrompts[2]?.includes("HOST VALIDATION LESSONS") === true && compositionPrompts[2]?.includes("provenance-anchor") === true;
|
|
3760
|
+
return {
|
|
3761
|
+
observation: {
|
|
3762
|
+
matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && value.claimConsistencyMeta?.findings === 0 && verdicts === "repair,accepted,repair,accepted" && pools === "0,1,0,1" && (value.rejectedFinishCandidates?.length ?? 0) === 2 && value.rejectedFinishCandidates?.every((row) => row.verdict === "repair") === true && lessonCarried && compositions.length === 2 && judges.length === 2,
|
|
3763
|
+
detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the repaired document" : "an unexpected document"}; verdicts [${verdicts}], repairsUsed [${pools}] (fresh pool at the invocation boundary); lesson carried=${String(lessonCarried)}; meta findings=${String(value?.claimConsistencyMeta?.findings)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3764
|
+
},
|
|
3765
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3766
|
+
status: outcome.status,
|
|
3767
|
+
value: outcome.value ?? null,
|
|
3768
|
+
envelope: outcome.envelope
|
|
3769
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3770
|
+
};
|
|
3771
|
+
}
|
|
3772
|
+
};
|
|
3773
|
+
/**
|
|
3774
|
+
* The convergence hold (RV3701): the round is a two invocation
|
|
3775
|
+
* bargain, and the third comparison run's class was an honest fail
|
|
3776
|
+
* closed WITHOUT convergence. RV3602 gave the round its own mechanical
|
|
3777
|
+
* pool; this hold closes the money twin: the verdict money is held
|
|
3778
|
+
* from the moment the round is admitted, so a round the budget can
|
|
3779
|
+
* only start refuses BEFORE any of its wires instead of paying for a
|
|
3780
|
+
* candidate nobody can rule on.
|
|
3781
|
+
*/
|
|
3782
|
+
const repairRoundVerdictReserve = {
|
|
3783
|
+
name: "repair-round-verdict-reserve",
|
|
3784
|
+
doctrine: "the repair round pays for its verdict up front (RV3701): the money of the round second judge pass (the declared judge estCost first, else the run own observed post draft judge price) is held from the moment the round is admitted, a round the budget can only start refuses through the honest pre dispatch class before any of its wires with the held convergence reserve named in the refusal arithmetic, and exactly one composition is ever paid for",
|
|
3785
|
+
async run() {
|
|
3786
|
+
const { engine, store } = tailEngine(tailAdapter({
|
|
3787
|
+
judge: () => TAIL_FINDS,
|
|
3788
|
+
finals: [TAIL_FINAL_INVERTED]
|
|
3789
|
+
}));
|
|
3790
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3791
|
+
...TAIL_OPTS,
|
|
3792
|
+
synthesis: {
|
|
3793
|
+
limits: { maxTurns: 3 },
|
|
3794
|
+
estCost: .2
|
|
3795
|
+
},
|
|
3796
|
+
budget: {
|
|
3797
|
+
capUsd: .3,
|
|
3798
|
+
capFraction: 1
|
|
3799
|
+
},
|
|
3800
|
+
claimConsistency: {
|
|
3801
|
+
stage: "final",
|
|
3802
|
+
onFound: "repair",
|
|
3803
|
+
judge: { estCost: .2 }
|
|
3804
|
+
}
|
|
3805
|
+
}), void 0, {
|
|
3806
|
+
runId: "fault-repair-verdict-reserve",
|
|
3807
|
+
budgetUsd: 10
|
|
3808
|
+
}).result;
|
|
3809
|
+
const data = outcome.error?.data ?? {};
|
|
3810
|
+
const entries = await store.load("fault-repair-verdict-reserve");
|
|
3811
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3812
|
+
const message = outcome.error?.message ?? "";
|
|
3813
|
+
const holdNamed = message.includes("held convergence reserve 0.2000");
|
|
3814
|
+
return {
|
|
3815
|
+
observation: {
|
|
3816
|
+
matched: outcome.status === "error" && message.includes("could not dispatch") && holdNamed && data.source === "orchestrator_claim_consistency" && data.roundDispatched === false && data.repairsUsed === 0 && data.claimConsistencyMeta?.findings === 1 && compositions.length === 1 && judges.length === 1,
|
|
3817
|
+
detail: `run '${outcome.status}': roundDispatched=${String(data.roundDispatched)}, repairsUsed=${String(data.repairsUsed)}, held convergence reserve named=${String(holdNamed)}; ${String(compositions.length)} composition(s) paid, ${String(judges.length)} final judge pass(es)`
|
|
3818
|
+
},
|
|
3819
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3820
|
+
status: outcome.status,
|
|
3821
|
+
error: outcome.error ?? null
|
|
3822
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3823
|
+
};
|
|
3824
|
+
}
|
|
3825
|
+
};
|
|
3826
|
+
/**
|
|
3827
|
+
* The mechanical leg of the convergence hold (RV3802): RV3701 holds
|
|
3828
|
+
* the verdict money and RV3602 gives the round its own repair pool,
|
|
3829
|
+
* but the one mechanical repair turn that pool can grant was funded by
|
|
3830
|
+
* nothing. This drives the ceiling where the round could pay its
|
|
3831
|
+
* composition and its verdict but not the granted repair: the refusal
|
|
3832
|
+
* fires pre dispatch with BOTH legs named in the printed arithmetic.
|
|
3833
|
+
*/
|
|
3834
|
+
const repairRoundMechanicalReserve = {
|
|
3835
|
+
name: "repair-round-mechanical-reserve",
|
|
3836
|
+
doctrine: "the repair round pays ahead for its one mechanical repair turn too (RV3802): the declared finishValidation.estRepairCostUsd is held beside the verdict money from the moment the round is admitted, a round that could seat only its composition and verdict refuses through the honest pre dispatch class before any of its wires, with the held convergence reserve AND the held repair reserve both named in the refusal arithmetic, and exactly one composition ever paid for",
|
|
3837
|
+
async run() {
|
|
3838
|
+
const { engine, store } = tailEngine(tailAdapter({
|
|
3839
|
+
judge: () => TAIL_FINDS,
|
|
3840
|
+
finals: [TAIL_FINAL_INVERTED]
|
|
3841
|
+
}));
|
|
3842
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3843
|
+
...TAIL_OPTS,
|
|
3844
|
+
synthesis: {
|
|
3845
|
+
limits: { maxTurns: 3 },
|
|
3846
|
+
estCost: .2
|
|
3847
|
+
},
|
|
3848
|
+
budget: {
|
|
3849
|
+
capUsd: .5,
|
|
3850
|
+
capFraction: 1
|
|
3851
|
+
},
|
|
3852
|
+
claimConsistency: {
|
|
3853
|
+
stage: "final",
|
|
3854
|
+
onFound: "repair",
|
|
3855
|
+
judge: { estCost: .2 }
|
|
3856
|
+
},
|
|
3857
|
+
finishValidation: {
|
|
3858
|
+
validators: [minMatchesValidator({
|
|
3859
|
+
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3860
|
+
min: 1,
|
|
3861
|
+
name: "provenance-anchor"
|
|
3862
|
+
})],
|
|
3863
|
+
maxRepairs: 1,
|
|
3864
|
+
estRepairCostUsd: .15
|
|
3865
|
+
}
|
|
3866
|
+
}), void 0, {
|
|
3867
|
+
runId: "fault-repair-mechanical-reserve",
|
|
3868
|
+
budgetUsd: 10
|
|
3869
|
+
}).result;
|
|
3870
|
+
const data = outcome.error?.data ?? {};
|
|
3871
|
+
const entries = await store.load("fault-repair-mechanical-reserve");
|
|
3872
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3873
|
+
const message = outcome.error?.message ?? "";
|
|
3874
|
+
const verdictNamed = message.includes("held convergence reserve 0.2000");
|
|
3875
|
+
const mechanicalNamed = message.includes("held repair reserve 0.1500");
|
|
3876
|
+
return {
|
|
3877
|
+
observation: {
|
|
3878
|
+
matched: outcome.status === "error" && message.includes("could not dispatch") && verdictNamed && mechanicalNamed && data.source === "orchestrator_claim_consistency" && data.roundDispatched === false && data.repairsUsed === 0 && data.claimConsistencyMeta?.findings === 1 && compositions.length === 1 && judges.length === 1,
|
|
3879
|
+
detail: `run '${outcome.status}': roundDispatched=${String(data.roundDispatched)}, repairsUsed=${String(data.repairsUsed)}, held convergence reserve named=${String(verdictNamed)}, held repair reserve named=${String(mechanicalNamed)}; ${String(compositions.length)} composition(s) paid, ${String(judges.length)} final judge pass(es)`
|
|
3880
|
+
},
|
|
3881
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3882
|
+
status: outcome.status,
|
|
3883
|
+
error: outcome.error ?? null
|
|
3884
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3885
|
+
};
|
|
3886
|
+
}
|
|
3887
|
+
};
|
|
3888
|
+
const SECTIONAL_PREFIX = "# Audit\n\n## Fine\n\nThe ledger holds one denominator.\n\n";
|
|
3889
|
+
const SECTIONAL_FINAL_INVERTED = `${SECTIONAL_PREFIX}## Verdict\n\nfinal: an audit-write failure does not turn success into failure [src/exec.ts:256-296].
|
|
3890
|
+
`;
|
|
3891
|
+
const SECTIONAL_VERDICT_BODY = "final: a failed audit write does not mask success [src/exec.ts:256-296].\n";
|
|
3892
|
+
const SECTIONAL_SPLICED = `${SECTIONAL_PREFIX}## Verdict\n${SECTIONAL_VERDICT_BODY}`;
|
|
3893
|
+
const sectionalRepairRound = {
|
|
3894
|
+
name: "sectional-repair-round",
|
|
3895
|
+
doctrine: "the bounded claim repair round repairs ONLY the sections owning the judged findings (RV3803): the round prompt retains the accepted document and names the target sections, the model resubmits one section body, the host splices it into the retained document with every other byte identical, and the FULL validator set plus the final judge rule on the spliced whole; two compositions, two judge passes, no mechanical repair spent",
|
|
3896
|
+
async run() {
|
|
3897
|
+
let judgeCalls = 0;
|
|
3898
|
+
let synthCalls = 0;
|
|
3899
|
+
let loopTurns = 0;
|
|
3900
|
+
const adapter = new FakeAdapter({ agents: {
|
|
3901
|
+
"claim-consistency-judge-final": () => (judgeCalls += 1) === 1 ? TAIL_FINDS : TAIL_AGREES,
|
|
3902
|
+
"final-composition": () => (synthCalls += 1) === 1 ? fakeToolCalls({
|
|
3903
|
+
name: "finish",
|
|
3904
|
+
args: { result: SECTIONAL_FINAL_INVERTED }
|
|
3905
|
+
}) : fakeToolCalls({
|
|
3906
|
+
name: "finish",
|
|
3907
|
+
args: { sections: { "## Verdict": SECTIONAL_VERDICT_BODY } }
|
|
3908
|
+
}),
|
|
3909
|
+
worker: TAIL_POOL_READING,
|
|
3910
|
+
"*": (call) => {
|
|
3911
|
+
loopTurns += 1;
|
|
3912
|
+
if (loopTurns === 1) return fakeToolCalls({
|
|
3913
|
+
name: "spawn_agent",
|
|
3914
|
+
args: {
|
|
3915
|
+
agentType: "worker",
|
|
3916
|
+
prompt: "read the recorded span"
|
|
3917
|
+
}
|
|
3918
|
+
});
|
|
3919
|
+
if (loopTurns === 2) return fakeToolCalls({
|
|
3920
|
+
name: "await_all",
|
|
3921
|
+
args: { handles: tailHandles(call.req) }
|
|
3922
|
+
});
|
|
3923
|
+
return fakeToolCalls({
|
|
3924
|
+
name: "finish",
|
|
3925
|
+
args: { result: TAIL_DRAFT_INVERTED }
|
|
3926
|
+
});
|
|
3927
|
+
}
|
|
3928
|
+
} });
|
|
3929
|
+
const { engine, store } = tailEngine(adapter);
|
|
3930
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3931
|
+
...TAIL_OPTS,
|
|
3932
|
+
claimConsistency: {
|
|
3933
|
+
stage: "final",
|
|
3934
|
+
onFound: "repair"
|
|
3935
|
+
},
|
|
3936
|
+
finishValidation: {
|
|
3937
|
+
validators: [minMatchesValidator({
|
|
3938
|
+
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3939
|
+
min: 1,
|
|
3940
|
+
name: "provenance-anchor"
|
|
3941
|
+
})],
|
|
3942
|
+
maxRepairs: 1
|
|
3943
|
+
}
|
|
3944
|
+
}), void 0, {
|
|
3945
|
+
runId: "fault-sectional-round",
|
|
3946
|
+
budgetUsd: 10
|
|
3947
|
+
}).result;
|
|
3948
|
+
const value = outcome.value;
|
|
3949
|
+
const entries = await store.load("fault-sectional-round");
|
|
3950
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3951
|
+
const verdictRows = entries.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finish_validation").map((entry) => entry.value);
|
|
3952
|
+
const roundPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
|
|
3953
|
+
const sectionalPrompt = roundPrompts.length === 2 && roundPrompts[0]?.includes("SECTIONAL ROUND") === false && roundPrompts[1]?.includes("SECTIONAL ROUND") === true && roundPrompts[1]?.includes("\"## Verdict\"") === true;
|
|
3954
|
+
const byteIdentity = value?.result === SECTIONAL_SPLICED && String(value.result).startsWith(SECTIONAL_PREFIX);
|
|
3955
|
+
return {
|
|
3956
|
+
observation: {
|
|
3957
|
+
matched: outcome.status === "ok" && byteIdentity && value.claimConsistencyMeta?.findings === 0 && verdictRows.map((row) => row.verdict).join(",") === "accepted,accepted" && verdictRows.map((row) => String(row.repairsUsed)).join(",") === "0,0" && sectionalPrompt && compositions.length === 2 && judges.length === 2,
|
|
3958
|
+
detail: `run '${outcome.status}' shipped ${value?.result === SECTIONAL_SPLICED ? "the spliced whole" : "an unexpected document"}; byte identity=${String(byteIdentity)}; sectional prompt=${String(sectionalPrompt)}; verdicts [${verdictRows.map((row) => row.verdict).join(",")}]; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3959
|
+
},
|
|
3960
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
3961
|
+
status: outcome.status,
|
|
3962
|
+
value: outcome.value ?? null,
|
|
3963
|
+
envelope: outcome.envelope
|
|
3964
|
+
}), jsonArtifact("journal.json", entries)]
|
|
3965
|
+
};
|
|
3966
|
+
}
|
|
3967
|
+
};
|
|
3968
|
+
const sectionalRepairRoundFallback = {
|
|
3969
|
+
name: "sectional-repair-round-fallback",
|
|
3970
|
+
doctrine: "a round whose plan cannot be exact regenerates in full, byte for byte the historical round (RV3803): a document without H2 headings arms no sectional context, the round prompt carries neither the retained document nor a target list, and the full regeneration converges exactly as before",
|
|
3971
|
+
async run() {
|
|
3972
|
+
let judgeCalls = 0;
|
|
3973
|
+
const adapter = tailAdapter({
|
|
3974
|
+
judge: () => (judgeCalls += 1) === 1 ? TAIL_FINDS : TAIL_AGREES,
|
|
3975
|
+
finals: [TAIL_FINAL_INVERTED, TAIL_FINAL_CLEAN]
|
|
3976
|
+
});
|
|
3977
|
+
const { engine, store } = tailEngine(adapter);
|
|
3978
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3979
|
+
...TAIL_OPTS,
|
|
3980
|
+
claimConsistency: {
|
|
3981
|
+
stage: "final",
|
|
3982
|
+
onFound: "repair"
|
|
3983
|
+
},
|
|
3984
|
+
finishValidation: {
|
|
3985
|
+
validators: [minMatchesValidator({
|
|
3986
|
+
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3987
|
+
min: 1,
|
|
3988
|
+
name: "provenance-anchor"
|
|
3989
|
+
})],
|
|
3990
|
+
maxRepairs: 1
|
|
3991
|
+
}
|
|
3992
|
+
}), void 0, {
|
|
3993
|
+
runId: "fault-sectional-fallback",
|
|
3994
|
+
budgetUsd: 10
|
|
3995
|
+
}).result;
|
|
3996
|
+
const value = outcome.value;
|
|
3997
|
+
const entries = await store.load("fault-sectional-fallback");
|
|
3998
|
+
const { compositions, judges } = tailSpans(entries);
|
|
3999
|
+
const roundPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
|
|
4000
|
+
const fullRegeneration = roundPrompts.length === 2 && roundPrompts[1]?.includes("CLAIM CONTRADICTIONS") === true && roundPrompts[1]?.includes("SECTIONAL ROUND") === false && roundPrompts[1]?.includes("RETAINED FINAL") === false;
|
|
4001
|
+
return {
|
|
4002
|
+
observation: {
|
|
4003
|
+
matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && fullRegeneration && compositions.length === 2 && judges.length === 2,
|
|
4004
|
+
detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the regenerated document" : "an unexpected document"}; full regeneration=${String(fullRegeneration)} (sectional block absent); ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
4005
|
+
},
|
|
4006
|
+
artifacts: [jsonArtifact("outcome.json", {
|
|
4007
|
+
status: outcome.status,
|
|
4008
|
+
value: outcome.value ?? null,
|
|
4009
|
+
envelope: outcome.envelope
|
|
4010
|
+
}), jsonArtifact("journal.json", entries)]
|
|
4011
|
+
};
|
|
4012
|
+
}
|
|
4013
|
+
};
|
|
4014
|
+
/**
|
|
4015
|
+
* The armed posture doctrine on a dead judge (RV3307): a gate armed to
|
|
4016
|
+
* stop or to repair must not pass silently when its judge cannot rule.
|
|
4017
|
+
* Both armed postures refuse typed; the round never runs.
|
|
4018
|
+
*/
|
|
4019
|
+
const claimJudgeDeadArmedRefusal = {
|
|
4020
|
+
name: "claim-judge-dead-armed-refusal",
|
|
4021
|
+
doctrine: "a final stage judge that dies on the wire under an armed posture fails the run typed for both 'fail' and 'repair' (RV3307), each message naming its armed posture, with exactly one composition paid and no repair round dispatched, never a silent settle over findings nobody ruled on",
|
|
4022
|
+
async run() {
|
|
4023
|
+
const judgeDeath = () => fakeWireError({
|
|
4024
|
+
code: "agent",
|
|
4025
|
+
message: "the judge died mid stream",
|
|
4026
|
+
retryable: false,
|
|
4027
|
+
data: {}
|
|
4028
|
+
});
|
|
4029
|
+
const runs = [];
|
|
4030
|
+
const journals = {};
|
|
4031
|
+
for (const posture of ["fail", "repair"]) {
|
|
4032
|
+
const { engine, store } = tailEngine(tailAdapter({
|
|
4033
|
+
judge: judgeDeath,
|
|
4034
|
+
finals: [TAIL_FINAL_INVERTED]
|
|
4035
|
+
}));
|
|
4036
|
+
const runId = `fault-dead-judge-${posture}`;
|
|
4037
|
+
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
4038
|
+
...TAIL_OPTS,
|
|
4039
|
+
claimConsistency: {
|
|
4040
|
+
stage: "final",
|
|
4041
|
+
onFound: posture
|
|
4042
|
+
}
|
|
4043
|
+
}), void 0, {
|
|
4044
|
+
runId,
|
|
4045
|
+
budgetUsd: 10
|
|
4046
|
+
}).result;
|
|
4047
|
+
const entries = await store.load(runId);
|
|
4048
|
+
const { compositions, judges } = tailSpans(entries);
|
|
4049
|
+
journals[posture] = entries;
|
|
4050
|
+
runs.push({
|
|
4051
|
+
posture,
|
|
4052
|
+
status: outcome.status,
|
|
4053
|
+
message: outcome.error?.message ?? "",
|
|
4054
|
+
compositions: compositions.length,
|
|
4055
|
+
judges: judges.length
|
|
4056
|
+
});
|
|
4057
|
+
}
|
|
4058
|
+
return {
|
|
4059
|
+
observation: {
|
|
4060
|
+
matched: runs.every((run) => run.status === "error" && run.message.includes(`armed ${run.posture} posture`) && run.compositions === 1 && run.judges === 1),
|
|
4061
|
+
detail: runs.map((run) => `${run.posture}: '${run.status}' (${run.compositions} composition(s), ${run.judges} judge span(s)) ${run.message.slice(0, 120)}`).join(" | ")
|
|
4062
|
+
},
|
|
4063
|
+
artifacts: [jsonArtifact("runs.json", runs), jsonArtifact("journals.json", journals)]
|
|
4064
|
+
};
|
|
4065
|
+
}
|
|
4066
|
+
};
|
|
4067
|
+
/**
|
|
4068
|
+
* The deterministic provenance patch (RV3801): the third comparison
|
|
4069
|
+
* run died twice on a failure class whose remedy the evidence-grade
|
|
4070
|
+
* verdict prescribes word for word (write this run's id inside each
|
|
4071
|
+
* offending sentence). The finish loop now performs that prescription
|
|
4072
|
+
* host side; this drives the arc on the real engine in the adversarial
|
|
4073
|
+
* shape the patch must never mask: a POSITIVE production claim that
|
|
4074
|
+
* inverts the pool reading, healed into mechanical validity by the
|
|
4075
|
+
* inserted id, then caught semantically by the final judge.
|
|
4076
|
+
*/
|
|
4077
|
+
const PATCH_RUN_ID = "fault-deterministic-patch";
|
|
4078
|
+
const PATCH_FINAL_INVERTED_GRADED = "final: an audit-write failure does not turn success into failure and the fix is production-proven [src/exec.ts:256-296].";
|
|
3580
4079
|
const SCENARIOS = [
|
|
3581
4080
|
inFlightExposure,
|
|
3582
4081
|
duplicateQuotaRule,
|
|
@@ -3605,134 +4104,22 @@ const SCENARIOS = [
|
|
|
3605
4104
|
parityReserveLineRedemption,
|
|
3606
4105
|
resumeSpawnFamine,
|
|
3607
4106
|
validatorGuidanceConflict,
|
|
4107
|
+
repairRoundHonesty,
|
|
4108
|
+
repairSurvivorRefusal,
|
|
4109
|
+
repairRoundHostRejection,
|
|
4110
|
+
repairRoundOwnPool,
|
|
4111
|
+
repairRoundVerdictReserve,
|
|
4112
|
+
repairRoundMechanicalReserve,
|
|
4113
|
+
sectionalRepairRound,
|
|
4114
|
+
sectionalRepairRoundFallback,
|
|
3608
4115
|
{
|
|
3609
|
-
name: "
|
|
3610
|
-
doctrine: "the
|
|
3611
|
-
async run() {
|
|
3612
|
-
const { engine, store } = tailEngine(tailAdapter({
|
|
3613
|
-
judge: (call) => call === 1 ? TAIL_FINDS : TAIL_AGREES,
|
|
3614
|
-
finals: [TAIL_FINAL_INVERTED, TAIL_FINAL_CLEAN]
|
|
3615
|
-
}));
|
|
3616
|
-
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3617
|
-
...TAIL_OPTS,
|
|
3618
|
-
claimConsistency: {
|
|
3619
|
-
stage: "final",
|
|
3620
|
-
onFound: "repair"
|
|
3621
|
-
}
|
|
3622
|
-
}), void 0, {
|
|
3623
|
-
runId: "fault-repair-honesty",
|
|
3624
|
-
budgetUsd: 10
|
|
3625
|
-
}).result;
|
|
3626
|
-
const value = outcome.value;
|
|
3627
|
-
const entries = await store.load("fault-repair-honesty");
|
|
3628
|
-
const { compositions, judges } = tailSpans(entries);
|
|
3629
|
-
const invoice = invoiceFromJournal(entries, () => 0);
|
|
3630
|
-
return {
|
|
3631
|
-
observation: {
|
|
3632
|
-
matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && Array.isArray(value.claimContradictions) && value.claimContradictions.length === 0 && value.claimConsistencyMeta?.judgedStage === "final" && value.claimConsistencyMeta.findings === 0 && typeof value.claimConsistencyMeta.judgedHash === "string" && value.claimConsistencyMeta.judgedHash === value.draftToFinal?.finalHash && compositions.length === 2 && judges.length === 2 && [...compositions, ...judges].every((entry) => entry.status === "ok") && typeof outcome.envelope.wireRequests === "number" && outcome.envelope.wireRequests === invoice.cardinality.wireRequests,
|
|
3633
|
-
detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the repaired document" : "an unexpected document"}; meta judgedStage='${String(value?.claimConsistencyMeta?.judgedStage)}' findings=${String(value?.claimConsistencyMeta?.findings)} judgedHash==finalHash=${String(value?.claimConsistencyMeta?.judgedHash === value?.draftToFinal?.finalHash)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es); wires ${String(outcome.envelope.wireRequests)} == invoice ` + String(invoice.cardinality.wireRequests)
|
|
3634
|
-
},
|
|
3635
|
-
artifacts: [jsonArtifact("outcome.json", {
|
|
3636
|
-
status: outcome.status,
|
|
3637
|
-
value: outcome.value ?? null,
|
|
3638
|
-
envelope: outcome.envelope
|
|
3639
|
-
}), jsonArtifact("journal.json", entries)]
|
|
3640
|
-
};
|
|
3641
|
-
}
|
|
3642
|
-
},
|
|
3643
|
-
{
|
|
3644
|
-
name: "repair-survivor-refusal",
|
|
3645
|
-
doctrine: "findings that survive the bounded repair round fail the run typed (RV3307): source 'orchestrator_claim_consistency', repairsUsed 1, preRepairHash distinct from repairedHash (the round demonstrably produced a different document and the judge still refused it), two compositions and two final judge passes paid, never a silent ok over the surviving contradiction",
|
|
3646
|
-
async run() {
|
|
3647
|
-
const { engine, store } = tailEngine(tailAdapter({
|
|
3648
|
-
judge: () => TAIL_FINDS,
|
|
3649
|
-
finals: [TAIL_FINAL_INVERTED, TAIL_FINAL_STILL_INVERTED]
|
|
3650
|
-
}));
|
|
3651
|
-
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3652
|
-
...TAIL_OPTS,
|
|
3653
|
-
claimConsistency: {
|
|
3654
|
-
stage: "final",
|
|
3655
|
-
onFound: "repair"
|
|
3656
|
-
}
|
|
3657
|
-
}), void 0, {
|
|
3658
|
-
runId: "fault-repair-survivor",
|
|
3659
|
-
budgetUsd: 10
|
|
3660
|
-
}).result;
|
|
3661
|
-
const data = outcome.error?.data ?? {};
|
|
3662
|
-
const entries = await store.load("fault-repair-survivor");
|
|
3663
|
-
const { compositions, judges } = tailSpans(entries);
|
|
3664
|
-
return {
|
|
3665
|
-
observation: {
|
|
3666
|
-
matched: outcome.status === "error" && (outcome.error?.message ?? "").includes("after the bounded repair round") && data.source === "orchestrator_claim_consistency" && data.repairsUsed === 1 && typeof data.preRepairHash === "string" && typeof data.repairedHash === "string" && data.preRepairHash !== data.repairedHash && compositions.length === 2 && judges.length === 2,
|
|
3667
|
-
detail: `run '${outcome.status}': ${outcome.error?.message ?? ""}; repairsUsed=${String(data.repairsUsed)}, hashes distinct=${String(data.preRepairHash !== data.repairedHash)}, ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3668
|
-
},
|
|
3669
|
-
artifacts: [jsonArtifact("outcome.json", {
|
|
3670
|
-
status: outcome.status,
|
|
3671
|
-
error: outcome.error ?? null
|
|
3672
|
-
}), jsonArtifact("journal.json", entries)]
|
|
3673
|
-
};
|
|
3674
|
-
}
|
|
3675
|
-
},
|
|
3676
|
-
{
|
|
3677
|
-
name: "repair-round-host-rejection",
|
|
3678
|
-
doctrine: "a repair round that dispatched and lost its candidate to the finish contract fails typed as a host rejection (RV3601): the message names the dispatch, never 'could not dispatch'; data carries roundDispatched true, repairsUsed 1, the judge meta beside the findings, and the finish verdict verbatim (failed validators, candidateHash, candidateChars); the outcome lifts the findings beside the meta",
|
|
3679
|
-
async run() {
|
|
3680
|
-
const { engine, store } = tailEngine(tailAdapter({
|
|
3681
|
-
judge: () => TAIL_FINDS,
|
|
3682
|
-
finals: [
|
|
3683
|
-
TAIL_FINAL_INVERTED,
|
|
3684
|
-
TAIL_FINAL_UNGROUNDED,
|
|
3685
|
-
TAIL_FINAL_UNGROUNDED
|
|
3686
|
-
]
|
|
3687
|
-
}));
|
|
3688
|
-
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3689
|
-
...TAIL_OPTS,
|
|
3690
|
-
claimConsistency: {
|
|
3691
|
-
stage: "final",
|
|
3692
|
-
onFound: "repair"
|
|
3693
|
-
},
|
|
3694
|
-
finishValidation: {
|
|
3695
|
-
validators: [minMatchesValidator({
|
|
3696
|
-
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3697
|
-
min: 1,
|
|
3698
|
-
name: "provenance-anchor"
|
|
3699
|
-
})],
|
|
3700
|
-
maxRepairs: 1
|
|
3701
|
-
}
|
|
3702
|
-
}), void 0, {
|
|
3703
|
-
runId: "fault-repair-host-rejection",
|
|
3704
|
-
budgetUsd: 10
|
|
3705
|
-
}).result;
|
|
3706
|
-
const data = outcome.error?.data ?? {};
|
|
3707
|
-
const entries = await store.load("fault-repair-host-rejection");
|
|
3708
|
-
const { compositions, judges } = tailSpans(entries);
|
|
3709
|
-
const liftedFindings = outcome.claimContradictions;
|
|
3710
|
-
const message = outcome.error?.message ?? "";
|
|
3711
|
-
return {
|
|
3712
|
-
observation: {
|
|
3713
|
-
matched: outcome.status === "error" && message.includes("dispatched and its repaired candidate failed host validation") && !message.includes("could not dispatch") && data.source === "orchestrator_claim_consistency" && data.roundDispatched === true && data.repairsUsed === 1 && data.claimConsistencyMeta?.findings === 1 && Array.isArray(data.claimContradictions) && data.claimContradictions.length === 1 && typeof data.finishValidation?.callId === "string" && data.finishValidation.repairsUsed === 1 && data.finishValidation.maxRepairs === 1 && typeof data.finishValidation.candidateHash === "string" && typeof data.finishValidation.candidateChars === "number" && JSON.stringify(data.finishValidation.failed ?? null).includes("provenance-anchor") && Array.isArray(liftedFindings) && liftedFindings.length === 1 && compositions.length === 2 && judges.length === 1,
|
|
3714
|
-
detail: `run '${outcome.status}': roundDispatched=${String(data.roundDispatched)}, repairsUsed=${String(data.repairsUsed)}, meta findings=${String(data.claimConsistencyMeta?.findings)}, finish verdict carries hash=${String(typeof data.finishValidation?.candidateHash === "string")} chars=${String(data.finishValidation?.candidateChars)}, outcome lifts findings=${String(Array.isArray(liftedFindings))}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3715
|
-
},
|
|
3716
|
-
artifacts: [jsonArtifact("outcome.json", {
|
|
3717
|
-
status: outcome.status,
|
|
3718
|
-
error: outcome.error ?? null
|
|
3719
|
-
}), jsonArtifact("journal.json", entries)]
|
|
3720
|
-
};
|
|
3721
|
-
}
|
|
3722
|
-
},
|
|
3723
|
-
{
|
|
3724
|
-
name: "repair-round-own-pool",
|
|
3725
|
-
doctrine: "the mechanical repair pool belongs to one composition invocation (RV3602): the bounded claim repair round enters with the full maxRepairs even after the initial composition spent its own, so the frozen third comparison sequence converges to ok/complete with verdicts repair/accepted twice and repairsUsed restarting at the invocation boundary, never a host rejection born of an inherited spent pool; the round's prompt also carries the HOST VALIDATION LESSONS block naming the failure the run already paid for (RV3603), while the initial composition carries none",
|
|
4116
|
+
name: "deterministic-provenance-patch",
|
|
4117
|
+
doctrine: "the finish loop performs the evidence-grade prescription host side (RV3801): a candidate whose every failure carries repair hints is patched without a provider wire and accepted with the mechanical pool untouched, the healed failure still teaches the repair round through HOST VALIDATION LESSONS, and the claim judge rules on the PATCHED document, so an inserted id satisfies provenance mechanics but never masks a false positive production claim",
|
|
3726
4118
|
async run() {
|
|
3727
4119
|
let judgeCalls = 0;
|
|
3728
4120
|
const adapter = tailAdapter({
|
|
3729
4121
|
judge: () => (judgeCalls += 1) === 1 ? TAIL_FINDS : TAIL_AGREES,
|
|
3730
|
-
finals: [
|
|
3731
|
-
TAIL_FINAL_UNGROUNDED_INITIAL,
|
|
3732
|
-
TAIL_FINAL_INVERTED,
|
|
3733
|
-
TAIL_FINAL_UNGROUNDED,
|
|
3734
|
-
TAIL_FINAL_CLEAN
|
|
3735
|
-
]
|
|
4122
|
+
finals: [PATCH_FINAL_INVERTED_GRADED, TAIL_FINAL_CLEAN]
|
|
3736
4123
|
});
|
|
3737
4124
|
const { engine, store } = tailEngine(adapter);
|
|
3738
4125
|
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
@@ -3742,29 +4129,27 @@ const SCENARIOS = [
|
|
|
3742
4129
|
onFound: "repair"
|
|
3743
4130
|
},
|
|
3744
4131
|
finishValidation: {
|
|
3745
|
-
validators: [
|
|
3746
|
-
pattern: "src/[a-z]+\\.ts:\\d+",
|
|
3747
|
-
min: 1,
|
|
3748
|
-
name: "provenance-anchor"
|
|
3749
|
-
})],
|
|
4132
|
+
validators: [evidenceGradeValidator({ artifactPattern: "run[ -]?[0-9A-HJKMNP-TV-Z]{6,26}" })],
|
|
3750
4133
|
maxRepairs: 1
|
|
3751
4134
|
}
|
|
3752
4135
|
}), void 0, {
|
|
3753
|
-
runId:
|
|
4136
|
+
runId: PATCH_RUN_ID,
|
|
3754
4137
|
budgetUsd: 10
|
|
3755
4138
|
}).result;
|
|
3756
4139
|
const value = outcome.value;
|
|
3757
|
-
const entries = await store.load(
|
|
4140
|
+
const entries = await store.load(PATCH_RUN_ID);
|
|
3758
4141
|
const { compositions, judges } = tailSpans(entries);
|
|
3759
4142
|
const verdictRows = entries.filter((entry) => entry.kind === "decision" && entry.value?.decisionType === "orchestrator_finish_validation").map((entry) => entry.value);
|
|
3760
4143
|
const verdicts = verdictRows.map((row) => row.verdict).join(",");
|
|
3761
4144
|
const pools = verdictRows.map((row) => String(row.repairsUsed)).join(",");
|
|
3762
|
-
const
|
|
3763
|
-
const
|
|
4145
|
+
const patch = verdictRows[0]?.deterministicRepair;
|
|
4146
|
+
const judgeSawPatched = adapter.calls.filter((call) => call.label === "claim-consistency-judge-final").map((call) => call.prompt)[0]?.includes(`(run ${PATCH_RUN_ID})`) === true;
|
|
4147
|
+
const roundPrompts = adapter.calls.filter((call) => call.label === "final-composition").map((call) => call.prompt);
|
|
4148
|
+
const lessonCarried = roundPrompts.length === 2 && roundPrompts[0]?.includes("HOST VALIDATION LESSONS") === false && roundPrompts[1]?.includes("HOST VALIDATION LESSONS") === true && roundPrompts[1]?.includes("evidence-grade") === true;
|
|
3764
4149
|
return {
|
|
3765
4150
|
observation: {
|
|
3766
|
-
matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && value.claimConsistencyMeta?.findings === 0 && verdicts === "
|
|
3767
|
-
detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the
|
|
4151
|
+
matched: outcome.status === "ok" && value?.result === TAIL_FINAL_CLEAN && value.claimConsistencyMeta?.findings === 0 && verdicts === "accepted,accepted" && pools === "0,0" && patch?.outcome === "accepted" && patch.healed?.map((healed) => healed.name).join(",") === "evidence-grade" && judgeSawPatched && lessonCarried && compositions.length === 2 && judges.length === 2,
|
|
4152
|
+
detail: `run '${outcome.status}' shipped ${value?.result === TAIL_FINAL_CLEAN ? "the clean round document" : "an unexpected document"}; verdicts [${verdicts}], repairsUsed [${pools}] (no wire, no pool spent); patch outcome='${String(patch?.outcome)}'; judge saw patched=${String(judgeSawPatched)}; lesson carried=${String(lessonCarried)}; ${String(compositions.length)} composition(s), ${String(judges.length)} final judge pass(es)`
|
|
3768
4153
|
},
|
|
3769
4154
|
artifacts: [jsonArtifact("outcome.json", {
|
|
3770
4155
|
status: outcome.status,
|
|
@@ -3774,98 +4159,7 @@ const SCENARIOS = [
|
|
|
3774
4159
|
};
|
|
3775
4160
|
}
|
|
3776
4161
|
},
|
|
3777
|
-
|
|
3778
|
-
name: "repair-round-verdict-reserve",
|
|
3779
|
-
doctrine: "the repair round pays for its verdict up front (RV3701): the money of the round second judge pass (the declared judge estCost first, else the run own observed post draft judge price) is held from the moment the round is admitted, a round the budget can only start refuses through the honest pre dispatch class before any of its wires with the held convergence reserve named in the refusal arithmetic, and exactly one composition is ever paid for",
|
|
3780
|
-
async run() {
|
|
3781
|
-
const { engine, store } = tailEngine(tailAdapter({
|
|
3782
|
-
judge: () => TAIL_FINDS,
|
|
3783
|
-
finals: [TAIL_FINAL_INVERTED]
|
|
3784
|
-
}));
|
|
3785
|
-
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3786
|
-
...TAIL_OPTS,
|
|
3787
|
-
synthesis: {
|
|
3788
|
-
limits: { maxTurns: 3 },
|
|
3789
|
-
estCost: .2
|
|
3790
|
-
},
|
|
3791
|
-
budget: {
|
|
3792
|
-
capUsd: .3,
|
|
3793
|
-
capFraction: 1
|
|
3794
|
-
},
|
|
3795
|
-
claimConsistency: {
|
|
3796
|
-
stage: "final",
|
|
3797
|
-
onFound: "repair",
|
|
3798
|
-
judge: { estCost: .2 }
|
|
3799
|
-
}
|
|
3800
|
-
}), void 0, {
|
|
3801
|
-
runId: "fault-repair-verdict-reserve",
|
|
3802
|
-
budgetUsd: 10
|
|
3803
|
-
}).result;
|
|
3804
|
-
const data = outcome.error?.data ?? {};
|
|
3805
|
-
const entries = await store.load("fault-repair-verdict-reserve");
|
|
3806
|
-
const { compositions, judges } = tailSpans(entries);
|
|
3807
|
-
const message = outcome.error?.message ?? "";
|
|
3808
|
-
const holdNamed = message.includes("held convergence reserve 0.2000");
|
|
3809
|
-
return {
|
|
3810
|
-
observation: {
|
|
3811
|
-
matched: outcome.status === "error" && message.includes("could not dispatch") && holdNamed && data.source === "orchestrator_claim_consistency" && data.roundDispatched === false && data.repairsUsed === 0 && data.claimConsistencyMeta?.findings === 1 && compositions.length === 1 && judges.length === 1,
|
|
3812
|
-
detail: `run '${outcome.status}': roundDispatched=${String(data.roundDispatched)}, repairsUsed=${String(data.repairsUsed)}, held convergence reserve named=${String(holdNamed)}; ${String(compositions.length)} composition(s) paid, ${String(judges.length)} final judge pass(es)`
|
|
3813
|
-
},
|
|
3814
|
-
artifacts: [jsonArtifact("outcome.json", {
|
|
3815
|
-
status: outcome.status,
|
|
3816
|
-
error: outcome.error ?? null
|
|
3817
|
-
}), jsonArtifact("journal.json", entries)]
|
|
3818
|
-
};
|
|
3819
|
-
}
|
|
3820
|
-
},
|
|
3821
|
-
{
|
|
3822
|
-
name: "claim-judge-dead-armed-refusal",
|
|
3823
|
-
doctrine: "a final stage judge that dies on the wire under an armed posture fails the run typed for both 'fail' and 'repair' (RV3307), each message naming its armed posture, with exactly one composition paid and no repair round dispatched, never a silent settle over findings nobody ruled on",
|
|
3824
|
-
async run() {
|
|
3825
|
-
const judgeDeath = () => fakeWireError({
|
|
3826
|
-
code: "agent",
|
|
3827
|
-
message: "the judge died mid stream",
|
|
3828
|
-
retryable: false,
|
|
3829
|
-
data: {}
|
|
3830
|
-
});
|
|
3831
|
-
const runs = [];
|
|
3832
|
-
const journals = {};
|
|
3833
|
-
for (const posture of ["fail", "repair"]) {
|
|
3834
|
-
const { engine, store } = tailEngine(tailAdapter({
|
|
3835
|
-
judge: judgeDeath,
|
|
3836
|
-
finals: [TAIL_FINAL_INVERTED]
|
|
3837
|
-
}));
|
|
3838
|
-
const runId = `fault-dead-judge-${posture}`;
|
|
3839
|
-
const outcome = await engine.run(makeOrchestratorWorkflow("audit the executor", {
|
|
3840
|
-
...TAIL_OPTS,
|
|
3841
|
-
claimConsistency: {
|
|
3842
|
-
stage: "final",
|
|
3843
|
-
onFound: posture
|
|
3844
|
-
}
|
|
3845
|
-
}), void 0, {
|
|
3846
|
-
runId,
|
|
3847
|
-
budgetUsd: 10
|
|
3848
|
-
}).result;
|
|
3849
|
-
const entries = await store.load(runId);
|
|
3850
|
-
const { compositions, judges } = tailSpans(entries);
|
|
3851
|
-
journals[posture] = entries;
|
|
3852
|
-
runs.push({
|
|
3853
|
-
posture,
|
|
3854
|
-
status: outcome.status,
|
|
3855
|
-
message: outcome.error?.message ?? "",
|
|
3856
|
-
compositions: compositions.length,
|
|
3857
|
-
judges: judges.length
|
|
3858
|
-
});
|
|
3859
|
-
}
|
|
3860
|
-
return {
|
|
3861
|
-
observation: {
|
|
3862
|
-
matched: runs.every((run) => run.status === "error" && run.message.includes(`armed ${run.posture} posture`) && run.compositions === 1 && run.judges === 1),
|
|
3863
|
-
detail: runs.map((run) => `${run.posture}: '${run.status}' (${run.compositions} composition(s), ${run.judges} judge span(s)) ${run.message.slice(0, 120)}`).join(" | ")
|
|
3864
|
-
},
|
|
3865
|
-
artifacts: [jsonArtifact("runs.json", runs), jsonArtifact("journals.json", journals)]
|
|
3866
|
-
};
|
|
3867
|
-
}
|
|
3868
|
-
}
|
|
4162
|
+
claimJudgeDeadArmedRefusal
|
|
3869
4163
|
];
|
|
3870
4164
|
/** The scenario names in run order. */
|
|
3871
4165
|
const FAULT_SCENARIO_NAMES = SCENARIOS.map((scenario) => scenario.name);
|
|
@@ -4040,6 +4334,49 @@ const CLAIM_CORPUS = [
|
|
|
4040
4334
|
},
|
|
4041
4335
|
runFactTerms: ["in total", "end to end"],
|
|
4042
4336
|
expect: { minRunFactPairs: 1 }
|
|
4337
|
+
},
|
|
4338
|
+
{
|
|
4339
|
+
id: "bound-conflation-mcp-caps-guards",
|
|
4340
|
+
class: "bound-conflation",
|
|
4341
|
+
draft: "MCP resource reads are bounded unconditionally: byte caps, page caps, and the cursor guards all refuse oversized or cyclic reads by default (packages/core/src/tools/mcp.ts:44).",
|
|
4342
|
+
pool: [{
|
|
4343
|
+
nodeId: "agent:2",
|
|
4344
|
+
text: "The MCP read caps are OPT-IN: an absent maxResourceBytes or maxResourcePages means unbounded, and only requireBounds: true refuses their absence (packages/core/src/tools/mcp.ts:44-57). The cursor echo and visited-cursor guards are the unconditional half: they refuse a cycle regardless of any cap (packages/core/src/tools/mcp.ts:168-174)."
|
|
4345
|
+
}],
|
|
4346
|
+
critical: ["packages/core/src/tools/mcp.ts"],
|
|
4347
|
+
expect: {
|
|
4348
|
+
minPairs: 1,
|
|
4349
|
+
anchors: ["packages/core/src/tools/mcp.ts:44"],
|
|
4350
|
+
coverage: "full"
|
|
4351
|
+
}
|
|
4352
|
+
},
|
|
4353
|
+
{
|
|
4354
|
+
id: "derived-premise-slot-arithmetic",
|
|
4355
|
+
class: "derived-premise",
|
|
4356
|
+
draft: "The gateway sustains 2000 slots per burst window, a figure derived from the 30 minute window at the recorded admission rate.",
|
|
4357
|
+
runFacts: {
|
|
4358
|
+
text: "The declared burst window is 20 minutes end to end; at the recorded admission rate a 20 minute window admits 1333 slots, and 2000 slots would require the 30 minute window no configuration declares.",
|
|
4359
|
+
ids: ["comparison-run-aug13"],
|
|
4360
|
+
numbers: [
|
|
4361
|
+
20,
|
|
4362
|
+
1333,
|
|
4363
|
+
2e3
|
|
4364
|
+
]
|
|
4365
|
+
},
|
|
4366
|
+
runFactTerms: ["2000 slots", "30 minute window"],
|
|
4367
|
+
expect: { minRunFactPairs: 1 }
|
|
4368
|
+
},
|
|
4369
|
+
{
|
|
4370
|
+
id: "cost-basis-local-estimate-as-bill",
|
|
4371
|
+
class: "cost-basis",
|
|
4372
|
+
draft: "The provider bill for the whole workflow came to 5.5807 USD, charged and settled by the provider for this run.",
|
|
4373
|
+
runFacts: {
|
|
4374
|
+
text: "The invoice cost basis is 'locally-estimated': recorded usage priced at the pinned local rate table; no provider statement was reconciled for this run and no charged amount was recorded.",
|
|
4375
|
+
ids: ["comparison-run-aug13"],
|
|
4376
|
+
numbers: []
|
|
4377
|
+
},
|
|
4378
|
+
runFactTerms: ["provider bill", "charged and settled"],
|
|
4379
|
+
expect: { minRunFactPairs: 1 }
|
|
4043
4380
|
}
|
|
4044
4381
|
];
|
|
4045
4382
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulvar/evals",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.243.0",
|
|
4
4
|
"description": "Rulvar evals: eval cases, golden outputs, rubric and judge graders, matrix sweeps, canary fingerprint.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -22,11 +22,11 @@
|
|
|
22
22
|
"access": "public"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@rulvar/
|
|
26
|
-
"@rulvar/
|
|
27
|
-
"@rulvar/
|
|
28
|
-
"@rulvar/
|
|
29
|
-
"@rulvar/
|
|
25
|
+
"@rulvar/core": "1.243.0",
|
|
26
|
+
"@rulvar/openai": "1.243.0",
|
|
27
|
+
"@rulvar/testing": "1.243.0",
|
|
28
|
+
"@rulvar/anthropic": "1.243.0",
|
|
29
|
+
"@rulvar/plan": "1.243.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node": "^22.20.1",
|