@tangle-network/agent-eval 0.187.2 → 0.188.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-uAIQK3Nr.d.ts → agent-profile-CfbuZk8b.d.ts} +4 -4
- package/dist/{agent-profile-uAIQK3Nr.d.ts.map → agent-profile-CfbuZk8b.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +12 -12
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +6 -6
- package/dist/{analyze-runs-DapYZz02.d.ts → analyze-runs-CLz5O01x.d.ts} +5 -5
- package/dist/{analyze-runs-DapYZz02.d.ts.map → analyze-runs-CLz5O01x.d.ts.map} +1 -1
- package/dist/{benchmark-command-DGokT8zd.js → benchmark-command-CdUGku3W.js} +12 -12
- package/dist/benchmark-command-CdUGku3W.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-DlxdTOvw.js → campaign-DZuysBbS.js} +6 -6
- package/dist/{campaign-DlxdTOvw.js.map → campaign-DZuysBbS.js.map} +1 -1
- package/dist/{chat-client-DqPqWzVV.js → chat-client-C4dg7Ng_.js} +2 -2
- package/dist/{chat-client-DqPqWzVV.js.map → chat-client-C4dg7Ng_.js.map} +1 -1
- package/dist/{chat-json-call-C-DnDxEy.js → chat-json-call-JVGbfx95.js} +2 -2
- package/dist/{chat-json-call-C-DnDxEy.js.map → chat-json-call-JVGbfx95.js.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/{client-DCc7a8ZC.d.ts → client-CFj9dfcn.d.ts} +4 -4
- package/dist/{client-DCc7a8ZC.d.ts.map → client-CFj9dfcn.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +6 -6
- package/dist/{counterfactual-CLgrwhkY.d.ts → counterfactual-CPaWN99-.d.ts} +4 -4
- package/dist/{counterfactual-CLgrwhkY.d.ts.map → counterfactual-CPaWN99-.d.ts.map} +1 -1
- package/dist/{counterfactual-Bjq1mlUu.js → counterfactual-DtMWvn7D.js} +2 -2
- package/dist/{counterfactual-Bjq1mlUu.js.map → counterfactual-DtMWvn7D.js.map} +1 -1
- package/dist/{default-registry-DMxeb4GD.d.ts → default-registry-BLsGfnoO.d.ts} +6 -6
- package/dist/{default-registry-DMxeb4GD.d.ts.map → default-registry-BLsGfnoO.d.ts.map} +1 -1
- package/dist/{default-registry-Ca2IWH1X.js → default-registry-BtZ_t-9b.js} +2 -2
- package/dist/{default-registry-Ca2IWH1X.js.map → default-registry-BtZ_t-9b.js.map} +1 -1
- package/dist/{define-agent-eval-C0EqJ9Gz.js → define-agent-eval-ChtddasD.js} +3 -3
- package/dist/{define-agent-eval-C0EqJ9Gz.js.map → define-agent-eval-ChtddasD.js.map} +1 -1
- package/dist/{define-agent-eval-CGBwPPaq.d.ts → define-agent-eval-CsicmDxt.d.ts} +7 -7
- package/dist/{define-agent-eval-CGBwPPaq.d.ts.map → define-agent-eval-CsicmDxt.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +38 -96
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +83 -134
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-BE9lCzzL.js → dspy-rlm-engine-sPX7B5ac.js} +3 -3
- package/dist/{dspy-rlm-engine-BE9lCzzL.js.map → dspy-rlm-engine-sPX7B5ac.js.map} +1 -1
- package/dist/emitter-DbMXUIVP.js +334 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -0
- package/dist/{emitter-Cs0egaFd.d.ts → emitter-HcCmFQl_.d.ts} +37 -6
- package/dist/emitter-HcCmFQl_.d.ts.map +1 -0
- package/dist/{engine-DNlKhIm0.d.ts → engine-Gw2n1irt.d.ts} +15 -5
- package/dist/engine-Gw2n1irt.d.ts.map +1 -0
- package/dist/{eval-campaign-XFHEpN86.js → eval-campaign-CeotQ1ZR.js} +7 -4
- package/dist/{eval-campaign-XFHEpN86.js.map → eval-campaign-CeotQ1ZR.js.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts → exact-types-CrhPoDp8.d.ts} +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts.map → exact-types-CrhPoDp8.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +1 -2
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-BbhvXTFx.js → external-optimizer-process-D3BLpZyR.js} +2 -2
- package/dist/{external-optimizer-process-BbhvXTFx.js.map → external-optimizer-process-D3BLpZyR.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js → external-optimizer-subprocess-BrfsgL92.js} +2 -2
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js.map → external-optimizer-subprocess-BrfsgL92.js.map} +1 -1
- package/dist/{failure-cluster-OldNRoAt.d.ts → failure-cluster-CbgEDcFJ.d.ts} +3 -3
- package/dist/{failure-cluster-OldNRoAt.d.ts.map → failure-cluster-CbgEDcFJ.d.ts.map} +1 -1
- package/dist/failure-taxonomy-CjabTATj.js +538 -0
- package/dist/failure-taxonomy-CjabTATj.js.map +1 -0
- package/dist/{feedback-trajectory-CytTR7u4.d.ts → feedback-trajectory-BJi4t3Vf.d.ts} +6 -6
- package/dist/{feedback-trajectory-CytTR7u4.d.ts.map → feedback-trajectory-BJi4t3Vf.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-CKZa_rnt.d.ts → index-B-s8qDRJ.d.ts} +2 -2
- package/dist/{index-CKZa_rnt.d.ts.map → index-B-s8qDRJ.d.ts.map} +1 -1
- package/dist/{index-YZrbnBSV.d.ts → index-BD1oSbHr.d.ts} +11 -11
- package/dist/{index-YZrbnBSV.d.ts.map → index-BD1oSbHr.d.ts.map} +1 -1
- package/dist/{index-vYNHCV2G.d.ts → index-Ca0_IjFu.d.ts} +7 -7
- package/dist/{index-vYNHCV2G.d.ts.map → index-Ca0_IjFu.d.ts.map} +1 -1
- package/dist/{index-BRsrfSHT.d.ts → index-eTTnPe3r.d.ts} +9 -9
- package/dist/{index-BRsrfSHT.d.ts.map → index-eTTnPe3r.d.ts.map} +1 -1
- package/dist/index.d.ts +383 -68
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1482 -207
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DpOo-3iL.d.ts → insight-report-DeE1solA.d.ts} +4 -4
- package/dist/{insight-report-DpOo-3iL.d.ts.map → insight-report-DeE1solA.d.ts.map} +1 -1
- package/dist/{integrity-Cy9WHAtb.js → integrity-BFP4kjGj.js} +6 -1
- package/dist/integrity-BFP4kjGj.js.map +1 -0
- package/dist/{integrity-rGOfSUle.d.ts → integrity-CaeNeusS.d.ts} +4 -4
- package/dist/integrity-CaeNeusS.d.ts.map +1 -0
- package/dist/jev.d.ts +2 -2
- package/dist/{kind-factory-V79Hw0AU.js → kind-factory-g1ROw67A.js} +15 -4
- package/dist/{kind-factory-V79Hw0AU.js.map → kind-factory-g1ROw67A.js.map} +1 -1
- package/dist/{llm-client-DDhZ29Vm.js → llm-client-DF9bPNYA.js} +2 -2
- package/dist/{llm-client-DDhZ29Vm.js.map → llm-client-DF9bPNYA.js.map} +1 -1
- package/dist/{llm-judge-BVso5x-u.js → llm-judge-DF18J3CR.js} +9 -9
- package/dist/llm-judge-DF18J3CR.js.map +1 -0
- package/dist/{matrix-CPKsRY8Y.d.ts → matrix-BvA1WUpj.d.ts} +2 -2
- package/dist/{matrix-CPKsRY8Y.d.ts.map → matrix-BvA1WUpj.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +4 -4
- package/dist/{mint-BI14xehJ.d.ts → mint-CJFSYBIU.d.ts} +3 -3
- package/dist/{mint-BI14xehJ.d.ts.map → mint-CJFSYBIU.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +67 -12
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +178 -41
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{prime-protocol-CDBcL9NM.d.ts → prime-protocol-CF8Z43Fj.d.ts} +2 -2
- package/dist/{prime-protocol-CDBcL9NM.d.ts.map → prime-protocol-CF8Z43Fj.d.ts.map} +1 -1
- package/dist/{produced-state-Cn9ECr4v.js → produced-state-xsMMjYVc.js} +3 -3
- package/dist/{produced-state-Cn9ECr4v.js.map → produced-state-xsMMjYVc.js.map} +1 -1
- package/dist/{query-D6W6MaGx.d.ts → query-DBUQhehH.d.ts} +3 -3
- package/dist/{query-D6W6MaGx.d.ts.map → query-DBUQhehH.d.ts.map} +1 -1
- package/dist/random-DGEyr3zG.d.ts +83 -0
- package/dist/random-DGEyr3zG.d.ts.map +1 -0
- package/dist/{raw-provider-sink-BQd7mzyT.js → raw-provider-sink-C96CcqeG.js} +19 -56
- package/dist/raw-provider-sink-C96CcqeG.js.map +1 -0
- package/dist/redact-DPoH1-OW.d.ts +158 -0
- package/dist/redact-DPoH1-OW.d.ts.map +1 -0
- package/dist/redact-DxwlUAvh.js +824 -0
- package/dist/redact-DxwlUAvh.js.map +1 -0
- package/dist/{registry-D5Ly3GRV.d.ts → registry-BM2Kdk8T.d.ts} +4 -4
- package/dist/{registry-D5Ly3GRV.d.ts.map → registry-BM2Kdk8T.d.ts.map} +1 -1
- package/dist/{release-confidence-CSp3MsU4.d.ts → release-confidence-CG4Z9QLB.d.ts} +4 -4
- package/dist/{release-confidence-CSp3MsU4.d.ts.map → release-confidence-CG4Z9QLB.d.ts.map} +1 -1
- package/dist/{report-command-CSTjzFGY.js → report-command-DKJ_yAIj.js} +21 -12
- package/dist/{report-command-CSTjzFGY.js.map → report-command-DKJ_yAIj.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/{researcher-D7E92STT.d.ts → researcher-Bjcd_8T_.d.ts} +7 -7
- package/dist/{researcher-D7E92STT.d.ts.map → researcher-Bjcd_8T_.d.ts.map} +1 -1
- package/dist/{reward-hacking-BIQ2BWLG.d.ts → reward-hacking-DrKXJtgH.d.ts} +2 -2
- package/dist/{reward-hacking-BIQ2BWLG.d.ts.map → reward-hacking-DrKXJtgH.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts → rubric-predictive-validity-B14yBAmM.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts.map → rubric-predictive-validity-B14yBAmM.d.ts.map} +1 -1
- package/dist/{run-record-D9wS35fK.d.ts → run-record-xEULpgML.d.ts} +2 -2
- package/dist/{run-record-D9wS35fK.d.ts.map → run-record-xEULpgML.d.ts.map} +1 -1
- package/dist/{schema-CR5cpjQ3.d.ts → schema-2g7trUwH.d.ts} +17 -2
- package/dist/schema-2g7trUwH.d.ts.map +1 -0
- package/dist/schema-CSf6qWgZ.js.map +1 -1
- package/dist/{semantic-concept-judge-Cnt0TWp0.js → semantic-concept-judge-70LxrCSL.js} +5 -5
- package/dist/{semantic-concept-judge-Cnt0TWp0.js.map → semantic-concept-judge-70LxrCSL.js.map} +1 -1
- package/dist/{series-convergence-BnMs_uAr.d.ts → series-convergence-DY7ha0kx.d.ts} +2 -2
- package/dist/{series-convergence-BnMs_uAr.d.ts.map → series-convergence-DY7ha0kx.d.ts.map} +1 -1
- package/dist/{server-DmpcFiVK.js → server-C8XvqCg8.js} +2 -2
- package/dist/{server-DmpcFiVK.js.map → server-C8XvqCg8.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js → skillopt-optimization-method-C60h0Got.js} +5 -5
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js.map → skillopt-optimization-method-C60h0Got.js.map} +1 -1
- package/dist/spans-W0o8pbEg.d.ts +60 -0
- package/dist/spans-W0o8pbEg.d.ts.map +1 -0
- package/dist/{statistical-heldout-DTmhTW8P.d.ts → statistical-heldout-CPv-y6sK.d.ts} +3 -3
- package/dist/{statistical-heldout-DTmhTW8P.d.ts.map → statistical-heldout-CPv-y6sK.d.ts.map} +1 -1
- package/dist/{store-BErPvYBr.d.ts → store-C4avFjIi.d.ts} +2 -2
- package/dist/{store-BErPvYBr.d.ts.map → store-C4avFjIi.d.ts.map} +1 -1
- package/dist/{store-otlp-2gYCy-v0.js → store-otlp-DPgzSQKS.js} +2 -2
- package/dist/{store-otlp-2gYCy-v0.js.map → store-otlp-DPgzSQKS.js.map} +1 -1
- package/dist/{store-tool-spans-B3bX9yqn.d.ts → store-tool-spans-C2lLwDzg.d.ts} +9 -54
- package/dist/store-tool-spans-C2lLwDzg.d.ts.map +1 -0
- package/dist/{store-tool-spans-B1e6Auc2.js → store-tool-spans-ClPxcvh2.js} +4 -4
- package/dist/{store-tool-spans-B1e6Auc2.js.map → store-tool-spans-ClPxcvh2.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-Iy6kXo8x.d.ts → summary-report-SNvJoAvD.d.ts} +3 -3
- package/dist/{summary-report-Iy6kXo8x.d.ts.map → summary-report-SNvJoAvD.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{tool-groups-veQYPiTy.d.ts → tool-groups-CCQqAa16.d.ts} +3 -3
- package/dist/tool-groups-CCQqAa16.d.ts.map +1 -0
- package/dist/{tool-waste-nbHPAxei.js → tool-waste-CJDfxH45.js} +3 -537
- package/dist/tool-waste-CJDfxH45.js.map +1 -0
- package/dist/{tool-waste-D23I0zWm.d.ts → tool-waste-CP5DO383.d.ts} +4 -4
- package/dist/{tool-waste-D23I0zWm.d.ts.map → tool-waste-CP5DO383.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +12 -11
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +8 -8
- package/dist/{trajectory-D7qrNvaN.d.ts → trajectory-Y1fmdlw7.d.ts} +3 -3
- package/dist/{trajectory-D7qrNvaN.d.ts.map → trajectory-Y1fmdlw7.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -3
- package/dist/trajectory-replay/index.js +2 -2
- package/dist/{types--18oHwVF.d.ts → types-5ak2PqnJ.d.ts} +3 -3
- package/dist/{types--18oHwVF.d.ts.map → types-5ak2PqnJ.d.ts.map} +1 -1
- package/dist/{types-BTRcIqFr.d.ts → types-B1vDZ694.d.ts} +4 -4
- package/dist/{types-BTRcIqFr.d.ts.map → types-B1vDZ694.d.ts.map} +1 -1
- package/dist/{types-CBbLtr2J.d.ts → types-C0bWoSKC.d.ts} +10 -11
- package/dist/types-C0bWoSKC.d.ts.map +1 -0
- package/dist/{types-BjB2wZM9.d.ts → types-DDp2Mb8Y.d.ts} +3 -3
- package/dist/{types-BjB2wZM9.d.ts.map → types-DDp2Mb8Y.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/diagnosis.md +2 -3
- package/docs/eval-surface-map.md +1 -1
- package/docs/experiment.md +0 -1
- package/docs/redaction.md +59 -0
- package/docs/trace-contracts.md +102 -0
- package/docs/verdicts.md +1 -1
- package/package.json +3 -2
- package/dist/benchmark-command-DGokT8zd.js.map +0 -1
- package/dist/emitter-Cs0egaFd.d.ts.map +0 -1
- package/dist/emitter-DeQHiDMm.js +0 -257
- package/dist/emitter-DeQHiDMm.js.map +0 -1
- package/dist/engine-DNlKhIm0.d.ts.map +0 -1
- package/dist/experiment-tracker-BKEumQug.js +0 -280
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-URRGP9ek.d.ts +0 -237
- package/dist/experiment-tracker-URRGP9ek.d.ts.map +0 -1
- package/dist/integrity-Cy9WHAtb.js.map +0 -1
- package/dist/integrity-rGOfSUle.d.ts.map +0 -1
- package/dist/llm-judge-BVso5x-u.js.map +0 -1
- package/dist/raw-provider-sink-BQd7mzyT.js.map +0 -1
- package/dist/redact-7Aq1ukl-.js +0 -107
- package/dist/redact-7Aq1ukl-.js.map +0 -1
- package/dist/schema-CR5cpjQ3.d.ts.map +0 -1
- package/dist/store-tool-spans-B3bX9yqn.d.ts.map +0 -1
- package/dist/tool-groups-veQYPiTy.d.ts.map +0 -1
- package/dist/tool-waste-nbHPAxei.js.map +0 -1
- package/dist/types-CBbLtr2J.d.ts.map +0 -1
package/dist/reporting.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as benjaminiHochberg } from "./multiplicity-DIWHvysC.js";
|
|
2
2
|
import { d as PairedBootstrapResult, h as pairedBootstrap, u as PairedBootstrapOptions } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
3
|
-
import { _ as bootstrapCi, a as ReleaseConfidenceIssue, c as ReleaseConfidenceStatus, d as assertReleaseConfidence, f as evaluateReleaseConfidence, g as Verdict, h as JudgeReplayGateArgs, i as ReleaseConfidenceInput, k as wilcoxonSignedRank, l as ReleaseConfidenceThresholds, m as BootstrapResult, n as ReleaseConfidenceAxis, o as ReleaseConfidenceMetrics, p as BootstrapOptions, r as ReleaseConfidenceAxisName, s as ReleaseConfidenceScorecard, u as ReleaseTraceEvidence, v as judgeReplayGate } from "./release-confidence-
|
|
4
|
-
import { _ as paretoChart, a as ParetoPoint, c as ResearchReportCandidate, d as ResearchReportOptions, f as ResearchReportRecommendation, g as gainHistogram, h as SummaryTableRow, i as ParetoFigureSpec, l as ResearchReportDecision, m as SummaryTableOptions, n as GainDistributionFigureSpec, o as RESEARCH_REPORT_HARD_PAIR_FLOOR, p as SummaryTable, r as GainDistributionOptions, s as ResearchReport, t as GainDistributionBin, u as ResearchReportMethodology, v as researchReport, y as summaryTable } from "./summary-report-
|
|
3
|
+
import { _ as bootstrapCi, a as ReleaseConfidenceIssue, c as ReleaseConfidenceStatus, d as assertReleaseConfidence, f as evaluateReleaseConfidence, g as Verdict, h as JudgeReplayGateArgs, i as ReleaseConfidenceInput, k as wilcoxonSignedRank, l as ReleaseConfidenceThresholds, m as BootstrapResult, n as ReleaseConfidenceAxis, o as ReleaseConfidenceMetrics, p as BootstrapOptions, r as ReleaseConfidenceAxisName, s as ReleaseConfidenceScorecard, u as ReleaseTraceEvidence, v as judgeReplayGate } from "./release-confidence-CG4Z9QLB.js";
|
|
4
|
+
import { _ as paretoChart, a as ParetoPoint, c as ResearchReportCandidate, d as ResearchReportOptions, f as ResearchReportRecommendation, g as gainHistogram, h as SummaryTableRow, i as ParetoFigureSpec, l as ResearchReportDecision, m as SummaryTableOptions, n as GainDistributionFigureSpec, o as RESEARCH_REPORT_HARD_PAIR_FLOOR, p as SummaryTable, r as GainDistributionOptions, s as ResearchReport, t as GainDistributionBin, u as ResearchReportMethodology, v as researchReport, y as summaryTable } from "./summary-report-SNvJoAvD.js";
|
|
5
5
|
import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-BhsrMupG.js";
|
|
6
|
-
import { a as RubricPredictiveValidityReport, i as RubricPredictiveValidityInput, o as RubricRanking, r as RubricOutcomePair, s as rubricPredictiveValidity } from "./rubric-predictive-validity-
|
|
6
|
+
import { a as RubricPredictiveValidityReport, i as RubricPredictiveValidityInput, o as RubricRanking, r as RubricOutcomePair, s as rubricPredictiveValidity } from "./rubric-predictive-validity-B14yBAmM.js";
|
|
7
7
|
export { type BootstrapOptions, type BootstrapResult, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type InterimReleaseConfidence, type InterimReleaseConfidenceInput, type JudgeReplayGateArgs, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type ParetoFigureSpec, type ParetoPoint, RESEARCH_REPORT_HARD_PAIR_FLOOR, type ReleaseConfidenceAxis, type ReleaseConfidenceAxisName, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseConfidenceStatus, type ReleaseConfidenceThresholds, type ReleaseTraceEvidence, type ResearchReport, type ResearchReportCandidate, type ResearchReportDecision, type ResearchReportMethodology, type ResearchReportOptions, type ResearchReportRecommendation, type RubricOutcomePair, type RubricPredictiveValidityInput, type RubricPredictiveValidityReport, type RubricRanking, type SequentialDecision, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type Verdict, assertReleaseConfidence, benjaminiHochberg, bootstrapCi, evaluateInterimReleaseConfidence, evaluateReleaseConfidence, gainHistogram, judgeReplayGate, pairedBootstrap, pairedEvalueSequence, paretoChart, researchReport, rubricPredictiveValidity, summaryTable, wilcoxonSignedRank };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { i as AgentProfileCellInput, r as AgentProfileCell } from "./agent-profile-cell-BHcB-mZX.js";
|
|
2
|
-
import { a as RunRecord, c as RunTaskFailure, n as RunCostProvenance, r as RunJudgeMetadata, s as RunSplitTag, t as JudgeScoresRecord, u as RunTokenUsage } from "./run-record-
|
|
3
|
-
import { rt as RawProviderSink, x as ChatClient } from "./types-
|
|
4
|
-
import { s as TraceStore } from "./store-
|
|
5
|
-
import { b as GateDecision, d as ResearchReportOptions, s as ResearchReport } from "./summary-report-
|
|
6
|
-
import {
|
|
7
|
-
import { a as RunIntegrityReport, n as RunIntegrityExpectations } from "./integrity-
|
|
2
|
+
import { a as RunRecord, c as RunTaskFailure, n as RunCostProvenance, r as RunJudgeMetadata, s as RunSplitTag, t as JudgeScoresRecord, u as RunTokenUsage } from "./run-record-xEULpgML.js";
|
|
3
|
+
import { rt as RawProviderSink, x as ChatClient } from "./types-C0bWoSKC.js";
|
|
4
|
+
import { s as TraceStore } from "./store-C4avFjIi.js";
|
|
5
|
+
import { b as GateDecision, d as ResearchReportOptions, s as ResearchReport } from "./summary-report-SNvJoAvD.js";
|
|
6
|
+
import { o as TraceEmitter, r as RunCompleteHook } from "./emitter-HcCmFQl_.js";
|
|
7
|
+
import { a as RunIntegrityReport, n as RunIntegrityExpectations } from "./integrity-CaeNeusS.js";
|
|
8
8
|
//#region src/eval-campaign.d.ts
|
|
9
9
|
interface CampaignVariant<V> {
|
|
10
10
|
id: string;
|
|
@@ -290,4 +290,4 @@ interface Researcher {
|
|
|
290
290
|
}
|
|
291
291
|
//#endregion
|
|
292
292
|
export { SteeringChange as a, CampaignRunContext as c, CampaignVariant as d, EvalCampaignOptions as f, runEvalCampaign as h, Researcher as i, CampaignRunOutcome as l, FailedRun as m, ExperimentResult as n, CampaignFactoryParams as o, EvalCampaignResult as p, FailureMode as r, CampaignIntegrityPolicy as s, ExperimentPlan as t, CampaignScenario as u };
|
|
293
|
-
//# sourceMappingURL=researcher-
|
|
293
|
+
//# sourceMappingURL=researcher-Bjcd_8T_.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"researcher-
|
|
1
|
+
{"version":3,"file":"researcher-Bjcd_8T_.d.ts","names":[],"sources":["../src/eval-campaign.ts","../src/researcher.ts"],"mappings":";;;;;;;;UAuEiB,gBAAgB;EAC/B;EACA,SAAS;;UAGM;EACf;;EAEA,OAAO;;UAGQ,mBAAmB;;EAElC;;EAEA;EACA,SAAS;EACT;EACA;EACA,cAAc;EACd;EACA,UAAU;;;;;;;EAOV,SAAS;EACT,OAAO;EACP,SAAS;;;;;EAKT,MAAM;;;UAIS;;;;;;EAMf,SAAS;EACT;;UAGQ;;EAER;;EAEA;;EAEA;;EAEA,gBAAgB;EAChB,YAAY;;EAEZ;;EAEA;;EAEA;;EAEA,MAAM;;EAEN,gBAAgB;;;;;;EAMhB,cAAc;;;;;;EAMd,eAAe,mBAAmB;;;KAIxB,qBAAqB,2BAA2B;KAEhD,eAAe,MAAM,KAAK,mBAAmB,OAAO,QAAQ;KAE5D;UAEK,oBAAoB;;;;;EAKnC;EACA,UAAU,gBAAgB;EAC1B,WAAW;;EAEX;;EAEA,WAAW;;EAEX;;;;;;;;EAQA,cAAc,QAAQ,uBAAuB;;;;;;;EAO7C;;;;;;EAMA,eAAe,QAAQ,0BAA0B;;;;;;;EAOjD,kBAAkB,QAAQ,0BAA0B;;;;;EAKpD;;;;;EAKA,gBAAgB;;;;;;EAMhB,YAAY;;EAEZ,qBAAqB;;;;;EAKrB,QAAQ,eAAe;;;;;EAKvB;IAAW;MAAwB,KACjC;;;;;EAOF;;EAEA;;;;EAIA;;EAEA,SAAS,QAAQ;;;;;;;EAOjB,eACI,mBACA,0BAEE,QAAQ;IACN,SAAS;IACT,cAAc;QAGd,mBACA,wBACA,QAAQ,mBAAmB;;UAGpB;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;EACA;;EAEA,MAAM;;EAEN,kBAAkB;EAClB,YAAY;;EAEZ,SAAS;EACT;EACA;;iBAWoB,gBAAgB,GACpC,MAAM,oBAAoB,KACzB,QAAQ;;;;UClRM;;;EAGf;;EAEA;EACA;;;IAGE;;IAEA;;;;UAKa;EACf;;;;EAIA;;;EAGA;;EAEA;;;UAIe;EACf;EACA;EACA,SAAS;;;EAGT;;EAEA;IAAU;IAAkB;;;;UAIb;EACf,MAAM;EACN,MAAM;EACN,cAAc;;;;;;;;;;;;;;;;;;;UAoBC;EACf,gBAAgB,MAAM,cAAc,QAAQ;EAC5C,cAAc,UAAU,gBAAgB,QAAQ;EAChD,YAAY,SAAS,kBAAkB,UAAU,iBAAiB,QAAQ;EAC1E,eAAe,MAAM,iBAAiB,QAAQ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as RunRecord } from "./run-record-
|
|
1
|
+
import { a as RunRecord } from "./run-record-xEULpgML.js";
|
|
2
2
|
import { x as VerificationStrategySource } from "./verdict-E4eRNf7-.js";
|
|
3
3
|
import { s as VerificationReport } from "./multi-layer-verifier-BUaQ4C17.js";
|
|
4
4
|
//#region src/rl/verifiable-reward.d.ts
|
|
@@ -251,4 +251,4 @@ interface DetectRewardHackingInput {
|
|
|
251
251
|
declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
|
|
252
252
|
//#endregion
|
|
253
253
|
export { detectRewardHacking as a, VerifiableRewardSource as c, filterDeterministicallyRewarded as d, RewardHackingSignal as i, extractVerifiableReward as l, RewardHackingFinding as n, VerifiableReward as o, RewardHackingReport as r, VerifiableRewardExtractionOptions as s, DetectRewardHackingInput as t, extractVerifiableRewardsFromRecords as u };
|
|
254
|
-
//# sourceMappingURL=reward-hacking-
|
|
254
|
+
//# sourceMappingURL=reward-hacking-DrKXJtgH.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reward-hacking-
|
|
1
|
+
{"version":3,"file":"reward-hacking-DrKXJtgH.d.ts","names":[],"sources":["../src/rl/verifiable-reward.ts","../src/rl/reward-hacking.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA+DY,yBAAyB;UAEpB;;EAEf;;EAEA,QAAQ;;;;;;;;EAQR;;;;;;EAMA;;EAEA;;;;;;;EAOA,YAAY;;;;;;;;;;;EAWZ;;;;;;;;;;;;;;;;;;EAkBA;;UAGe;;;;;;;;EAQf;;;;EAIA,aAAa,sBAAsB;;;;;;;EAOnC;;;;;EAKA;;;;;;;;;;;;EAYA;;;;;;;;;;;;;;;iBA+Dc,wBACd,QAAQ,oBACR,OAAM,oCACL;;;;;;;;;;;;;;;;;;;;;;;iBAqGa,oCACd,MAAM,aACN,OAAM,oCACL;EAAQ;EAAe,QAAQ;;;;;;;;;;;iBA4GlB,gCACd,MAAM,aACN,OAAM,oCACL;EAAQ,KAAK;EAAW,QAAQ;;;;KC1YvB;UAMK;EACf,QAAQ;;EAER;EACA;;EAEA,QAAQ;;UAGO;EACf,UAAU;;EAEV,kBAAkB;;;;;;EAMlB;;EAEA;;EAEA;;UAGe;;;;;EAKf,MAAM;;;;;EAKN,WAAW,KAAK;;;;;;EAMhB,WAAW,KAAK;;;;;;EAMhB,qBAAqB,KAAK;;;;;EAK1B;;;;;EAKA;IAAe;IAAkB;;;;;EAIjC,0BAA0B;;iBAgBZ,oBAAoB,OAAO,2BAA2B"}
|
package/dist/rl.d.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { a as RunRecord, s as RunSplitTag } from "./run-record-
|
|
1
|
+
import { v as Span } from "./schema-2g7trUwH.js";
|
|
2
|
+
import { a as RunRecord, s as RunSplitTag } from "./run-record-xEULpgML.js";
|
|
3
3
|
import { a as PairedPromotionDecision, d as PairedBootstrapResult } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
4
|
-
import { a as CampaignResult } from "./types-
|
|
5
|
-
import { s as TraceStore } from "./store-
|
|
4
|
+
import { a as CampaignResult } from "./types-B1vDZ694.js";
|
|
5
|
+
import { s as TraceStore } from "./store-C4avFjIi.js";
|
|
6
6
|
import { s as VerificationReport } from "./multi-layer-verifier-BUaQ4C17.js";
|
|
7
|
-
import { a as detectRewardHacking, c as VerifiableRewardSource, d as filterDeterministicallyRewarded, i as RewardHackingSignal, l as extractVerifiableReward, n as RewardHackingFinding, o as VerifiableReward, r as RewardHackingReport, s as VerifiableRewardExtractionOptions, t as DetectRewardHackingInput, u as extractVerifiableRewardsFromRecords } from "./reward-hacking-
|
|
7
|
+
import { a as detectRewardHacking, c as VerifiableRewardSource, d as filterDeterministicallyRewarded, i as RewardHackingSignal, l as extractVerifiableReward, n as RewardHackingFinding, o as VerifiableReward, r as RewardHackingReport, s as VerifiableRewardExtractionOptions, t as DetectRewardHackingInput, u as extractVerifiableRewardsFromRecords } from "./reward-hacking-DrKXJtgH.js";
|
|
8
8
|
import { i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "./outcome-store-CNt4iZ67.js";
|
|
9
9
|
import { a as MintedRolloutLine, t as ChatMessage, y as RolloutSplit } from "./schema-BzWDXhOR.js";
|
|
10
10
|
import { t as InterimReleaseConfidence } from "./sequential-BhsrMupG.js";
|
|
11
11
|
import { t as AdversarialMutation } from "./adversarial-smnADNFS.js";
|
|
12
|
-
import { a as SteeringChange, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, n as ExperimentResult, p as EvalCampaignResult, r as FailureMode, t as ExperimentPlan } from "./researcher-
|
|
13
|
-
import { t as MintRolloutOptions, y as TrainingExportOptions } from "./mint-
|
|
14
|
-
import { a as RubricPredictiveValidityReport, t as OutcomeMetricSpec } from "./rubric-predictive-validity-
|
|
12
|
+
import { a as SteeringChange, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, n as ExperimentResult, p as EvalCampaignResult, r as FailureMode, t as ExperimentPlan } from "./researcher-Bjcd_8T_.js";
|
|
13
|
+
import { t as MintRolloutOptions, y as TrainingExportOptions } from "./mint-CJFSYBIU.js";
|
|
14
|
+
import { a as RubricPredictiveValidityReport, t as OutcomeMetricSpec } from "./rubric-predictive-validity-B14yBAmM.js";
|
|
15
15
|
//#region src/rl/active-curriculum.d.ts
|
|
16
16
|
interface CellObservation {
|
|
17
17
|
variantId: string;
|
package/dist/rl.js
CHANGED
|
@@ -10,7 +10,7 @@ import { s as runTaskScore } from "./run-record-tg_5H2IK.js";
|
|
|
10
10
|
import { a as campaignCellToRunRecord } from "./run-record-CNNzWGpB.js";
|
|
11
11
|
import { i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
|
|
12
12
|
import { n as InMemoryTraceStore } from "./store-DNe_Uv1Q.js";
|
|
13
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
13
|
+
import { t as runEvalCampaign } from "./eval-campaign-CeotQ1ZR.js";
|
|
14
14
|
import { s as assertRewardGate } from "./schema-C1aaAxTf.js";
|
|
15
15
|
import { n as isTrainingLineEligible, s as toSftRows$1, t as isSplitEligible } from "./exporters-DEcXbvgO.js";
|
|
16
16
|
import { t as mintRolloutRows } from "./mint-B4zgSAJW.js";
|
package/dist/rollout/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { C as assertMinted, D as isRolloutLine, E as gateGamedOutcome, O as isTrainableSplit, S as ToolDef, T as assertRolloutLine, _ as RolloutProvenance, a as MintedRolloutLine, b as RolloutStep, c as ROLLOUT_ROLES, d as RolloutArtifacts, f as RolloutCapture, g as RolloutPolicy, h as RolloutOutcome, i as GatedEvidence, k as validateRolloutLine, l as ROLLOUT_SCHEMA, m as RolloutLine, n as ChatRole, o as MintedRolloutOutcome, p as RolloutCostBlock, r as ChatToolCall, s as ROLLOUT_CAPTURES, t as ChatMessage, u as ROLLOUT_SPLITS, v as RolloutRole, w as assertMintedLines, x as RolloutTask, y as RolloutSplit } from "../schema-BzWDXhOR.js";
|
|
2
|
-
import { C as toRewardRows, D as toVerifiersRolloutOutput, E as toSftRows, O as toVerifiersRolloutOutputs, S as toJsonl, T as toRftItems, _ as SftExportOptions, a as unmintableReasons, b as VerifiersRolloutOutput, c as isRealnessGated, d as scoreOrigin, f as trainingReward, g as RftItem, h as RewardRow, i as mintRolloutRows, l as observedScore, m as RealnessLabels, n as MintRolloutResult, o as ScoreOrigin, p as trainingScore, r as RolloutScrubber, s as ScorePreference, t as MintRolloutOptions, u as observedSplitScore, v as SftRow, w as toRftItem, x as VerifiersTokenUsage } from "../mint-
|
|
3
|
-
import { $ as HarborFinalMetrics, A as measureFormatGate, B as DEFAULT_CLAUDE_PROJECTS_DIR, C as FORMAT_GATE_DISPOSITION, Ct as gatedEvidenceOf, D as ReleaseRowRef, E as GateReport, F as openOpencodeDb, G as readRolloutJournal, H as findClaudeTranscripts, I as readOpencodeSessionMessages, J as ATIF_SCHEMA_VERSION, K as readRolloutLedger, L as ClaudeTranscript, M as DEFAULT_OPENCODE_DB, N as OpencodeSessionRow, O as assertGateReport, P as findOpencodeSessionsByDirectory, Q as HarborContentPart, R as ClaudeTranscriptRef, S as EmittedEvidence, St as gateErrors, T as GateDisposition, U as readClaudeTranscript, V as claudeProjectSlug, W as appendRolloutLines, X as HARBOR_IMPORT_GAP, Y as FromHarborOptions, Z as HarborAgent, _ as defaultRolloutScrubber, _t as GateCheckDisposition, a as buildHfDataset, at as HarborStepSource, b as scrubRolloutLine, bt as GateEntryPoint, c as runRolloutReleaseCli, ct as HarborTrajectory, d as RELEASE_FORMATS, dt as toHarborTrajectories, et as HarborImageSource, f as ReleaseFormat, ft as toHarborTrajectory, g as addScrubCounts, gt as GateCheck, h as ScrubRule, ht as GATE_POLICIES, i as ScrubReport, it as HarborStep, j as releaseRowRefs, k as gatedRolloutIds, l as DatasetCardInputs, lt as fromHarborTrajectory, m as ScrubCounts, mt as GATE_CHECK_IDS, n as BuildSummary, nt as HarborObservation, o as parseRolloutReleaseArgs, ot as HarborSubagentTrajectoryRef, p as buildDatasetCard, pt as GATE_CHECKS, q as writeRolloutLedger, r as RolloutReleaseCliArgs, rt as HarborObservationResult, s as planPushCommand, st as HarborToolCall, t as BuildOptions, tt as HarborMetrics, u as FORMAT_FILES, ut as relabelImportedSplit, v as emptyScrubCounts, vt as GateCheckId, w as FormatGateCounts, x as scrubText, xt as GatePolicy, y as scrubLines, yt as GateCheckedOutcome, z as ClaudeUsageTotals } from "../index-
|
|
2
|
+
import { C as toRewardRows, D as toVerifiersRolloutOutput, E as toSftRows, O as toVerifiersRolloutOutputs, S as toJsonl, T as toRftItems, _ as SftExportOptions, a as unmintableReasons, b as VerifiersRolloutOutput, c as isRealnessGated, d as scoreOrigin, f as trainingReward, g as RftItem, h as RewardRow, i as mintRolloutRows, l as observedScore, m as RealnessLabels, n as MintRolloutResult, o as ScoreOrigin, p as trainingScore, r as RolloutScrubber, s as ScorePreference, t as MintRolloutOptions, u as observedSplitScore, v as SftRow, w as toRftItem, x as VerifiersTokenUsage } from "../mint-CJFSYBIU.js";
|
|
3
|
+
import { $ as HarborFinalMetrics, A as measureFormatGate, B as DEFAULT_CLAUDE_PROJECTS_DIR, C as FORMAT_GATE_DISPOSITION, Ct as gatedEvidenceOf, D as ReleaseRowRef, E as GateReport, F as openOpencodeDb, G as readRolloutJournal, H as findClaudeTranscripts, I as readOpencodeSessionMessages, J as ATIF_SCHEMA_VERSION, K as readRolloutLedger, L as ClaudeTranscript, M as DEFAULT_OPENCODE_DB, N as OpencodeSessionRow, O as assertGateReport, P as findOpencodeSessionsByDirectory, Q as HarborContentPart, R as ClaudeTranscriptRef, S as EmittedEvidence, St as gateErrors, T as GateDisposition, U as readClaudeTranscript, V as claudeProjectSlug, W as appendRolloutLines, X as HARBOR_IMPORT_GAP, Y as FromHarborOptions, Z as HarborAgent, _ as defaultRolloutScrubber, _t as GateCheckDisposition, a as buildHfDataset, at as HarborStepSource, b as scrubRolloutLine, bt as GateEntryPoint, c as runRolloutReleaseCli, ct as HarborTrajectory, d as RELEASE_FORMATS, dt as toHarborTrajectories, et as HarborImageSource, f as ReleaseFormat, ft as toHarborTrajectory, g as addScrubCounts, gt as GateCheck, h as ScrubRule, ht as GATE_POLICIES, i as ScrubReport, it as HarborStep, j as releaseRowRefs, k as gatedRolloutIds, l as DatasetCardInputs, lt as fromHarborTrajectory, m as ScrubCounts, mt as GATE_CHECK_IDS, n as BuildSummary, nt as HarborObservation, o as parseRolloutReleaseArgs, ot as HarborSubagentTrajectoryRef, p as buildDatasetCard, pt as GATE_CHECKS, q as writeRolloutLedger, r as RolloutReleaseCliArgs, rt as HarborObservationResult, s as planPushCommand, st as HarborToolCall, t as BuildOptions, tt as HarborMetrics, u as FORMAT_FILES, ut as relabelImportedSplit, v as emptyScrubCounts, vt as GateCheckId, w as FormatGateCounts, x as scrubText, xt as GatePolicy, y as scrubLines, yt as GateCheckedOutcome, z as ClaudeUsageTotals } from "../index-B-s8qDRJ.js";
|
|
4
4
|
export { ATIF_SCHEMA_VERSION, type BuildOptions, type BuildSummary, type ChatMessage, type ChatRole, type ChatToolCall, type ClaudeTranscript, type ClaudeTranscriptRef, type ClaudeUsageTotals, DEFAULT_CLAUDE_PROJECTS_DIR, DEFAULT_OPENCODE_DB, type DatasetCardInputs, type EmittedEvidence, FORMAT_FILES, FORMAT_GATE_DISPOSITION, type FormatGateCounts, type FromHarborOptions, GATE_CHECKS, GATE_CHECK_IDS, GATE_POLICIES, type GateCheck, type GateCheckDisposition, type GateCheckId, type GateCheckedOutcome, type GateDisposition, type GateEntryPoint, type GatePolicy, type GateReport, type GatedEvidence, HARBOR_IMPORT_GAP, type HarborAgent, type HarborContentPart, type HarborFinalMetrics, type HarborImageSource, type HarborMetrics, type HarborObservation, type HarborObservationResult, type HarborStep, type HarborStepSource, type HarborSubagentTrajectoryRef, type HarborToolCall, type HarborTrajectory, type MintRolloutOptions, type MintRolloutResult, type MintedRolloutLine, type MintedRolloutOutcome, type OpencodeSessionRow, RELEASE_FORMATS, ROLLOUT_CAPTURES, ROLLOUT_ROLES, ROLLOUT_SCHEMA, ROLLOUT_SPLITS, type RealnessLabels, type ReleaseFormat, type ReleaseRowRef, type RewardRow, type RftItem, type RolloutArtifacts, type RolloutCapture, type RolloutCostBlock, type RolloutLine, type RolloutOutcome, type RolloutPolicy, type RolloutProvenance, type RolloutReleaseCliArgs, type RolloutRole, type RolloutScrubber, type RolloutSplit, type RolloutStep, type RolloutTask, type ScoreOrigin, type ScorePreference, type ScrubCounts, type ScrubReport, type ScrubRule, type SftExportOptions, type SftRow, type ToolDef, type VerifiersRolloutOutput, type VerifiersTokenUsage, addScrubCounts, appendRolloutLines, assertGateReport, assertMinted, assertMintedLines, assertRolloutLine, buildDatasetCard, buildHfDataset, claudeProjectSlug, defaultRolloutScrubber, emptyScrubCounts, findClaudeTranscripts, findOpencodeSessionsByDirectory, fromHarborTrajectory, gateErrors, gateGamedOutcome, gatedEvidenceOf, gatedRolloutIds, isRealnessGated, isRolloutLine, isTrainableSplit, measureFormatGate, mintRolloutRows, observedScore, observedSplitScore, openOpencodeDb, parseRolloutReleaseArgs, planPushCommand, readClaudeTranscript, readOpencodeSessionMessages, readRolloutJournal, readRolloutLedger, relabelImportedSplit, releaseRowRefs, runRolloutReleaseCli, scoreOrigin, scrubLines, scrubRolloutLine, scrubText, toHarborTrajectories, toHarborTrajectory, toJsonl, toRewardRows, toRftItem, toRftItems, toSftRows, toVerifiersRolloutOutput, toVerifiersRolloutOutputs, trainingReward, trainingScore, unmintableReasons, validateRolloutLine, writeRolloutLedger };
|
package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts → rubric-predictive-validity-B14yBAmM.d.ts}
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as RunRecord } from "./run-record-
|
|
1
|
+
import { a as RunRecord } from "./run-record-xEULpgML.js";
|
|
2
2
|
import { o as OutcomeStore } from "./outcome-store-CNt4iZ67.js";
|
|
3
3
|
//#region src/meta-eval/outcome-observations.d.ts
|
|
4
4
|
type OutcomeReduction = 'latest' | 'mean' | 'max';
|
|
@@ -76,4 +76,4 @@ interface RubricPredictiveValidityReport {
|
|
|
76
76
|
declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
|
|
77
77
|
//#endregion
|
|
78
78
|
export { RubricPredictiveValidityReport as a, CorrelationInterval as c, RubricPredictiveValidityInput as i, OutcomeReduction as l, RubricOutcomeExclusion as n, RubricRanking as o, RubricOutcomePair as r, rubricPredictiveValidity as s, OutcomeMetricSpec as t };
|
|
79
|
-
//# sourceMappingURL=rubric-predictive-validity-
|
|
79
|
+
//# sourceMappingURL=rubric-predictive-validity-B14yBAmM.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rubric-predictive-validity-
|
|
1
|
+
{"version":3,"file":"rubric-predictive-validity-B14yBAmM.d.ts","names":[],"sources":["../src/meta-eval/outcome-observations.ts","../src/meta-eval/rubric-predictive-validity.ts"],"mappings":";;;KAIY;UAEK;EACf;EACA;;;;UCWe;;EAEf;EACA;;UAGe;;EAEf,MAAM;EACN,UAAU;;EAEV,yBAAyB;;EAEzB;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,kBAAkB;EAClB;;EAEA;EACA;EACA,aAAa;EACb,cAAc;;EAEd;EACA;EACA,qBAAqB;;EAErB;;UAGe,sBAAsB,KAAK;;EAE1C;;UAGe;EACf;EACA;EACA,kBAAkB;;EAElB;EACA;;UAGe;EACf,gBAAgB;EAChB,OAAO;;EAEP,eAAe;;EAEf,QAAQ;;EAER;;EAEA;;EAEA;;iBAGoB,yBACpB,OAAO,gCACN,QAAQ"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { c as ValidationError } from "./errors-DEE6u6ot.js";
|
|
2
2
|
import { p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
3
3
|
import { r as AgentProfileCell } from "./agent-profile-cell-BHcB-mZX.js";
|
|
4
|
-
import { o as FailureClass } from "./schema-
|
|
4
|
+
import { o as FailureClass } from "./schema-2g7trUwH.js";
|
|
5
5
|
//#region src/run-record.d.ts
|
|
6
6
|
/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
|
|
7
7
|
* combined train+test pool that the optimizer is allowed to read. */
|
|
@@ -244,4 +244,4 @@ declare function roundTripRunRecord(record: RunRecord): RunRecord;
|
|
|
244
244
|
declare function modelHasSnapshot(model: string): boolean;
|
|
245
245
|
//#endregion
|
|
246
246
|
export { validateRunRecord as _, RunRecord as a, RunTaskFailure as c, UNKNOWN_MODEL as d, isRunRecord as f, runTaskScore as g, roundTripRunRecord as h, RunOutcome as i, RunTerminalOutcome as l, parseRunRecordSafe as m, RunCostProvenance as n, RunRecordValidationError as o, modelHasSnapshot as p, RunJudgeMetadata as r, RunSplitTag as s, JudgeScoresRecord as t, RunTokenUsage as u };
|
|
247
|
-
//# sourceMappingURL=run-record-
|
|
247
|
+
//# sourceMappingURL=run-record-xEULpgML.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-record-
|
|
1
|
+
{"version":3,"file":"run-record-xEULpgML.d.ts","names":[],"sources":["../src/run-record.ts"],"mappings":";;;;;;;KAsCY;;;;;;;KAQA;;cAGC;UAEI;EACf;;EAEA;;;EAGA;;EAEA;;EAEA;;EAEA;;;KAIU,oBAAoB;UAEf;EACf;EACA;;;EAGA;;;EAGA;;;;;;;;;;;;;;;;;;;;;UAsBe;;EAEf,UAAU,eAAe;;EAEzB,YAAY;;;EAGZ;;;;EAIA;;;EAGA;;UAGe;;;EAGf;;;EAGA;;;;EAIA,KAAK;;;;;;EAML,cAAc;;;;;;;EAOd;IAAa;IAAe;IAAgB;;;;;;;;;;;;;;;;;;;;;;UAsB7B;;EAEf;;;EAGA;;;EAGA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB,YAAY;;EAEZ,iBAAiB;;;EAGjB;;EAEA,gBAAgB;;EAEhB,SAAS;;;;;EAKT,eAAe;;;EAGf;;EAEA,UAAU;;;;;EAKV;;;;;;;;EAQA,eAAe;;;;;;;;;KAUL;EACN;EAA0B;;EAC1B;EAAyB;;EAEzB,cAAc,QAAQ;EACtB;;;;;;;;;;iBAWU,aAAa,QAAQ;cAmCxB,iCAAiC;WACnC;EACT,YAAY,iBAAiB;;;;;;;iBAWf,kBAAkB,iBAAiB;;iBAgPnC,YAAY,iBAAiB,SAAS;;iBAUtC,mBACd;EACG;EAAU,OAAO;;EAAgB;EAAW,OAAO;;;iBAUxC,mBAAmB,QAAQ,YAAY;;;;;;;iBAuFvC,iBAAiB"}
|
|
@@ -74,6 +74,21 @@ interface Run {
|
|
|
74
74
|
budget?: BudgetSpec;
|
|
75
75
|
/** Free-form labels for downstream grouping. */
|
|
76
76
|
tags?: Record<string, string>;
|
|
77
|
+
/**
|
|
78
|
+
* The producer's own account of its store writes for this run, written when
|
|
79
|
+
* the run ends. Absent when the producer kept no account, which is unknown,
|
|
80
|
+
* not complete.
|
|
81
|
+
*/
|
|
82
|
+
capture?: RunCaptureReport;
|
|
83
|
+
}
|
|
84
|
+
/** Store writes a producer made for one run. */
|
|
85
|
+
interface RunCaptureReport {
|
|
86
|
+
/** Writes that landed. */
|
|
87
|
+
written: number;
|
|
88
|
+
/** Writes that failed; those records are missing from the store. */
|
|
89
|
+
dropped: number;
|
|
90
|
+
/** The most recent failure, prefixed with the write that failed. */
|
|
91
|
+
lastError?: string;
|
|
77
92
|
}
|
|
78
93
|
type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
|
|
79
94
|
type SpanStatus = 'ok' | 'error';
|
|
@@ -259,5 +274,5 @@ declare function isLlmSpan(s: Span): s is LlmSpan;
|
|
|
259
274
|
declare function isToolSpan(s: Span): s is ToolSpan;
|
|
260
275
|
declare function isJudgeSpan(s: Span): s is JudgeSpan;
|
|
261
276
|
//#endregion
|
|
262
|
-
export {
|
|
263
|
-
//# sourceMappingURL=schema-
|
|
277
|
+
export { ToolSpan as C, isToolSpan as D, isLlmSpan as E, TRACE_SCHEMA_VERSION as S, isJudgeSpan as T, SandboxSpan as _, FAILURE_CLASSES as a, SpanKind as b, JudgeSpan as c, RetrievalSpan as d, Run as f, RunStatus as g, RunOutcome as h, EventKind as i, LlmSpan as l, RunLayer as m, BudgetLedgerEntry as n, FailureClass as o, RunCaptureReport as p, BudgetSpec as r, GenericSpan as s, Artifact as t, Message as u, Span as v, TraceEvent as w, SpanStatus as x, SpanBase as y };
|
|
278
|
+
//# sourceMappingURL=schema-2g7trUwH.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-2g7trUwH.d.ts","names":[],"sources":["../src/trace/schema.ts"],"mappings":";;;;;;;;;;;;cAYa;KAID;UAEK;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,eAAe;EACf;;;;;;;;;KAUU;UAEK;EACf;;;;;;;;;;;EAWA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;EAEA,iBAAiB;;EAEjB;;;EAGA;;EAEA;;EAEA;;EAEA,QAAQ;EACR;EACA;EACA,QAAQ;EACR,UAAU;EACV,SAAS;;EAET,OAAO;;;;;;EAMP,UAAU;;;UAIK;;EAEf;;EAEA;;EAEA;;KAKU;KAEA;UAEK;EACf;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,SAAS;EACT;;EAEA,aAAa;;UAGE;EACf;EACA;EACA;;EAEA,SAAS;IAAQ;IAAqB;IAAc;;;UAGrC,gBAAgB;EAC/B;EACA;EACA,UAAU;EACV;EACA;;EAEA;EACA;EACA;;EAEA;EACA;EACA;;UAGe,iBAAiB;EAChC;EACA;EACA;;EAEA;EACA;EACA;;UAGe,sBAAsB;EACrC;EACA;EACA,MAAM;IAAQ;IAAe;IAAe;;;UAG7B,kBAAkB;EACjC;EACA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe,oBAAoB;EACnC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;UAGe,oBAAoB;EACnC;;KAGU,OAAO,UAAU,WAAW,gBAAgB,YAAY,cAAc;KAItE;UAUK;EACf;EACA;EACA;EACA,MAAM;EACN;EACA,SAAS;;UAKM;EACf;EACA,iBAAiB;EACjB;EACA;EACA;EACA;EACA;;EAEA;;UAKe;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;;;;;cAyBW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAgFD,uBAAuB;iBAInB,UAAU,GAAG,OAAO,KAAK;iBAGzB,WAAW,GAAG,OAAO,KAAK;iBAG1B,YAAY,GAAG,OAAO,KAAK"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-CSf6qWgZ.js","names":[],"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface RunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: RunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n /** All generated tokens, including the reasoning subset when present. */\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n /** Reasoning-token subset of `outputTokens`. */\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n /** False when the source observed the call but did not capture its arguments. */\n argsCaptured?: boolean\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\n/**\n * The failure taxonomy. `FailureClass` derives from this array, so the type\n * and the runtime list cannot name different sets.\n *\n * Two layers, and the split is what `FAILURE_BLAME` reads\n * (`src/failure-taxonomy.ts`):\n * - behaviour classes name what the agent did wrong, and a run that ends in\n * one is evidence about the agent;\n * - transport classes name a death before or around the model turn — the\n * bridge refused, the box never provisioned, the stream tore — and a run\n * that ends in one of the machine-blamed members is not evidence about the\n * agent at all.\n *\n * The transport layer exists because a taxonomy with no name for \"the run\n * never reached a model\" charges an infrastructure outage to the agent under\n * test. Measured: six director cells settled with the runtime journal's own\n * `infra` flag reading FALSE while the real reason was\n * `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the\n * bridge's host lane had saturated.\n */\nexport const FAILURE_CLASSES = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n\n // ── Transport: the run never reached a model, or the turn never completed ──\n /** The bridge refused the connection or dropped it before a terminal event. */\n 'bridge_unreachable',\n /** The bridge accepted the request and its execution lane was already full. */\n 'bridge_lane_saturated',\n /** The router could not reconcile a completion with its own recorded usage. */\n 'router_usage_mismatch',\n /** A session was bound to one AgentProfile and asked to serve another. */\n 'session_profile_conflict',\n /** The agent runtime restarted underneath a live run. */\n 'runtime_restart',\n /** The root driver of a recursive run failed before it could dispatch work. */\n 'root_driver_failed',\n /** A sandbox box never finished provisioning. Distinct from `sandbox_failure`,\n * which is a command the agent ran exiting non-zero inside a live box. */\n 'box_provision_failure',\n /** A sandbox box did not confirm destruction after the run. */\n 'box_teardown_failure',\n /** The harness refused an agent-authored profile: an unknown model, a\n * workspace file it may not overwrite, a system prompt it may not replace. */\n 'profile_refused',\n /** The model was reached and returned no visible output. */\n 'empty_completion',\n /** The response stream carried malformed protocol content. */\n 'stream_protocol_error',\n /** The response stream ended before a terminal event. */\n 'stream_incomplete',\n /** A message was appended to a conversation the provider had already sealed. */\n 'message_sealed',\n /** The model process was killed by a signal or exited with no code. */\n 'process_killed',\n /** The run was cancelled by an operator. */\n 'cancelled',\n /** The run was terminated without a more specific reason. */\n 'terminated',\n /** The agent exhausted its traversal or iteration cap. */\n 'traversal_cap_exhausted',\n\n /** No reason was recorded at all. Distinct from `unknown`, which means a\n * reason exists and no rule matched it. */\n 'unreported',\n 'unknown',\n] as const\n\nexport type FailureClass = (typeof FAILURE_CLASSES)[number]\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\n"],"mappings":";;;;;;;;;;;;AAYA,MAAa,uBAAuB;;;;;;;;;;;;;;;;;;;;;AA6OpC,MAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA;CAGA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAIA;CACA;AACF;AAMA,SAAgB,UAAU,GAAuB;CAC/C,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,WAAW,GAAwB;CACjD,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,YAAY,GAAyB;CACnD,OAAO,EAAE,SAAS;AACpB"}
|
|
1
|
+
{"version":3,"file":"schema-CSf6qWgZ.js","names":[],"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface RunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: RunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n /**\n * The producer's own account of its store writes for this run, written when\n * the run ends. Absent when the producer kept no account, which is unknown,\n * not complete.\n */\n capture?: RunCaptureReport\n}\n\n/** Store writes a producer made for one run. */\nexport interface RunCaptureReport {\n /** Writes that landed. */\n written: number\n /** Writes that failed; those records are missing from the store. */\n dropped: number\n /** The most recent failure, prefixed with the write that failed. */\n lastError?: string\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n /** All generated tokens, including the reasoning subset when present. */\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n /** Reasoning-token subset of `outputTokens`. */\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n /** False when the source observed the call but did not capture its arguments. */\n argsCaptured?: boolean\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\n/**\n * The failure taxonomy. `FailureClass` derives from this array, so the type\n * and the runtime list cannot name different sets.\n *\n * Two layers, and the split is what `FAILURE_BLAME` reads\n * (`src/failure-taxonomy.ts`):\n * - behaviour classes name what the agent did wrong, and a run that ends in\n * one is evidence about the agent;\n * - transport classes name a death before or around the model turn — the\n * bridge refused, the box never provisioned, the stream tore — and a run\n * that ends in one of the machine-blamed members is not evidence about the\n * agent at all.\n *\n * The transport layer exists because a taxonomy with no name for \"the run\n * never reached a model\" charges an infrastructure outage to the agent under\n * test. Measured: six director cells settled with the runtime journal's own\n * `infra` flag reading FALSE while the real reason was\n * `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the\n * bridge's host lane had saturated.\n */\nexport const FAILURE_CLASSES = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n\n // ── Transport: the run never reached a model, or the turn never completed ──\n /** The bridge refused the connection or dropped it before a terminal event. */\n 'bridge_unreachable',\n /** The bridge accepted the request and its execution lane was already full. */\n 'bridge_lane_saturated',\n /** The router could not reconcile a completion with its own recorded usage. */\n 'router_usage_mismatch',\n /** A session was bound to one AgentProfile and asked to serve another. */\n 'session_profile_conflict',\n /** The agent runtime restarted underneath a live run. */\n 'runtime_restart',\n /** The root driver of a recursive run failed before it could dispatch work. */\n 'root_driver_failed',\n /** A sandbox box never finished provisioning. Distinct from `sandbox_failure`,\n * which is a command the agent ran exiting non-zero inside a live box. */\n 'box_provision_failure',\n /** A sandbox box did not confirm destruction after the run. */\n 'box_teardown_failure',\n /** The harness refused an agent-authored profile: an unknown model, a\n * workspace file it may not overwrite, a system prompt it may not replace. */\n 'profile_refused',\n /** The model was reached and returned no visible output. */\n 'empty_completion',\n /** The response stream carried malformed protocol content. */\n 'stream_protocol_error',\n /** The response stream ended before a terminal event. */\n 'stream_incomplete',\n /** A message was appended to a conversation the provider had already sealed. */\n 'message_sealed',\n /** The model process was killed by a signal or exited with no code. */\n 'process_killed',\n /** The run was cancelled by an operator. */\n 'cancelled',\n /** The run was terminated without a more specific reason. */\n 'terminated',\n /** The agent exhausted its traversal or iteration cap. */\n 'traversal_cap_exhausted',\n\n /** No reason was recorded at all. Distinct from `unknown`, which means a\n * reason exists and no rule matched it. */\n 'unreported',\n 'unknown',\n] as const\n\nexport type FailureClass = (typeof FAILURE_CLASSES)[number]\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\n"],"mappings":";;;;;;;;;;;;AAYA,MAAa,uBAAuB;;;;;;;;;;;;;;;;;;;;;AA6PpC,MAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA;CAGA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAIA;CACA;AACF;AAMA,SAAgB,UAAU,GAAuB;CAC/C,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,WAAW,GAAwB;CACjD,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,YAAY,GAAyB;CACnD,OAAO,EAAE,SAAS;AACpB"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { i as CostLedger } from "./cost-ledger-_IbkejCF.js";
|
|
2
2
|
import { u as Mutex } from "./journal-BuRLm6ho.js";
|
|
3
|
-
import {
|
|
4
|
-
import { c as extractJsonPayload } from "./llm-client-
|
|
5
|
-
import { n as paidJsonChat, t as paidChat } from "./chat-json-call-
|
|
6
|
-
import { n as decodeRawFindingArray } from "./dspy-rlm-engine-
|
|
3
|
+
import { N as RawAnalystFindingSchema } from "./kind-factory-g1ROw67A.js";
|
|
4
|
+
import { c as extractJsonPayload } from "./llm-client-DF9bPNYA.js";
|
|
5
|
+
import { n as paidJsonChat, t as paidChat } from "./chat-json-call-JVGbfx95.js";
|
|
6
|
+
import { n as decodeRawFindingArray } from "./dspy-rlm-engine-sPX7B5ac.js";
|
|
7
7
|
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
8
8
|
import { z } from "zod";
|
|
9
9
|
import { dirname } from "node:path";
|
|
@@ -777,4 +777,4 @@ async function runSemanticConceptJudge(input, options) {
|
|
|
777
777
|
//#endregion
|
|
778
778
|
export { diffFindings as a, defaultIsMaterial as i, runSemanticConceptJudge as n, createChatTraceEngine as o, FindingsStore as r, SEMANTIC_CONCEPT_JUDGE_VERSION as t };
|
|
779
779
|
|
|
780
|
-
//# sourceMappingURL=semantic-concept-judge-
|
|
780
|
+
//# sourceMappingURL=semantic-concept-judge-70LxrCSL.js.map
|