@tangle-network/agent-eval 0.187.2 → 0.188.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-uAIQK3Nr.d.ts → agent-profile-CfbuZk8b.d.ts} +4 -4
- package/dist/{agent-profile-uAIQK3Nr.d.ts.map → agent-profile-CfbuZk8b.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +12 -12
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +6 -6
- package/dist/{analyze-runs-DapYZz02.d.ts → analyze-runs-CLz5O01x.d.ts} +5 -5
- package/dist/{analyze-runs-DapYZz02.d.ts.map → analyze-runs-CLz5O01x.d.ts.map} +1 -1
- package/dist/{benchmark-command-DGokT8zd.js → benchmark-command-CdUGku3W.js} +12 -12
- package/dist/benchmark-command-CdUGku3W.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-DlxdTOvw.js → campaign-DZuysBbS.js} +6 -6
- package/dist/{campaign-DlxdTOvw.js.map → campaign-DZuysBbS.js.map} +1 -1
- package/dist/{chat-client-DqPqWzVV.js → chat-client-C4dg7Ng_.js} +2 -2
- package/dist/{chat-client-DqPqWzVV.js.map → chat-client-C4dg7Ng_.js.map} +1 -1
- package/dist/{chat-json-call-C-DnDxEy.js → chat-json-call-JVGbfx95.js} +2 -2
- package/dist/{chat-json-call-C-DnDxEy.js.map → chat-json-call-JVGbfx95.js.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/{client-DCc7a8ZC.d.ts → client-CFj9dfcn.d.ts} +4 -4
- package/dist/{client-DCc7a8ZC.d.ts.map → client-CFj9dfcn.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +6 -6
- package/dist/{counterfactual-CLgrwhkY.d.ts → counterfactual-CPaWN99-.d.ts} +4 -4
- package/dist/{counterfactual-CLgrwhkY.d.ts.map → counterfactual-CPaWN99-.d.ts.map} +1 -1
- package/dist/{counterfactual-Bjq1mlUu.js → counterfactual-DtMWvn7D.js} +2 -2
- package/dist/{counterfactual-Bjq1mlUu.js.map → counterfactual-DtMWvn7D.js.map} +1 -1
- package/dist/{default-registry-DMxeb4GD.d.ts → default-registry-BLsGfnoO.d.ts} +6 -6
- package/dist/{default-registry-DMxeb4GD.d.ts.map → default-registry-BLsGfnoO.d.ts.map} +1 -1
- package/dist/{default-registry-Ca2IWH1X.js → default-registry-BtZ_t-9b.js} +2 -2
- package/dist/{default-registry-Ca2IWH1X.js.map → default-registry-BtZ_t-9b.js.map} +1 -1
- package/dist/{define-agent-eval-C0EqJ9Gz.js → define-agent-eval-ChtddasD.js} +3 -3
- package/dist/{define-agent-eval-C0EqJ9Gz.js.map → define-agent-eval-ChtddasD.js.map} +1 -1
- package/dist/{define-agent-eval-CGBwPPaq.d.ts → define-agent-eval-CsicmDxt.d.ts} +7 -7
- package/dist/{define-agent-eval-CGBwPPaq.d.ts.map → define-agent-eval-CsicmDxt.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +38 -96
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +83 -134
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-BE9lCzzL.js → dspy-rlm-engine-sPX7B5ac.js} +3 -3
- package/dist/{dspy-rlm-engine-BE9lCzzL.js.map → dspy-rlm-engine-sPX7B5ac.js.map} +1 -1
- package/dist/emitter-DbMXUIVP.js +334 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -0
- package/dist/{emitter-Cs0egaFd.d.ts → emitter-HcCmFQl_.d.ts} +37 -6
- package/dist/emitter-HcCmFQl_.d.ts.map +1 -0
- package/dist/{engine-DNlKhIm0.d.ts → engine-Gw2n1irt.d.ts} +15 -5
- package/dist/engine-Gw2n1irt.d.ts.map +1 -0
- package/dist/{eval-campaign-XFHEpN86.js → eval-campaign-CeotQ1ZR.js} +7 -4
- package/dist/{eval-campaign-XFHEpN86.js.map → eval-campaign-CeotQ1ZR.js.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts → exact-types-CrhPoDp8.d.ts} +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts.map → exact-types-CrhPoDp8.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +1 -2
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-BbhvXTFx.js → external-optimizer-process-D3BLpZyR.js} +2 -2
- package/dist/{external-optimizer-process-BbhvXTFx.js.map → external-optimizer-process-D3BLpZyR.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js → external-optimizer-subprocess-BrfsgL92.js} +2 -2
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js.map → external-optimizer-subprocess-BrfsgL92.js.map} +1 -1
- package/dist/{failure-cluster-OldNRoAt.d.ts → failure-cluster-CbgEDcFJ.d.ts} +3 -3
- package/dist/{failure-cluster-OldNRoAt.d.ts.map → failure-cluster-CbgEDcFJ.d.ts.map} +1 -1
- package/dist/failure-taxonomy-CjabTATj.js +538 -0
- package/dist/failure-taxonomy-CjabTATj.js.map +1 -0
- package/dist/{feedback-trajectory-CytTR7u4.d.ts → feedback-trajectory-BJi4t3Vf.d.ts} +6 -6
- package/dist/{feedback-trajectory-CytTR7u4.d.ts.map → feedback-trajectory-BJi4t3Vf.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-CKZa_rnt.d.ts → index-B-s8qDRJ.d.ts} +2 -2
- package/dist/{index-CKZa_rnt.d.ts.map → index-B-s8qDRJ.d.ts.map} +1 -1
- package/dist/{index-YZrbnBSV.d.ts → index-BD1oSbHr.d.ts} +11 -11
- package/dist/{index-YZrbnBSV.d.ts.map → index-BD1oSbHr.d.ts.map} +1 -1
- package/dist/{index-vYNHCV2G.d.ts → index-Ca0_IjFu.d.ts} +7 -7
- package/dist/{index-vYNHCV2G.d.ts.map → index-Ca0_IjFu.d.ts.map} +1 -1
- package/dist/{index-BRsrfSHT.d.ts → index-eTTnPe3r.d.ts} +9 -9
- package/dist/{index-BRsrfSHT.d.ts.map → index-eTTnPe3r.d.ts.map} +1 -1
- package/dist/index.d.ts +383 -68
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1482 -207
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DpOo-3iL.d.ts → insight-report-DeE1solA.d.ts} +4 -4
- package/dist/{insight-report-DpOo-3iL.d.ts.map → insight-report-DeE1solA.d.ts.map} +1 -1
- package/dist/{integrity-Cy9WHAtb.js → integrity-BFP4kjGj.js} +6 -1
- package/dist/integrity-BFP4kjGj.js.map +1 -0
- package/dist/{integrity-rGOfSUle.d.ts → integrity-CaeNeusS.d.ts} +4 -4
- package/dist/integrity-CaeNeusS.d.ts.map +1 -0
- package/dist/jev.d.ts +2 -2
- package/dist/{kind-factory-V79Hw0AU.js → kind-factory-g1ROw67A.js} +15 -4
- package/dist/{kind-factory-V79Hw0AU.js.map → kind-factory-g1ROw67A.js.map} +1 -1
- package/dist/{llm-client-DDhZ29Vm.js → llm-client-DF9bPNYA.js} +2 -2
- package/dist/{llm-client-DDhZ29Vm.js.map → llm-client-DF9bPNYA.js.map} +1 -1
- package/dist/{llm-judge-BVso5x-u.js → llm-judge-DF18J3CR.js} +9 -9
- package/dist/llm-judge-DF18J3CR.js.map +1 -0
- package/dist/{matrix-CPKsRY8Y.d.ts → matrix-BvA1WUpj.d.ts} +2 -2
- package/dist/{matrix-CPKsRY8Y.d.ts.map → matrix-BvA1WUpj.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +4 -4
- package/dist/{mint-BI14xehJ.d.ts → mint-CJFSYBIU.d.ts} +3 -3
- package/dist/{mint-BI14xehJ.d.ts.map → mint-CJFSYBIU.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +67 -12
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +178 -41
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{prime-protocol-CDBcL9NM.d.ts → prime-protocol-CF8Z43Fj.d.ts} +2 -2
- package/dist/{prime-protocol-CDBcL9NM.d.ts.map → prime-protocol-CF8Z43Fj.d.ts.map} +1 -1
- package/dist/{produced-state-Cn9ECr4v.js → produced-state-xsMMjYVc.js} +3 -3
- package/dist/{produced-state-Cn9ECr4v.js.map → produced-state-xsMMjYVc.js.map} +1 -1
- package/dist/{query-D6W6MaGx.d.ts → query-DBUQhehH.d.ts} +3 -3
- package/dist/{query-D6W6MaGx.d.ts.map → query-DBUQhehH.d.ts.map} +1 -1
- package/dist/random-DGEyr3zG.d.ts +83 -0
- package/dist/random-DGEyr3zG.d.ts.map +1 -0
- package/dist/{raw-provider-sink-BQd7mzyT.js → raw-provider-sink-C96CcqeG.js} +19 -56
- package/dist/raw-provider-sink-C96CcqeG.js.map +1 -0
- package/dist/redact-DPoH1-OW.d.ts +158 -0
- package/dist/redact-DPoH1-OW.d.ts.map +1 -0
- package/dist/redact-DxwlUAvh.js +824 -0
- package/dist/redact-DxwlUAvh.js.map +1 -0
- package/dist/{registry-D5Ly3GRV.d.ts → registry-BM2Kdk8T.d.ts} +4 -4
- package/dist/{registry-D5Ly3GRV.d.ts.map → registry-BM2Kdk8T.d.ts.map} +1 -1
- package/dist/{release-confidence-CSp3MsU4.d.ts → release-confidence-CG4Z9QLB.d.ts} +4 -4
- package/dist/{release-confidence-CSp3MsU4.d.ts.map → release-confidence-CG4Z9QLB.d.ts.map} +1 -1
- package/dist/{report-command-CSTjzFGY.js → report-command-DKJ_yAIj.js} +21 -12
- package/dist/{report-command-CSTjzFGY.js.map → report-command-DKJ_yAIj.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/{researcher-D7E92STT.d.ts → researcher-Bjcd_8T_.d.ts} +7 -7
- package/dist/{researcher-D7E92STT.d.ts.map → researcher-Bjcd_8T_.d.ts.map} +1 -1
- package/dist/{reward-hacking-BIQ2BWLG.d.ts → reward-hacking-DrKXJtgH.d.ts} +2 -2
- package/dist/{reward-hacking-BIQ2BWLG.d.ts.map → reward-hacking-DrKXJtgH.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts → rubric-predictive-validity-B14yBAmM.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts.map → rubric-predictive-validity-B14yBAmM.d.ts.map} +1 -1
- package/dist/{run-record-D9wS35fK.d.ts → run-record-xEULpgML.d.ts} +2 -2
- package/dist/{run-record-D9wS35fK.d.ts.map → run-record-xEULpgML.d.ts.map} +1 -1
- package/dist/{schema-CR5cpjQ3.d.ts → schema-2g7trUwH.d.ts} +17 -2
- package/dist/schema-2g7trUwH.d.ts.map +1 -0
- package/dist/schema-CSf6qWgZ.js.map +1 -1
- package/dist/{semantic-concept-judge-Cnt0TWp0.js → semantic-concept-judge-70LxrCSL.js} +5 -5
- package/dist/{semantic-concept-judge-Cnt0TWp0.js.map → semantic-concept-judge-70LxrCSL.js.map} +1 -1
- package/dist/{series-convergence-BnMs_uAr.d.ts → series-convergence-DY7ha0kx.d.ts} +2 -2
- package/dist/{series-convergence-BnMs_uAr.d.ts.map → series-convergence-DY7ha0kx.d.ts.map} +1 -1
- package/dist/{server-DmpcFiVK.js → server-C8XvqCg8.js} +2 -2
- package/dist/{server-DmpcFiVK.js.map → server-C8XvqCg8.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js → skillopt-optimization-method-C60h0Got.js} +5 -5
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js.map → skillopt-optimization-method-C60h0Got.js.map} +1 -1
- package/dist/spans-W0o8pbEg.d.ts +60 -0
- package/dist/spans-W0o8pbEg.d.ts.map +1 -0
- package/dist/{statistical-heldout-DTmhTW8P.d.ts → statistical-heldout-CPv-y6sK.d.ts} +3 -3
- package/dist/{statistical-heldout-DTmhTW8P.d.ts.map → statistical-heldout-CPv-y6sK.d.ts.map} +1 -1
- package/dist/{store-BErPvYBr.d.ts → store-C4avFjIi.d.ts} +2 -2
- package/dist/{store-BErPvYBr.d.ts.map → store-C4avFjIi.d.ts.map} +1 -1
- package/dist/{store-otlp-2gYCy-v0.js → store-otlp-DPgzSQKS.js} +2 -2
- package/dist/{store-otlp-2gYCy-v0.js.map → store-otlp-DPgzSQKS.js.map} +1 -1
- package/dist/{store-tool-spans-B3bX9yqn.d.ts → store-tool-spans-C2lLwDzg.d.ts} +9 -54
- package/dist/store-tool-spans-C2lLwDzg.d.ts.map +1 -0
- package/dist/{store-tool-spans-B1e6Auc2.js → store-tool-spans-ClPxcvh2.js} +4 -4
- package/dist/{store-tool-spans-B1e6Auc2.js.map → store-tool-spans-ClPxcvh2.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-Iy6kXo8x.d.ts → summary-report-SNvJoAvD.d.ts} +3 -3
- package/dist/{summary-report-Iy6kXo8x.d.ts.map → summary-report-SNvJoAvD.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{tool-groups-veQYPiTy.d.ts → tool-groups-CCQqAa16.d.ts} +3 -3
- package/dist/tool-groups-CCQqAa16.d.ts.map +1 -0
- package/dist/{tool-waste-nbHPAxei.js → tool-waste-CJDfxH45.js} +3 -537
- package/dist/tool-waste-CJDfxH45.js.map +1 -0
- package/dist/{tool-waste-D23I0zWm.d.ts → tool-waste-CP5DO383.d.ts} +4 -4
- package/dist/{tool-waste-D23I0zWm.d.ts.map → tool-waste-CP5DO383.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +12 -11
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +8 -8
- package/dist/{trajectory-D7qrNvaN.d.ts → trajectory-Y1fmdlw7.d.ts} +3 -3
- package/dist/{trajectory-D7qrNvaN.d.ts.map → trajectory-Y1fmdlw7.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -3
- package/dist/trajectory-replay/index.js +2 -2
- package/dist/{types--18oHwVF.d.ts → types-5ak2PqnJ.d.ts} +3 -3
- package/dist/{types--18oHwVF.d.ts.map → types-5ak2PqnJ.d.ts.map} +1 -1
- package/dist/{types-BTRcIqFr.d.ts → types-B1vDZ694.d.ts} +4 -4
- package/dist/{types-BTRcIqFr.d.ts.map → types-B1vDZ694.d.ts.map} +1 -1
- package/dist/{types-CBbLtr2J.d.ts → types-C0bWoSKC.d.ts} +10 -11
- package/dist/types-C0bWoSKC.d.ts.map +1 -0
- package/dist/{types-BjB2wZM9.d.ts → types-DDp2Mb8Y.d.ts} +3 -3
- package/dist/{types-BjB2wZM9.d.ts.map → types-DDp2Mb8Y.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/diagnosis.md +2 -3
- package/docs/eval-surface-map.md +1 -1
- package/docs/experiment.md +0 -1
- package/docs/redaction.md +59 -0
- package/docs/trace-contracts.md +102 -0
- package/docs/verdicts.md +1 -1
- package/package.json +3 -2
- package/dist/benchmark-command-DGokT8zd.js.map +0 -1
- package/dist/emitter-Cs0egaFd.d.ts.map +0 -1
- package/dist/emitter-DeQHiDMm.js +0 -257
- package/dist/emitter-DeQHiDMm.js.map +0 -1
- package/dist/engine-DNlKhIm0.d.ts.map +0 -1
- package/dist/experiment-tracker-BKEumQug.js +0 -280
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-URRGP9ek.d.ts +0 -237
- package/dist/experiment-tracker-URRGP9ek.d.ts.map +0 -1
- package/dist/integrity-Cy9WHAtb.js.map +0 -1
- package/dist/integrity-rGOfSUle.d.ts.map +0 -1
- package/dist/llm-judge-BVso5x-u.js.map +0 -1
- package/dist/raw-provider-sink-BQd7mzyT.js.map +0 -1
- package/dist/redact-7Aq1ukl-.js +0 -107
- package/dist/redact-7Aq1ukl-.js.map +0 -1
- package/dist/schema-CR5cpjQ3.d.ts.map +0 -1
- package/dist/store-tool-spans-B3bX9yqn.d.ts.map +0 -1
- package/dist/tool-groups-veQYPiTy.d.ts.map +0 -1
- package/dist/tool-waste-nbHPAxei.js.map +0 -1
- package/dist/types-CBbLtr2J.d.ts.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,51 +1,53 @@
|
|
|
1
1
|
import { a as JudgeError, c as ValidationError, i as ConfigError, n as AgentEvalErrorCode, s as NotFoundError, t as AgentEvalError } from "./errors-DEE6u6ot.js";
|
|
2
2
|
import { C as PendingCostCall, D as costForUsage, E as costForTokenPricing, O as modelPriceKey, S as PaidCallResult, T as RunPaidCallInput, _ as CostReservationExceededError, a as CostChannel, b as CustomTokenPricing, c as CostLedgerHandle, d as CostLedgerPersistenceError, f as CostLedgerSummary, g as CostReceiptInput, h as CostReceiptCaptureError, i as CostCeilingReachedError, l as CostLedgerOptions, m as CostReceipt, n as CostAccountingIncompleteError, o as CostLedger, p as CostProvenance, r as CostCallConflictError, s as CostLedgerFilter, t as ChannelRollup, u as CostLedgerPersistence, v as CostResult, w as PendingCostCallView, x as MaximumCharge, y as CostUsage } from "./cost-ledger-DbQdN3nO.js";
|
|
3
3
|
import { C as verifyAgentProfileCell, h as agentProfileCellKey, i as AgentProfileCellInput, l as AgentProfileJson, m as agentProfileCellHashMaterial, p as AgentProfileSourceInput, r as AgentProfileCell, s as AgentProfileDimensionValue, t as AGENT_PROFILE_KINDS, v as buildAgentProfileCell, x as toAgentProfileJson, y as groupRunsByAgentProfileCell } from "./agent-profile-cell-BHcB-mZX.js";
|
|
4
|
-
import { C as
|
|
5
|
-
import { _ as validateRunRecord, a as RunRecord, c as RunTaskFailure, d as UNKNOWN_MODEL, f as isRunRecord, g as runTaskScore, h as roundTripRunRecord, i as RunOutcome, l as RunTerminalOutcome, m as parseRunRecordSafe, n as RunCostProvenance, o as RunRecordValidationError, p as modelHasSnapshot, r as RunJudgeMetadata, s as RunSplitTag, t as JudgeScoresRecord, u as RunTokenUsage } from "./run-record-
|
|
6
|
-
import { A as DatasetOverview, B as TraceAnalystSpanKind, D as TraceAnalysisStore, F as SpanMatchRecord, G as ViewTraceResult, H as TraceAnalystTraceSummary, I as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, L as TraceAnalystByteBudgets, M as QueryTracesPage, N as SearchSpanResult, P as SearchTraceResult, R as TraceAnalystFilters, U as ViewSpansResult, V as TraceAnalystSpanStatus, W as ViewTraceOversized, _ as ProposalFinding, b as makeFinding, c as AnalystRunInputs, d as AnalystSeverity, f as AnalystUsageReceipt, i as AnalystFinding, j as ErrorCluster, k as DEFAULT_TRACE_ANALYST_BUDGETS, l as AnalystRunResult, n as AnalystContext, p as EvidenceRef, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary, x as makeProposalFinding, y as computeFindingId, z as TraceAnalystSpan } from "./types-
|
|
7
|
-
import {
|
|
8
|
-
import { $ as NoopRawProviderSink, A as createChatClient, B as LlmToolCall, C as ChatResponse, Ct as judgeFamily, D as MockTransportOpts, E as CustomTransportOpts, F as LlmChargeBounds, G as costReceiptFromLlmError, H as LlmToolDefinition, I as LlmMessage, J as stripFencedJson, K as isTransientLlmError, L as LlmResponseError, M as LlmCallMetadata, N as LlmCallRequest, O as OpenAiCompatibleTransportOpts, P as LlmCallResult, S as ChatRequest, St as assertCrossFamily, T as CreateChatClientOpts, U as LlmUsage, V as LlmToolChoice, W as costReceiptFromLlm, Y as FileSystemRawProviderSink, Z as InMemoryRawProviderSink, _ as ScenarioResult, _t as checkServedModel, a as CompletionCriterion, b as ChatCallOpts, bt as CrossFamilyError, c as JudgeInput, ct as AssertServedModelOptions, d as PersonaConfig, dt as ServedModelCheck, f as ProductClientConfig, ft as ServedModelPolicy, g as ScenarioFile, gt as assertServedModels, h as Scenario, ht as assertServedModel, i as CollectedArtifacts, j as LlmCallError, k as SandboxSdkTransportOpts, l as JudgeRubric, lt as ModelSubstitutionError, m as RubricDimension, mt as assertCrossFamilyServed, n as ArtifactResult, nt as RawProviderEvent, o as DriverState, p as RouteMap, pt as ServedModelVerdict, q as maximumChargeForLlmRequest, r as CheckResult, rt as RawProviderSink, s as JudgeFn, st as AssertCrossFamilyServedOptions, t as ArtifactCheck, u as JudgeScore, ut as ServedCrossFamilyError, v as Turn, vt as servedModelAcceptable, w as ChatTransport, x as ChatClient, xt as JudgeFamily, y as TurnResult, yt as AssertCrossFamilyOptions, z as LlmTokenLogprob } from "./types-
|
|
4
|
+
import { C as ToolSpan, D as isToolSpan, E as isLlmSpan, T as isJudgeSpan, a as FAILURE_CLASSES, c as JudgeSpan, f as Run, g as RunStatus, l as LlmSpan, n as BudgetLedgerEntry, o as FailureClass, p as RunCaptureReport, r as BudgetSpec, s as GenericSpan, t as Artifact, v as Span, w as TraceEvent } from "./schema-2g7trUwH.js";
|
|
5
|
+
import { _ as validateRunRecord, a as RunRecord, c as RunTaskFailure, d as UNKNOWN_MODEL, f as isRunRecord, g as runTaskScore, h as roundTripRunRecord, i as RunOutcome, l as RunTerminalOutcome, m as parseRunRecordSafe, n as RunCostProvenance, o as RunRecordValidationError, p as modelHasSnapshot, r as RunJudgeMetadata, s as RunSplitTag, t as JudgeScoresRecord, u as RunTokenUsage } from "./run-record-xEULpgML.js";
|
|
6
|
+
import { A as DatasetOverview, B as TraceAnalystSpanKind, D as TraceAnalysisStore, F as SpanMatchRecord, G as ViewTraceResult, H as TraceAnalystTraceSummary, I as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, L as TraceAnalystByteBudgets, M as QueryTracesPage, N as SearchSpanResult, P as SearchTraceResult, R as TraceAnalystFilters, U as ViewSpansResult, V as TraceAnalystSpanStatus, W as ViewTraceOversized, _ as ProposalFinding, b as makeFinding, c as AnalystRunInputs, d as AnalystSeverity, f as AnalystUsageReceipt, i as AnalystFinding, j as ErrorCluster, k as DEFAULT_TRACE_ANALYST_BUDGETS, l as AnalystRunResult, n as AnalystContext, p as EvidenceRef, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary, x as makeProposalFinding, y as computeFindingId, z as TraceAnalystSpan } from "./types-DDp2Mb8Y.js";
|
|
7
|
+
import { C as inferDomainKeywords, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, W as ExtractedUsage, Y as createBoundedTraceAnalysisStore, g as TraceInsightSuite, h as TraceInsightReadiness, k as tokenizeDomainWords, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore, v as buildTraceInsightContext, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-C2lLwDzg.js";
|
|
8
|
+
import { $ as NoopRawProviderSink, A as createChatClient, B as LlmToolCall, C as ChatResponse, Ct as judgeFamily, D as MockTransportOpts, E as CustomTransportOpts, F as LlmChargeBounds, G as costReceiptFromLlmError, H as LlmToolDefinition, I as LlmMessage, J as stripFencedJson, K as isTransientLlmError, L as LlmResponseError, M as LlmCallMetadata, N as LlmCallRequest, O as OpenAiCompatibleTransportOpts, P as LlmCallResult, S as ChatRequest, St as assertCrossFamily, T as CreateChatClientOpts, U as LlmUsage, V as LlmToolChoice, W as costReceiptFromLlm, Y as FileSystemRawProviderSink, Z as InMemoryRawProviderSink, _ as ScenarioResult, _t as checkServedModel, a as CompletionCriterion, b as ChatCallOpts, bt as CrossFamilyError, c as JudgeInput, ct as AssertServedModelOptions, d as PersonaConfig, dt as ServedModelCheck, f as ProductClientConfig, ft as ServedModelPolicy, g as ScenarioFile, gt as assertServedModels, h as Scenario, ht as assertServedModel, i as CollectedArtifacts, j as LlmCallError, k as SandboxSdkTransportOpts, l as JudgeRubric, lt as ModelSubstitutionError, m as RubricDimension, mt as assertCrossFamilyServed, n as ArtifactResult, nt as RawProviderEvent, o as DriverState, p as RouteMap, pt as ServedModelVerdict, q as maximumChargeForLlmRequest, r as CheckResult, rt as RawProviderSink, s as JudgeFn, st as AssertCrossFamilyServedOptions, t as ArtifactCheck, u as JudgeScore, ut as ServedCrossFamilyError, v as Turn, vt as servedModelAcceptable, w as ChatTransport, x as ChatClient, xt as JudgeFamily, y as TurnResult, yt as AssertCrossFamilyOptions, z as LlmTokenLogprob } from "./types-C0bWoSKC.js";
|
|
9
9
|
import { a as ContinuousCalibrationResult, d as calibrateJudgeContinuous, f as continuousAgreement, h as verbosityBias, i as ContinuousAgreementOptions, l as VerbosityBiasResult, n as CandidateScore, o as GoldenItem, r as ContinuousAgreement, t as CalibrationResult, u as calibrateJudge } from "./judge-calibration-DFtEMlde.js";
|
|
10
|
-
import { a as CorpusAgreementPerDimension, c as corpusInterRaterAgreement, i as CorpusAgreementOptions, l as corpusInterRaterAgreementFromJudgeScores, n as SeriesConvergenceResult, o as CorpusAgreementReport, r as analyzeSeries, s as CorpusScoreRecord, t as SeriesConvergenceOptions, u as interRaterReliability } from "./series-convergence-
|
|
10
|
+
import { a as CorpusAgreementPerDimension, c as corpusInterRaterAgreement, i as CorpusAgreementOptions, l as corpusInterRaterAgreementFromJudgeScores, n as SeriesConvergenceResult, o as CorpusAgreementReport, r as analyzeSeries, s as CorpusScoreRecord, t as SeriesConvergenceOptions, u as interRaterReliability } from "./series-convergence-DY7ha0kx.js";
|
|
11
11
|
import { a as partialCredit, c as spearmanR, d as weightedMean, i as confidenceInterval, l as summarizeNumberSeries, n as WeightedCompositeInput, o as pearsonR, r as WeightedCompositeResult, s as ranks, t as SeriesDistribution, u as weightedComposite } from "./descriptive-B2iPaT9J.js";
|
|
12
12
|
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-DIWHvysC.js";
|
|
13
|
-
import { $t as wilson, A as PairArmsOptions, Bt as eProcess, C as hashJson, Ct as verifyAttestation, D as ComparePairedArmsOptions, F as PairedCorrectness, Gt as ScoreRiskDifferenceResult, Ht as McNemarResult, I as PairedMetricDelta, It as EProcess, Jt as pairedBinaryScale, Kt as isBinaryOutcomeVector, L as comparePairedArms, Lt as EProcessOptions, M as PairRunRecordsResult, N as PairedArmRow, O as MatchedPair, P as PairedArmsComparison, Qt as passAtK, R as pairArms, Rt as EProcessState, St as attest, Ut as ProportionInterval, Vt as ExactRiskDifferenceResult, Wt as RiskDifferenceResult, Xt as pairedRiskDifferenceExact, Yt as pairedRiskDifference, Zt as pairedRiskDifferenceScore, bt as AttestationVerification, et as Objective, j as PairArmsResult, k as MatchedRunRecordPair, nt as dominates, qt as mcnemar, rt as paretoFrontier, tt as ParetoResult, vt as ATTESTATION_ALGORITHM, w as manifestContentDigest, xt as AttestedReport, yt as AttestationProvenance, z as pairRunRecords, zt as EProcessStep } from "./statistical-heldout-
|
|
13
|
+
import { $t as wilson, A as PairArmsOptions, Bt as eProcess, C as hashJson, Ct as verifyAttestation, D as ComparePairedArmsOptions, F as PairedCorrectness, Gt as ScoreRiskDifferenceResult, Ht as McNemarResult, I as PairedMetricDelta, It as EProcess, Jt as pairedBinaryScale, Kt as isBinaryOutcomeVector, L as comparePairedArms, Lt as EProcessOptions, M as PairRunRecordsResult, N as PairedArmRow, O as MatchedPair, P as PairedArmsComparison, Qt as passAtK, R as pairArms, Rt as EProcessState, St as attest, Ut as ProportionInterval, Vt as ExactRiskDifferenceResult, Wt as RiskDifferenceResult, Xt as pairedRiskDifferenceExact, Yt as pairedRiskDifference, Zt as pairedRiskDifferenceScore, bt as AttestationVerification, et as Objective, j as PairArmsResult, k as MatchedRunRecordPair, nt as dominates, qt as mcnemar, rt as paretoFrontier, tt as ParetoResult, vt as ATTESTATION_ALGORITHM, w as manifestContentDigest, xt as AttestedReport, yt as AttestationProvenance, z as pairRunRecords, zt as EProcessStep } from "./statistical-heldout-CPv-y6sK.js";
|
|
14
14
|
import { _ as pairedSignTest, a as PairedPromotionDecision, c as BOOTSTRAP_GATE_MIN_N, d as PairedBootstrapResult, f as PairedSignTestResult, g as pairedDeltaTieFraction, h as pairedBootstrap, i as PairedMcNemarEvidence, l as DECISION_PAIRED_DELTA_STATISTIC, m as SignTestAlternative, n as PairedDecisionShape, o as PairedPromotionDecisionOptions, p as PairedTTestResult, r as PairedDecisionStatistic, s as decidePairedPromotion, t as PairedDecisionMethod, u as PairedBootstrapOptions, v as pairedTTest } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
15
|
-
import {
|
|
16
|
-
import { C as RankTestMethod, D as WilcoxonSignedRankResult, E as WILCOXON_EXACT_MAX_N, O as mannWhitneyU, S as MannWhitneyResult, T as RankTestOptions, _ as bootstrapCi, a as ReleaseConfidenceIssue, b as MANN_WHITNEY_EXACT_MAX_STATES, f as evaluateReleaseConfidence, g as Verdict, i as ReleaseConfidenceInput, k as wilcoxonSignedRank, m as BootstrapResult, o as ReleaseConfidenceMetrics, p as BootstrapOptions, s as ReleaseConfidenceScorecard, t as ActionableSideInfo, u as ReleaseTraceEvidence, w as RankTestMethodRequest, x as MANN_WHITNEY_EXACT_MAX_WORK, y as DEFAULT_PERMUTATIONS } from "./release-confidence-
|
|
17
|
-
import { R as Scenario$1, S as JudgeConfig, _ as GateDecision, a as CampaignResult, w as JudgeScore$1 } from "./types-
|
|
18
|
-
import { Cr as scoreRedTeamOutput, Ct as RequirementCheck, Dr as CanaryOptions, Dt as createLlmCorrectnessChecker, Er as CanaryKind, Et as completionVerdict, Fa as RunCampaignOptions, Ia as runCampaign, Mi as quotaExhaustedUntil, Na as CampaignCellFailureReceipt, Ni as transientDispatchFailure, Oi as LlmJudgeOptions, Or as CanaryReport, Ot as verifyCompletion, Pa as CampaignCellRetryPolicy, Sr as redTeamReport, St as ProducedState, Tr as CanaryEvaluation, Tt as TaskGold, _r as RedTeamCase, _t as CompletionRequirement, br as RedTeamReport, bt as LlmCorrectnessCheckerOpts, ct as BackendIntegrityError, dt as summarizeBackendIntegrity, ft as ArtifactEventLike, gr as DEFAULT_RED_TEAM_CORPUS, gt as extractProducedState, ht as ToolCallEventLike, ki as llmJudge, kr as runCanaries, lt as BackendIntegrityReport, mt as RuntimeEventLike, pt as ProposalEventLike, ut as assertRealBackend, vr as RedTeamCategory, vt as CompletionVerdict, wr as CanaryAlert, wt as SatisfiedBy, xr as redTeamDataset, xt as ProducedProposal, yr as RedTeamFinding, yt as CorrectnessChecker } from "./index-
|
|
19
|
-
import { a as ProfileAxisSpec, c as expandProfileAxes, i as HarnessType, l as harnessAxisOf, n as CODING_HARNESSES, o as agentProfileHash, r as HARNESS_NATIVE_MODEL, s as agentProfileId, t as AgentProfile } from "./agent-profile-
|
|
15
|
+
import { a as requiredPairedSampleSize, i as pairedMde, n as mcnemarPower, o as requiredSampleSize, r as mcnemarRequiredN, t as mulberry32 } from "./random-DGEyr3zG.js";
|
|
16
|
+
import { C as RankTestMethod, D as WilcoxonSignedRankResult, E as WILCOXON_EXACT_MAX_N, O as mannWhitneyU, S as MannWhitneyResult, T as RankTestOptions, _ as bootstrapCi, a as ReleaseConfidenceIssue, b as MANN_WHITNEY_EXACT_MAX_STATES, f as evaluateReleaseConfidence, g as Verdict, i as ReleaseConfidenceInput, k as wilcoxonSignedRank, m as BootstrapResult, o as ReleaseConfidenceMetrics, p as BootstrapOptions, s as ReleaseConfidenceScorecard, t as ActionableSideInfo, u as ReleaseTraceEvidence, w as RankTestMethodRequest, x as MANN_WHITNEY_EXACT_MAX_WORK, y as DEFAULT_PERMUTATIONS } from "./release-confidence-CG4Z9QLB.js";
|
|
17
|
+
import { R as Scenario$1, S as JudgeConfig, _ as GateDecision, a as CampaignResult, w as JudgeScore$1 } from "./types-B1vDZ694.js";
|
|
18
|
+
import { Cr as scoreRedTeamOutput, Ct as RequirementCheck, Dr as CanaryOptions, Dt as createLlmCorrectnessChecker, Er as CanaryKind, Et as completionVerdict, Fa as RunCampaignOptions, Ia as runCampaign, Mi as quotaExhaustedUntil, Na as CampaignCellFailureReceipt, Ni as transientDispatchFailure, Oi as LlmJudgeOptions, Or as CanaryReport, Ot as verifyCompletion, Pa as CampaignCellRetryPolicy, Sr as redTeamReport, St as ProducedState, Tr as CanaryEvaluation, Tt as TaskGold, _r as RedTeamCase, _t as CompletionRequirement, br as RedTeamReport, bt as LlmCorrectnessCheckerOpts, ct as BackendIntegrityError, dt as summarizeBackendIntegrity, ft as ArtifactEventLike, gr as DEFAULT_RED_TEAM_CORPUS, gt as extractProducedState, ht as ToolCallEventLike, ki as llmJudge, kr as runCanaries, lt as BackendIntegrityReport, mt as RuntimeEventLike, pt as ProposalEventLike, ut as assertRealBackend, vr as RedTeamCategory, vt as CompletionVerdict, wr as CanaryAlert, wt as SatisfiedBy, xr as redTeamDataset, xt as ProducedProposal, yr as RedTeamFinding, yt as CorrectnessChecker } from "./index-eTTnPe3r.js";
|
|
19
|
+
import { a as ProfileAxisSpec, c as expandProfileAxes, i as HarnessType, l as harnessAxisOf, n as CODING_HARNESSES, o as agentProfileHash, r as HARNESS_NATIVE_MODEL, s as agentProfileId, t as AgentProfile } from "./agent-profile-CfbuZk8b.js";
|
|
20
20
|
import { i as DatasetSplit, n as DatasetManifest, r as DatasetScenario, t as Dataset } from "./dataset-DQqhOCPt.js";
|
|
21
|
-
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, s as TraceStore, t as EventFilter } from "./store-
|
|
21
|
+
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, s as TraceStore, t as EventFilter } from "./store-C4avFjIi.js";
|
|
22
22
|
import { C as defineEquivalenceCheck, S as buildEquivalenceRecord, _ as StrategyChecker, a as CheckerIdentity, b as VerificationStrategyProfile, c as EquivalenceCheckDefinition, d as EquivalenceCheckerInput, f as EquivalenceCheckerResult, g as EquivalenceRecord, h as EquivalenceProtocolError, i as equivalenceVerdict, l as EquivalenceCheckSpec, m as EquivalenceObligationStatus, n as VerdictCertification, o as CheckerOutcome, p as EquivalenceObligation, r as certificationEvidenceDigest, s as EquivalenceArm, t as DefaultVerdict, u as EquivalenceChecker, v as VERIFICATION_STRATEGIES, w as runEquivalenceCheck, x as VerificationStrategySource, y as VERIFICATION_STRATEGY_SOURCES } from "./verdict-E4eRNf7-.js";
|
|
23
23
|
import { a as MultiLayerVerifier, c as VerifyOptions, i as LayerStatus, l as gradeSemanticStatus, n as Layer, o as Severity, r as LayerResult, s as VerificationReport, t as Finding } from "./multi-layer-verifier-BUaQ4C17.js";
|
|
24
|
-
import { C as HeldOutGateConfig, S as HeldOutGate, T as SplitCoverage, _ as paretoChart, a as ParetoPoint, b as GateDecision$1, d as ResearchReportOptions, g as gainHistogram, h as SummaryTableRow, m as SummaryTableOptions, n as GainDistributionFigureSpec, p as SummaryTable, r as GainDistributionOptions, s as ResearchReport, t as GainDistributionBin, w as HeldOutGateRejectionCode, x as GateEvidence, y as summaryTable } from "./summary-report-
|
|
25
|
-
import { a as FAILURE_BLAME, c as FailureContext, d as INFRA_FAILURE_BLAMES, f as classifyFailure, i as DEFAULT_FAILURE_REASON_RULES, l as FailureReasonRule, m as failureBlame, o as FailureBlame, p as classifyFailureReason, r as failureClusterView, s as FailureClassification, u as FailureRule } from "./failure-cluster-
|
|
26
|
-
import { o as InsightReport } from "./insight-report-
|
|
27
|
-
import { i as ExactAnalystRunEvent, l as ExactCapableAnalyst, o as ExactAnalystRunResult } from "./exact-types-
|
|
28
|
-
import { c as RegistryRunOpts, i as BudgetPolicy, n as AnalystRegistry, r as AnalystRegistryOptions, s as ExactRegistryRunOpts } from "./registry-
|
|
29
|
-
import { _ as SelfImproveMethodResult, a as DefinedAgentEval, c as SelfImproveMethodOptions, d as SelfImproveProposerOptions, f as SelfImproveProposerResult, g as SelfImproveMethodProvenance, h as selfImprove, i as DefineAgentEvalOptions, l as SelfImproveOptions, n as AgentEvalEvaluateOptions, o as defineAgentEval, p as SelfImproveResult, r as AgentEvalImproveOptions, s as SelfImproveBudget, t as AgentEvalAgent, u as SelfImproveProgressEvent } from "./define-agent-eval-
|
|
30
|
-
import { a as createTraceAnalyst, i as TraceAnalystDefinition, n as buildDefaultAnalystRegistry, r as CreateTraceAnalystOptions, t as DefaultAnalystRegistryOptions } from "./default-registry-
|
|
31
|
-
import { i as analyzeRuns, t as AnalyzeRunsOptions } from "./analyze-runs-
|
|
32
|
-
import { i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS } from "./engine-
|
|
33
|
-
import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as ConceptFinding, E as createChatTraceEngine, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine } from "./index-
|
|
24
|
+
import { C as HeldOutGateConfig, S as HeldOutGate, T as SplitCoverage, _ as paretoChart, a as ParetoPoint, b as GateDecision$1, d as ResearchReportOptions, g as gainHistogram, h as SummaryTableRow, m as SummaryTableOptions, n as GainDistributionFigureSpec, p as SummaryTable, r as GainDistributionOptions, s as ResearchReport, t as GainDistributionBin, w as HeldOutGateRejectionCode, x as GateEvidence, y as summaryTable } from "./summary-report-SNvJoAvD.js";
|
|
25
|
+
import { a as FAILURE_BLAME, c as FailureContext, d as INFRA_FAILURE_BLAMES, f as classifyFailure, i as DEFAULT_FAILURE_REASON_RULES, l as FailureReasonRule, m as failureBlame, o as FailureBlame, p as classifyFailureReason, r as failureClusterView, s as FailureClassification, u as FailureRule } from "./failure-cluster-CbgEDcFJ.js";
|
|
26
|
+
import { o as InsightReport } from "./insight-report-DeE1solA.js";
|
|
27
|
+
import { i as ExactAnalystRunEvent, l as ExactCapableAnalyst, o as ExactAnalystRunResult } from "./exact-types-CrhPoDp8.js";
|
|
28
|
+
import { c as RegistryRunOpts, i as BudgetPolicy, n as AnalystRegistry, r as AnalystRegistryOptions, s as ExactRegistryRunOpts } from "./registry-BM2Kdk8T.js";
|
|
29
|
+
import { _ as SelfImproveMethodResult, a as DefinedAgentEval, c as SelfImproveMethodOptions, d as SelfImproveProposerOptions, f as SelfImproveProposerResult, g as SelfImproveMethodProvenance, h as selfImprove, i as DefineAgentEvalOptions, l as SelfImproveOptions, n as AgentEvalEvaluateOptions, o as defineAgentEval, p as SelfImproveResult, r as AgentEvalImproveOptions, s as SelfImproveBudget, t as AgentEvalAgent, u as SelfImproveProgressEvent } from "./define-agent-eval-CsicmDxt.js";
|
|
30
|
+
import { a as createTraceAnalyst, i as TraceAnalystDefinition, n as buildDefaultAnalystRegistry, r as CreateTraceAnalystOptions, t as DefaultAnalystRegistryOptions } from "./default-registry-BLsGfnoO.js";
|
|
31
|
+
import { i as analyzeRuns, t as AnalyzeRunsOptions } from "./analyze-runs-CLz5O01x.js";
|
|
32
|
+
import { i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS } from "./engine-Gw2n1irt.js";
|
|
33
|
+
import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as ConceptFinding, E as createChatTraceEngine, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine } from "./index-Ca0_IjFu.js";
|
|
34
34
|
import { a as MintedRolloutLine } from "./schema-BzWDXhOR.js";
|
|
35
|
-
import { i as BenchmarkEvaluation } from "./types
|
|
36
|
-
import {
|
|
35
|
+
import { i as BenchmarkEvaluation } from "./types-5ak2PqnJ.js";
|
|
36
|
+
import { a as SpanHandle, n as CaptureWrite, o as TraceEmitter, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-HcCmFQl_.js";
|
|
37
37
|
import { n as SandboxDriver } from "./sandbox-harness-BlSOu4LX.js";
|
|
38
|
+
import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-DPoH1-OW.js";
|
|
38
39
|
import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-BhsrMupG.js";
|
|
39
|
-
import { n as TrajectoryStep, r as buildTrajectory, t as Trajectory } from "./trajectory-
|
|
40
|
-
import { a as runCounterfactual, i as CounterfactualRunner, n as CounterfactualMutation, r as CounterfactualResult, t as CounterfactualContext } from "./counterfactual-
|
|
41
|
-
import { A as AnalystFindingDigest, B as ControlRunResult, C as createFeedbackTrajectory, D as renderPreferenceMemoryMarkdown, E as feedbackTrajectoryToOptimizerRow, F as ControlActionOutcome, G as StopDecision, H as ControlRuntimeError, I as ControlBudget, J as subjectiveEval, K as objectiveEval, L as ControlContext, M as AnalystRunDigest, N as analystFindingDigest, O as summarizePreferenceMemory, P as analystRunDigest, R as ControlDecision, S as controlRunToFeedbackTrajectory, T as feedbackTrajectoriesToOptimizerRows, U as ControlSeverity, V as ControlRuntimeConfig, W as ControlStep, _ as PreferenceMemoryEntry, a as FeedbackLabel, b as analystRunToReviewRequests, c as FeedbackOptimizerRow, d as FeedbackTask, f as FeedbackTrajectory, g as InMemoryFeedbackTrajectoryStore, h as FileSystemFeedbackTrajectoryStore, i as FeedbackAttempt, j as AnalystReviewDecision, k as withAssignedFeedbackSplit, l as FeedbackOutcome, m as FeedbackTrajectoryStore, n as AnalystReviewRequest, o as FeedbackLabelKind, p as FeedbackTrajectoryFilter, q as runAgentControlLoop, r as FeedbackArtifactType, s as FeedbackLabelSource, t as AnalystFeedbackTrajectoryOptions, u as FeedbackSplitPolicy, v as ProposedSideEffect, w as feedbackTrajectoriesToDatasetScenarios, x as assignFeedbackSplit, y as analystRunToFeedbackTrajectory, z as ControlEvalResult } from "./feedback-trajectory-
|
|
40
|
+
import { n as TrajectoryStep, r as buildTrajectory, t as Trajectory } from "./trajectory-Y1fmdlw7.js";
|
|
41
|
+
import { a as runCounterfactual, i as CounterfactualRunner, n as CounterfactualMutation, r as CounterfactualResult, t as CounterfactualContext } from "./counterfactual-CPaWN99-.js";
|
|
42
|
+
import { A as AnalystFindingDigest, B as ControlRunResult, C as createFeedbackTrajectory, D as renderPreferenceMemoryMarkdown, E as feedbackTrajectoryToOptimizerRow, F as ControlActionOutcome, G as StopDecision, H as ControlRuntimeError, I as ControlBudget, J as subjectiveEval, K as objectiveEval, L as ControlContext, M as AnalystRunDigest, N as analystFindingDigest, O as summarizePreferenceMemory, P as analystRunDigest, R as ControlDecision, S as controlRunToFeedbackTrajectory, T as feedbackTrajectoriesToOptimizerRows, U as ControlSeverity, V as ControlRuntimeConfig, W as ControlStep, _ as PreferenceMemoryEntry, a as FeedbackLabel, b as analystRunToReviewRequests, c as FeedbackOptimizerRow, d as FeedbackTask, f as FeedbackTrajectory, g as InMemoryFeedbackTrajectoryStore, h as FileSystemFeedbackTrajectoryStore, i as FeedbackAttempt, j as AnalystReviewDecision, k as withAssignedFeedbackSplit, l as FeedbackOutcome, m as FeedbackTrajectoryStore, n as AnalystReviewRequest, o as FeedbackLabelKind, p as FeedbackTrajectoryFilter, q as runAgentControlLoop, r as FeedbackArtifactType, s as FeedbackLabelSource, t as AnalystFeedbackTrajectoryOptions, u as FeedbackSplitPolicy, v as ProposedSideEffect, w as feedbackTrajectoriesToDatasetScenarios, x as assignFeedbackSplit, y as analystRunToFeedbackTrajectory, z as ControlEvalResult } from "./feedback-trajectory-BJi4t3Vf.js";
|
|
42
43
|
import { y as OUTPUT_VALUE } from "./attribute-vocabulary-DLJ6303h.js";
|
|
43
|
-
import { a as RunIntegrityReport, o as assertRunCaptured, t as RunIntegrityError } from "./integrity-
|
|
44
|
-
import { c as judgeSpans, f as runsForScenario, i as argHash } from "./query-
|
|
45
|
-
import { a as SteeringChange, c as CampaignRunContext, d as CampaignVariant, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, l as CampaignRunOutcome, m as FailedRun, n as ExperimentResult, o as CampaignFactoryParams, p as EvalCampaignResult, r as FailureMode, s as CampaignIntegrityPolicy, t as ExperimentPlan, u as CampaignScenario } from "./researcher-
|
|
46
|
-
import { i as mintRolloutRows, n as MintRolloutResult, s as ScorePreference, t as MintRolloutOptions } from "./mint-
|
|
47
|
-
import "./index-
|
|
48
|
-
import { _ as iqr, a as ToolStats, c as computeToolUseMetrics, d as judgeAgreementView, i as toolWasteView, m as budgetBreachView, o as ToolUseMetrics, s as ToolUseOptions } from "./tool-waste-
|
|
44
|
+
import { a as RunIntegrityReport, o as assertRunCaptured, t as RunIntegrityError } from "./integrity-CaeNeusS.js";
|
|
45
|
+
import { c as judgeSpans, f as runsForScenario, i as argHash } from "./query-DBUQhehH.js";
|
|
46
|
+
import { a as SteeringChange, c as CampaignRunContext, d as CampaignVariant, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, l as CampaignRunOutcome, m as FailedRun, n as ExperimentResult, o as CampaignFactoryParams, p as EvalCampaignResult, r as FailureMode, s as CampaignIntegrityPolicy, t as ExperimentPlan, u as CampaignScenario } from "./researcher-Bjcd_8T_.js";
|
|
47
|
+
import { i as mintRolloutRows, n as MintRolloutResult, s as ScorePreference, t as MintRolloutOptions } from "./mint-CJFSYBIU.js";
|
|
48
|
+
import "./index-B-s8qDRJ.js";
|
|
49
|
+
import { _ as iqr, a as ToolStats, c as computeToolUseMetrics, d as judgeAgreementView, i as toolWasteView, m as budgetBreachView, o as ToolUseMetrics, s as ToolUseOptions } from "./tool-waste-CP5DO383.js";
|
|
50
|
+
import { SpanKind } from "@tangle-network/agent-trace-contract";
|
|
49
51
|
//#region src/statistics/effect-sizes.d.ts
|
|
50
52
|
/**
|
|
51
53
|
* Standardized effect sizes: Cohen's d family and Cliff's delta, with the
|
|
@@ -382,95 +384,224 @@ declare function diffScorecard(scorecard: Scorecard, opts?: DiffScorecardOptions
|
|
|
382
384
|
*/
|
|
383
385
|
declare function formatScorecardDiff(diff: ScorecardDiff): string;
|
|
384
386
|
//#endregion
|
|
385
|
-
//#region src/trace-contracts.d.ts
|
|
387
|
+
//#region src/trace-contracts/types.d.ts
|
|
386
388
|
/**
|
|
387
|
-
* Minimal structural span the checker reads.
|
|
388
|
-
* (trace/schema)
|
|
389
|
-
*
|
|
389
|
+
* Minimal structural span the checker reads. The eval-side `Span`
|
|
390
|
+
* (trace/schema), the otel-bridge `ExportableSpan`, and the output of
|
|
391
|
+
* {@link contractSpansFromOtlp} all satisfy it.
|
|
390
392
|
*/
|
|
391
393
|
interface ContractSpan {
|
|
392
394
|
spanId?: string;
|
|
395
|
+
parentSpanId?: string | null;
|
|
393
396
|
name?: string;
|
|
397
|
+
/** Eval kinds (`tool`, `llm`, ...) or contract kinds (`TOOL`, `LLM`, ...). */
|
|
394
398
|
kind?: string;
|
|
395
399
|
startedAt?: number;
|
|
400
|
+
endedAt?: number;
|
|
396
401
|
status?: string;
|
|
397
402
|
error?: string;
|
|
398
403
|
/** Typed field on eval-side ToolSpans; OTLP flattenings drop it (see
|
|
399
|
-
*
|
|
404
|
+
* {@link contractSpanToolName}). */
|
|
400
405
|
toolName?: string;
|
|
406
|
+
/** Typed field on eval-side LlmSpans and ExportableSpans. */
|
|
407
|
+
model?: string;
|
|
408
|
+
inputTokens?: number;
|
|
409
|
+
outputTokens?: number;
|
|
410
|
+
/** Typed field on eval-side ToolSpans. */
|
|
411
|
+
args?: unknown;
|
|
412
|
+
/** False when the source observed the call but did not capture its arguments. */
|
|
413
|
+
argsCaptured?: boolean;
|
|
401
414
|
attributes?: Record<string, unknown>;
|
|
402
415
|
}
|
|
416
|
+
/**
|
|
417
|
+
* The run a `run` rule reads. The eval-side `Run` satisfies it. When no run is
|
|
418
|
+
* passed, the checker derives it from the trace's single root span; a trace
|
|
419
|
+
* with no unique root leaves the run unknown and every `run` rule fails.
|
|
420
|
+
*/
|
|
421
|
+
interface ContractRun {
|
|
422
|
+
status?: string;
|
|
423
|
+
startedAt?: number;
|
|
424
|
+
endedAt?: number;
|
|
425
|
+
}
|
|
403
426
|
/** JSON-safe RegExp form — what the builder normalizes RegExp matchers to. */
|
|
404
427
|
interface SerializedRegex {
|
|
405
428
|
$regex: string;
|
|
406
429
|
flags: string;
|
|
407
430
|
}
|
|
408
|
-
|
|
431
|
+
/** Matches when the value equals one of the listed strings. */
|
|
432
|
+
interface OneOfMatcher {
|
|
433
|
+
oneOf: string[];
|
|
434
|
+
}
|
|
435
|
+
type TextMatcher = string | RegExp | SerializedRegex | OneOfMatcher;
|
|
409
436
|
/**
|
|
410
|
-
* Proposition over one span. All specified fields must match (AND)
|
|
411
|
-
* At least one field is required — an empty predicate would
|
|
412
|
-
* span and is rejected.
|
|
437
|
+
* Proposition over one span. All specified fields must match (AND); `not`
|
|
438
|
+
* must NOT match. At least one field is required — an empty predicate would
|
|
439
|
+
* match every span and is rejected.
|
|
413
440
|
*
|
|
414
|
-
* `
|
|
415
|
-
*
|
|
416
|
-
* (OTLP
|
|
417
|
-
*
|
|
441
|
+
* `kind` compares against the span's contract kind (`TOOL`, `LLM`, `AGENT`,
|
|
442
|
+
* ...), resolved by agent-trace-contract's classifier, so eval kinds
|
|
443
|
+
* (`tool`, `llm`) and OTLP spans that declare no kind are read the same way.
|
|
444
|
+
*
|
|
445
|
+
* `tool` resolves through {@link contractSpanToolName}; `model` through the
|
|
446
|
+
* typed `model` field and the gen_ai model attributes.
|
|
418
447
|
*
|
|
419
448
|
* `attr` values match by strict equality, or regex-test when the value is a
|
|
420
449
|
* RegExp/SerializedRegex and the attribute is a string. Structured attribute
|
|
421
|
-
* values need `custom
|
|
450
|
+
* values need `custom`, or an `argument` rule for tool arguments.
|
|
422
451
|
*/
|
|
423
452
|
interface SpanPredicate {
|
|
424
453
|
name?: TextMatcher;
|
|
425
454
|
tool?: TextMatcher;
|
|
455
|
+
model?: TextMatcher;
|
|
456
|
+
kind?: SpanKind;
|
|
426
457
|
attr?: Record<string, unknown>;
|
|
458
|
+
not?: SpanPredicate;
|
|
427
459
|
custom?: (span: ContractSpan) => boolean;
|
|
428
460
|
/** Stamped by the builder when `custom` is present. Survives JSON while
|
|
429
461
|
* the function does not, so evaluation of a deserialized contract throws
|
|
430
462
|
* instead of silently dropping the check. */
|
|
431
463
|
requiresCustom?: true;
|
|
432
464
|
}
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
465
|
+
declare const ORDER_MODES: readonly ['start-order', 'finish-before-start', 'all-occurrences'];
|
|
466
|
+
type OrderMode = (typeof ORDER_MODES)[number];
|
|
467
|
+
declare const ARGUMENT_OCCURRENCES: readonly ['first', 'last', 'any', 'all'];
|
|
468
|
+
type ArgumentOccurrence = (typeof ARGUMENT_OCCURRENCES)[number];
|
|
469
|
+
declare const ARGUMENT_TYPES: readonly ['string', 'number', 'boolean', 'object', 'array', 'null'];
|
|
470
|
+
type ArgumentType = (typeof ARGUMENT_TYPES)[number];
|
|
471
|
+
type ArgumentCheck = {
|
|
472
|
+
op: 'exists';
|
|
473
|
+
} | {
|
|
474
|
+
op: 'equals';
|
|
475
|
+
value: unknown;
|
|
476
|
+
} | {
|
|
477
|
+
op: 'oneOf';
|
|
478
|
+
values: unknown[];
|
|
479
|
+
} | {
|
|
480
|
+
op: 'type';
|
|
481
|
+
type: ArgumentType;
|
|
482
|
+
};
|
|
483
|
+
interface RunChecks {
|
|
484
|
+
/** The run reached a terminal status (not `running`) with a recorded end. */
|
|
485
|
+
requireCompleted?: boolean;
|
|
486
|
+
/** The run's status must be one of these. */
|
|
487
|
+
allowedStatuses?: RunStatus[];
|
|
488
|
+
/** End minus start, in milliseconds. An unknown duration fails. */
|
|
489
|
+
maxDurationMs?: number;
|
|
490
|
+
}
|
|
491
|
+
type ContractRule = {
|
|
492
|
+
kind: 'always' | 'never' | 'eventually';
|
|
493
|
+
label: string;
|
|
494
|
+
p: SpanPredicate;
|
|
495
|
+
} | {
|
|
496
|
+
kind: 'precedes';
|
|
497
|
+
label: string;
|
|
498
|
+
a: SpanPredicate;
|
|
499
|
+
b: SpanPredicate;
|
|
500
|
+
order?: OrderMode;
|
|
501
|
+
} | {
|
|
502
|
+
kind: 'neverUnless';
|
|
503
|
+
label: string;
|
|
504
|
+
p: SpanPredicate;
|
|
505
|
+
prior: SpanPredicate;
|
|
506
|
+
order?: OrderMode;
|
|
507
|
+
} | {
|
|
508
|
+
kind: 'atMost';
|
|
437
509
|
label: string;
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
510
|
+
p: SpanPredicate;
|
|
511
|
+
max: number;
|
|
512
|
+
} | {
|
|
513
|
+
kind: 'tokensAtMost';
|
|
514
|
+
label: string;
|
|
515
|
+
p: SpanPredicate;
|
|
516
|
+
max: number;
|
|
517
|
+
} | ({
|
|
518
|
+
kind: 'run';
|
|
519
|
+
label: string;
|
|
520
|
+
} & RunChecks) | {
|
|
521
|
+
kind: 'argument';
|
|
522
|
+
label: string;
|
|
523
|
+
p: SpanPredicate;
|
|
524
|
+
/** RFC 6901 JSON Pointer into the call's arguments; `''` is the whole value. */
|
|
525
|
+
pointer: string;
|
|
526
|
+
check: ArgumentCheck;
|
|
527
|
+
/** Which matching calls must satisfy the check. Default `all`. */
|
|
528
|
+
occurrence?: ArgumentOccurrence;
|
|
529
|
+
};
|
|
530
|
+
/** One legitimate alternate path. The contract passes only when the base
|
|
531
|
+
* rules pass and at least one alternative passes completely. */
|
|
532
|
+
interface ContractAlternative {
|
|
533
|
+
id: string;
|
|
534
|
+
rules: ContractRule[];
|
|
446
535
|
}
|
|
447
536
|
/** Serializable plain object — `traceContract(name)....build()` output. */
|
|
448
537
|
interface TraceContract {
|
|
449
538
|
name: string;
|
|
539
|
+
/** What the contract protects, for reports and `explainTraceContract`. */
|
|
540
|
+
description?: string;
|
|
541
|
+
/** Evaluate only the span matching `root` and its descendants. Exactly one
|
|
542
|
+
* span must match; zero or several make every rule an error. */
|
|
543
|
+
scope?: {
|
|
544
|
+
root: SpanPredicate;
|
|
545
|
+
};
|
|
450
546
|
rules: ContractRule[];
|
|
547
|
+
alternatives?: ContractAlternative[];
|
|
451
548
|
}
|
|
549
|
+
type ContractStatus = 'pass' | 'fail' | 'error';
|
|
452
550
|
interface ContractViolation {
|
|
453
551
|
rule: string;
|
|
454
552
|
spanId?: string;
|
|
455
553
|
detail: string;
|
|
456
554
|
}
|
|
555
|
+
/** What happened to one rule. `error` means it could not be evaluated. */
|
|
556
|
+
interface ContractRuleExecution {
|
|
557
|
+
rule: string;
|
|
558
|
+
/** Set for rules inside `alternatives`. */
|
|
559
|
+
alternative?: string;
|
|
560
|
+
status: ContractStatus;
|
|
561
|
+
violations: number;
|
|
562
|
+
error?: string;
|
|
563
|
+
}
|
|
457
564
|
interface ContractVerdict extends DefaultVerdict {
|
|
458
565
|
/** Contract name — keys this verdict in multi-contract reports. */
|
|
459
566
|
contract: string;
|
|
567
|
+
status: ContractStatus;
|
|
568
|
+
/** `status === 'pass'`. */
|
|
460
569
|
valid: boolean;
|
|
461
|
-
/** Fraction of rules
|
|
570
|
+
/** Fraction of rules that passed, in [0, 1]. An errored rule is not a pass. */
|
|
462
571
|
score: number;
|
|
463
|
-
/**
|
|
572
|
+
/** 0|1 per evaluated base rule, keyed by label, plus `anyOf` when the
|
|
573
|
+
* contract has alternatives. Errored rules are absent, never 0. */
|
|
464
574
|
scores: Record<string, number>;
|
|
465
575
|
violations: ContractViolation[];
|
|
576
|
+
ruleExecutions: ContractRuleExecution[];
|
|
577
|
+
/** Why rules errored, one line per errored rule or scope failure. */
|
|
578
|
+
errors: string[];
|
|
466
579
|
}
|
|
467
580
|
interface ContractCheckResult {
|
|
468
581
|
verdicts: ContractVerdict[];
|
|
582
|
+
/** `error` if any verdict errored, else `fail` if any failed, else `pass`. */
|
|
583
|
+
status: ContractStatus;
|
|
469
584
|
allValid: boolean;
|
|
470
585
|
}
|
|
586
|
+
interface EvaluateTraceContractOptions {
|
|
587
|
+
/** The run `run` rules read. Ignored under `scope`, where the scope root is the run. */
|
|
588
|
+
run?: ContractRun;
|
|
589
|
+
}
|
|
590
|
+
//#endregion
|
|
591
|
+
//#region src/trace-contracts/builder.d.ts
|
|
592
|
+
interface OrderOptions {
|
|
593
|
+
order?: OrderMode;
|
|
594
|
+
}
|
|
595
|
+
interface ArgumentRuleOptions {
|
|
596
|
+
pointer: string;
|
|
597
|
+
check: ArgumentCheck;
|
|
598
|
+
occurrence?: ArgumentOccurrence;
|
|
599
|
+
}
|
|
471
600
|
declare class TraceContractBuilder {
|
|
472
601
|
private readonly name;
|
|
473
602
|
private readonly rules;
|
|
603
|
+
private readonly alternativeList;
|
|
604
|
+
private scopeRoot;
|
|
474
605
|
constructor(name: string);
|
|
475
606
|
/** Every span in the trace must satisfy `p`. */
|
|
476
607
|
always(p: SpanPredicate, label?: string): this;
|
|
@@ -478,19 +609,130 @@ declare class TraceContractBuilder {
|
|
|
478
609
|
never(p: SpanPredicate, label?: string): this;
|
|
479
610
|
/** At least one span in the trace must satisfy `p`. */
|
|
480
611
|
eventually(p: SpanPredicate, label?: string): this;
|
|
481
|
-
/**
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
612
|
+
/** Both `a` and `b` must occur, with `a` first under `options.order`
|
|
613
|
+
* (default `start-order`). For "every `b` needs an earlier `a`, and a run
|
|
614
|
+
* without `b` passes", use {@link neverUnless}. */
|
|
615
|
+
precedes(a: SpanPredicate, b: SpanPredicate, label?: string, options?: OrderOptions): this;
|
|
616
|
+
/** Every `p`-match needs an earlier `prior`-match; a run without `p` passes. */
|
|
617
|
+
neverUnless(p: SpanPredicate, prior: SpanPredicate, label?: string, options?: OrderOptions): this;
|
|
618
|
+
/** At most `max` spans may satisfy `p`. */
|
|
619
|
+
atMost(p: SpanPredicate, max: number, label?: string): this;
|
|
620
|
+
/** Spans satisfying `p` may use at most `max` input+output tokens in total. */
|
|
621
|
+
tokensAtMost(p: SpanPredicate, max: number, label?: string): this;
|
|
622
|
+
/** Terminal status and duration of the run. */
|
|
623
|
+
run(checks: RunChecks, label?: string): this;
|
|
624
|
+
/** A JSON Pointer check on the arguments of calls matching `p`. */
|
|
625
|
+
argument(p: SpanPredicate, options: ArgumentRuleOptions, label?: string): this;
|
|
626
|
+
/** Evaluate only the subtree rooted at the one span matching `root`. */
|
|
627
|
+
scope(root: SpanPredicate): this;
|
|
628
|
+
/** A named alternate path; at least one alternative must pass completely. */
|
|
629
|
+
alternative(id: string, define: (branch: TraceContractBuilder) => TraceContractBuilder): this;
|
|
485
630
|
build(): TraceContract;
|
|
631
|
+
private where;
|
|
486
632
|
private add;
|
|
487
633
|
}
|
|
488
634
|
declare function traceContract(name: string): TraceContractBuilder;
|
|
635
|
+
//#endregion
|
|
636
|
+
//#region src/trace-contracts/evaluate.d.ts
|
|
637
|
+
/**
|
|
638
|
+
* Evaluate one contract over a span sequence. Pure and synchronous — works on
|
|
639
|
+
* `Span[]` from a TraceStore, `ExportableSpan[]` from the otel-bridge
|
|
640
|
+
* flattening, or any array satisfying `ContractSpan`. Throws
|
|
641
|
+
* `ValidationError` for a malformed contract; a rule that cannot be evaluated
|
|
642
|
+
* is reported as `error` in the verdict instead.
|
|
643
|
+
*/
|
|
644
|
+
declare function evaluateTraceContract(contract: TraceContract, spans: readonly ContractSpan[], options?: EvaluateTraceContractOptions): ContractVerdict;
|
|
489
645
|
/**
|
|
490
646
|
* Evaluate many contracts over one span sequence. Throws on an empty
|
|
491
647
|
* contract list — `allValid: true` over zero contracts is a silent pass.
|
|
492
648
|
*/
|
|
493
|
-
declare function checkTraceContracts(spans: readonly ContractSpan[], contracts: readonly TraceContract[]): ContractCheckResult;
|
|
649
|
+
declare function checkTraceContracts(spans: readonly ContractSpan[], contracts: readonly TraceContract[], options?: EvaluateTraceContractOptions): ContractCheckResult;
|
|
650
|
+
//#endregion
|
|
651
|
+
//#region src/trace-contracts/explain.d.ts
|
|
652
|
+
/** Plain-language statement of everything a contract checks, one rule per line. */
|
|
653
|
+
declare function explainTraceContract(contract: TraceContract): string;
|
|
654
|
+
//#endregion
|
|
655
|
+
//#region src/trace-contracts/spec.d.ts
|
|
656
|
+
interface ToolArgumentSpec {
|
|
657
|
+
tool: string;
|
|
658
|
+
/** RFC 6901 JSON Pointer into the call's arguments; `''` is the whole value. */
|
|
659
|
+
pointer: string;
|
|
660
|
+
/** Which calls must satisfy the check. Default `all`. */
|
|
661
|
+
occurrence?: ArgumentOccurrence;
|
|
662
|
+
/** Exactly one of `exists`, `equals`, `oneOf`, `type`. */
|
|
663
|
+
exists?: true;
|
|
664
|
+
equals?: unknown;
|
|
665
|
+
oneOf?: unknown[];
|
|
666
|
+
type?: ArgumentType;
|
|
667
|
+
}
|
|
668
|
+
interface ToolsSpec {
|
|
669
|
+
/** Each must be called at least once. */
|
|
670
|
+
required?: string[];
|
|
671
|
+
/** None may be called. */
|
|
672
|
+
forbidden?: string[];
|
|
673
|
+
/** Every tool call must name one of these; a call without a name fails. */
|
|
674
|
+
allowed?: string[];
|
|
675
|
+
/** Ceiling on tool calls of any name. */
|
|
676
|
+
maxCalls?: number;
|
|
677
|
+
/** Ceiling per tool name. */
|
|
678
|
+
maxCallsPerTool?: Record<string, number>;
|
|
679
|
+
/** Each tool is called, and each comes before the next under `orderMode`. */
|
|
680
|
+
requiredOrder?: string[];
|
|
681
|
+
/** Default `start-order`. */
|
|
682
|
+
orderMode?: OrderMode;
|
|
683
|
+
arguments?: ToolArgumentSpec[];
|
|
684
|
+
}
|
|
685
|
+
interface LlmSpec {
|
|
686
|
+
maxCalls?: number;
|
|
687
|
+
/** Input plus output tokens across LLM calls; an unrecorded count fails. */
|
|
688
|
+
maxTotalTokens?: number;
|
|
689
|
+
/** Every LLM call must name one of these models; a call without one fails. */
|
|
690
|
+
allowedModels?: string[];
|
|
691
|
+
}
|
|
692
|
+
/** The checks one path carries: the base contract or one alternative. */
|
|
693
|
+
interface PathSpec {
|
|
694
|
+
run?: RunChecks;
|
|
695
|
+
tools?: ToolsSpec;
|
|
696
|
+
llm?: LlmSpec;
|
|
697
|
+
/** Low-level rules in their serialized form. */
|
|
698
|
+
rules?: ContractRule[];
|
|
699
|
+
}
|
|
700
|
+
interface AlternativeSpec extends PathSpec {
|
|
701
|
+
id: string;
|
|
702
|
+
}
|
|
703
|
+
interface TraceContractSpec extends PathSpec {
|
|
704
|
+
name: string;
|
|
705
|
+
description?: string;
|
|
706
|
+
/** Check only the one span matching `root` and its descendants: one
|
|
707
|
+
* sub-agent (`{ "kind": "AGENT", "name": "researcher" }`) or one search
|
|
708
|
+
* node (`{ "attr": { "agent.branch.id": "n7" } }`). */
|
|
709
|
+
scope?: {
|
|
710
|
+
root: SpanPredicate;
|
|
711
|
+
};
|
|
712
|
+
/** At least one alternative must pass completely, as well as the base checks. */
|
|
713
|
+
alternatives?: {
|
|
714
|
+
anyOf: AlternativeSpec[];
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
/**
|
|
718
|
+
* Parse and compile a declarative contract. Throws `ValidationError` naming
|
|
719
|
+
* the path of the first problem: an unknown key, a wrong type, an unknown
|
|
720
|
+
* status, or a contract that checks nothing.
|
|
721
|
+
*/
|
|
722
|
+
declare function compileTraceContractSpec(input: unknown): TraceContract;
|
|
723
|
+
interface ContractLintFinding {
|
|
724
|
+
/** `error`: the contract can never pass, or contradicts itself.
|
|
725
|
+
* `warning`: the contract works but likely does not say what was meant. */
|
|
726
|
+
level: 'error' | 'warning';
|
|
727
|
+
code: string;
|
|
728
|
+
path: string;
|
|
729
|
+
message: string;
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* Find contradictions and likely mistakes in a declarative contract. Parses
|
|
733
|
+
* strictly first, so a malformed spec throws instead of linting.
|
|
734
|
+
*/
|
|
735
|
+
declare function lintTraceContractSpec(input: unknown): ContractLintFinding[];
|
|
494
736
|
//#endregion
|
|
495
737
|
//#region src/active-learning.d.ts
|
|
496
738
|
type SynthesisReason = 'high-variance' | 'undersampled' | 'failure-cluster' | 'difficulty-gap';
|
|
@@ -933,6 +1175,79 @@ interface DiscoveredPersona {
|
|
|
933
1175
|
}
|
|
934
1176
|
declare function discoverPersonas(dir: string, opts?: DiscoverPersonasOptions): Promise<DiscoveredPersona[]>;
|
|
935
1177
|
//#endregion
|
|
1178
|
+
//#region src/improvement-verdict.d.ts
|
|
1179
|
+
/** Verdict for one candidate relative to its parent. ITERATE is the neutral
|
|
1180
|
+
* "keep collecting reps / no parent to compare against" state. */
|
|
1181
|
+
type ExperimentVerdict = 'KEEP' | 'ITERATE' | 'NOISE' | 'REGRESSION';
|
|
1182
|
+
/** A single repetition of a candidate run, carrying the score the verdict is
|
|
1183
|
+
* computed on plus any free-form per-rep metrics the consumer wants kept. */
|
|
1184
|
+
interface ExperimentRep {
|
|
1185
|
+
/** 0-indexed repetition number. */
|
|
1186
|
+
rep: number;
|
|
1187
|
+
/** The score this rep is judged on (same scale as the thresholds). */
|
|
1188
|
+
score: number;
|
|
1189
|
+
/** ISO timestamp the rep completed. */
|
|
1190
|
+
timestamp: string;
|
|
1191
|
+
/** Stable execution/run identity that produced this score. */
|
|
1192
|
+
runId?: string;
|
|
1193
|
+
/** Mechanically resolvable trace, artifact, metric, or finding evidence. */
|
|
1194
|
+
evidence?: EvidenceRef[];
|
|
1195
|
+
/** Whether this rep passed the consumer's own gate — folded into `passRate`. */
|
|
1196
|
+
passed?: boolean;
|
|
1197
|
+
/** Free-form numeric metrics retained for later analysis. */
|
|
1198
|
+
metrics?: Record<string, number>;
|
|
1199
|
+
}
|
|
1200
|
+
interface ExperimentStats {
|
|
1201
|
+
median: number;
|
|
1202
|
+
mean: number;
|
|
1203
|
+
min: number;
|
|
1204
|
+
max: number;
|
|
1205
|
+
/** Inter-quartile range of the rep scores. */
|
|
1206
|
+
iqr: number;
|
|
1207
|
+
/** Population standard deviation of the rep scores. */
|
|
1208
|
+
stddev: number;
|
|
1209
|
+
/** Fraction of reps with `passed === true`, over reps that set `passed`.
|
|
1210
|
+
* null when no rep declared a pass/fail outcome. */
|
|
1211
|
+
passRate: number | null;
|
|
1212
|
+
/** Number of reps. */
|
|
1213
|
+
n: number;
|
|
1214
|
+
/** True when the sample is tight enough to trust for a verdict. */
|
|
1215
|
+
stable: boolean;
|
|
1216
|
+
}
|
|
1217
|
+
interface ImprovementThresholds {
|
|
1218
|
+
/** medianDelta strictly above this ⇒ KEEP. Default 5. */
|
|
1219
|
+
keepThreshold?: number;
|
|
1220
|
+
/** medianDelta strictly below the negative of this ⇒ REGRESSION. Default 5. */
|
|
1221
|
+
regressionThreshold?: number;
|
|
1222
|
+
/** iqr at or above this ⇒ unstable. Default 10. */
|
|
1223
|
+
iqrUnstableAbove?: number;
|
|
1224
|
+
/** stddev at or above this ⇒ unstable. Default Infinity (iqr-only stability). */
|
|
1225
|
+
stddevUnstableAbove?: number;
|
|
1226
|
+
/** Reps required on BOTH candidate and parent before a verdict is rendered.
|
|
1227
|
+
* Default 3. */
|
|
1228
|
+
minRepsForVerdict?: number;
|
|
1229
|
+
}
|
|
1230
|
+
interface ImprovementVerdictResult {
|
|
1231
|
+
verdict: ExperimentVerdict;
|
|
1232
|
+
/** candidate.median − parent.median; null when no parent or insufficient reps. */
|
|
1233
|
+
medianDelta: number | null;
|
|
1234
|
+
/** Human-readable reason for the verdict — for dashboards and logs. */
|
|
1235
|
+
reason: string;
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Compute the N-rep statistics for a set of reps. Pure — no I/O. The `stable`
|
|
1239
|
+
* flag is the trust gate the verdict depends on: a sample whose spread exceeds
|
|
1240
|
+
* the configured bounds can't distinguish a real delta from run-to-run noise.
|
|
1241
|
+
*/
|
|
1242
|
+
declare function computeExperimentStats(reps: ExperimentRep[], thresholds?: ImprovementThresholds): ExperimentStats;
|
|
1243
|
+
/**
|
|
1244
|
+
* Verdict for a candidate against its parent. Pure — operates on already-computed
|
|
1245
|
+
* stats. KEEP/REGRESSION require both sides to have `>= minRepsForVerdict` reps
|
|
1246
|
+
* AND the candidate to be `stable`; otherwise the result is NOISE (unstable) or
|
|
1247
|
+
* ITERATE (not enough reps / no parent).
|
|
1248
|
+
*/
|
|
1249
|
+
declare function improvementVerdict(candidate: ExperimentStats, parent: ExperimentStats | null, thresholds?: ImprovementThresholds): ImprovementVerdictResult;
|
|
1250
|
+
//#endregion
|
|
936
1251
|
//#region src/propose-review.d.ts
|
|
937
1252
|
interface Verification {
|
|
938
1253
|
pass: boolean;
|
|
@@ -3332,5 +3647,5 @@ declare class PairwiseSteeringOptimizer {
|
|
|
3332
3647
|
optimize(rows: SteeringOptimizationRow[], config?: SteeringOptimizerConfig): SteeringOptimizationResult;
|
|
3333
3648
|
}
|
|
3334
3649
|
//#endregion
|
|
3335
|
-
export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, type ActionExecutionPolicy, type ActionPolicyDecision, type ActionableSideInfo, type ActiveLearningOptions, type AgentEvalAgent, AgentEvalError, type AgentEvalErrorCode, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentProfile, type AgentProfileCell, type AgentProfileCellInput, type AgentProfileDimensionValue, type AgentProfileJson, type AgentProfileSourceInput, type Analyst, type AnalystContext, type AnalystFeedbackTrajectoryOptions, type AnalystFinding, type AnalystFindingDigest, AnalystRegistry, type AnalystRegistryOptions, type AnalystReviewDecision, type AnalystReviewRequest, type AnalystRunDigest, type AnalystRunEvent, type AnalystRunInputs, type AnalystRunResult, type AnalystRunSummary, type AnalystSeverity, type AnalystUsageReceipt, type AnalyzeRunsOptions, type AnalyzeTracesResult, type Artifact, type ArtifactCheck, type ArtifactEventLike, type ArtifactResult, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertCrossFamilyServedOptions, type AssertServedModelOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, BOOTSTRAP_GATE_MIN_N, type BackendDescriptor, BackendIntegrityError, type BackendIntegrityReport, type BenchmarkEvaluation, type BlendWeights, type BootstrapOptions, type BootstrapResult, type BoundedProcessInput, type BoundedProcessResult, BudgetBreachError, BudgetGuard, type BudgetLedgerEntry, type BudgetPolicy, type BudgetSpec, CODING_HARNESSES, type CalibrationResult, type CampaignCellFailureReceipt, type CampaignCellRetryPolicy, type CampaignFactoryParams, type CampaignIntegrityPolicy, type CampaignResult, type CampaignRunContext, type CampaignRunOutcome, type CampaignScenario, type CampaignVariant, type CanaryAlert, type CanaryEvaluation, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CandidateScore, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CellVerdict, type ChannelRollup, type ChatCallOpts, type ChatClient, type ChatRequest, type ChatResponse, type ChatTraceEngineOptions, type ChatTransport, type CheckResult, type CheckerIdentity, type CheckerOutcome, type CliffsMagnitude, type CollectedArtifacts, type CommandRunner, type ComparePairedArmsOptions, type CompletionCriterion, type CompletionRequirement, type CompletionVerdict, type ConceptFinding, type ConceptSpec, ConfigError, type ContinuousAgreement, type ContinuousAgreementOptions, type ContinuousCalibrationResult, type ContractCheckResult, type ContractSpan, type ContractVerdict, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRuntimeConfig, type ControlRuntimeError, type ControlStep, type CorpusAgreementOptions, type CorpusAgreementPerDimension, type CorpusAgreementReport, type CorpusScoreRecord, type CorrectnessChecker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, type CostChannel, type CostEntry, CostLedger, type CostLedgerFilter, type CostLedgerHandle, type CostLedgerOptions, type CostLedgerPersistence, CostLedgerPersistenceError, type CostLedgerSummary, type CostProvenance, type CostReceipt, CostReceiptCaptureError, type CostReceiptInput, CostReservationExceededError, type CostResult, type CostSummary, CostTracker, type CostUsage, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateChatClientOpts, type CreateTraceAnalystOptions, CrossFamilyError, type CustomTokenPricing, type CustomTransportOpts, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS, DEFAULT_FAILURE_REASON_RULES, DEFAULT_PERMUTATIONS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, type DataAcquisitionPlan, type DatasetManifest, type DatasetOverview, type DatasetScenario, type DatasetSplit, type DefaultAnalystRegistryOptions, type DefaultVerdict, type DefineAgentEvalOptions, type DefinedAgentEval, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DetectorEvent, type DetectorSignal, type DiffPolicy, type DiffScorecardOptions, type JudgeScore as DimensionJudgeScore, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, type DriverState, type DspyRlmTraceEngineOptions, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type EquivalenceArm, type EquivalenceCheckDefinition, type EquivalenceCheckSpec, type EquivalenceChecker, type EquivalenceCheckerInput, type EquivalenceCheckerResult, type EquivalenceObligation, type EquivalenceObligationStatus, EquivalenceProtocolError, type EquivalenceRecord, type ErrorCluster, type ErrorCountPattern, type ErrorStreakOptions, type EvalCampaignOptions, type EvalCampaignResult, type EventFilter, type EvidenceRef, type ExactAnalystRunEvent, type ExactAnalystRunResult, type ExactCapableAnalyst, type ExactRegistryRunOpts, type ExactRiskDifferenceResult, type ExperimentPlan, type ExperimentRep, type ExperimentResult, type ExperimentStats, type ExtractOptions, type ExtractResult, type ExtractedUsage, FAILURE_BLAME, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, type FailedRun, type FailureBlame, type FailureClass, type FailureClassification, type FailureContext, type FailureMode, type FailureReasonRule, type FailureRule, type FeedbackArtifactType, type FeedbackAttempt, type FeedbackLabel, type FeedbackLabelKind, type FeedbackLabelSource, type FeedbackOptimizerRow, type FeedbackOutcome, type FeedbackSplitPolicy, type FeedbackTask, type FeedbackTrajectory, type FeedbackTrajectoryFilter, type FeedbackTrajectoryStore, type FieldDestination, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, type Finding, type FindingSubject, type FindingSubjectKind, type FindingToPolicyEditOptions, type FindingsDiff, FindingsStore, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type GateDecision, type GateEvidence, type GenericSpan, type GoldenItem, HARNESS_NATIVE_MODEL, type HarnessType, type HeadroomInput, HeldOutGate, type HeldOutGateConfig, type GateDecision$1 as HeldOutGateDecision, type HeldOutGateRejectionCode, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, INFRA_FAILURE_BLAMES, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, type InsightReport, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InterimReleaseConfidence, type InterimReleaseConfidenceInput, JudgeError, type JudgeFamily, type JudgeFn, type JudgeInput, type JudgeRetryOutcome, type JudgeRetryPolicy, type JudgeRubric, type JudgeScore$1 as JudgeScore, type JudgeScoreInput, type JudgeScoresRecord, type JudgeSpan, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeSensitivity, LOCAL_COMMAND_RUNNER_DEFAULT_CAP_MS, type Layer, type LayerResult, type LayerStatus, type LeaderboardOptions, type LeaderboardRow, LlmCallError, type LlmCallMetadata, type LlmCallRequest, type LlmCallResult, type LlmChargeBounds, type LlmCorrectnessCheckerOpts, type LlmJudgeOptions, type LlmMessage, LlmResponseError, type LlmReviewerConfig, type LlmSpan, type LlmTokenLogprob, type LlmToolCall, type LlmToolChoice, type LlmToolDefinition, type LlmUsage, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, type MannWhitneyResult, type MatchedPair, type MatchedRunRecordPair, type MaximumCharge, type McNemarResult, type MintRolloutOptions, type MintRolloutResult, type MintedRolloutLine, type MockTransportOpts, type ModelEndpointCheck, type ModelEndpointRequest, type ModelSeats, ModelSubstitutionError, MultiLayerVerifier, type NoLeakOptions, NoopRawProviderSink, NotFoundError, OUTPUT_VALUE, type Objective, type OpenAiCompatibleTransportOpts, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, OtlpFileTraceStore, type OtlpFlatLine, type OtlpSpan, POLICY_EDIT_AXES, POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, POLICY_EDIT_TARGET_SURFACES, type PaidCallResult, type PairArmsOptions, type PairArmsResult, type PairRunRecordsResult, type PairedArmRow, type PairedArmsComparison, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedCorrectness, type PairedDecisionMethod, type PairedDecisionShape, type PairedDecisionStatistic, type PairedDeltaTestOptions, type PairedDeltaTestResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedMcNemarEvidence, type PairedMetricDelta, type PairedPromotionAlternative, type PairedPromotionAlternativeCell, type PairedPromotionCallRates, type PairedPromotionDecision, type PairedPromotionDecisionOptions, type PairedPromotionPowerCall, type PairedPromotionPowerOptions, type PairedPromotionPowerResult, type PairedSignTestResult, type PairedTTestResult, type PairedValueLaw, PairwiseSteeringOptimizer, type ParetoPoint, type ParetoResult, type PartitionHeldOutOptions, type PendingCostCall, type PendingCostCallView, type PersistedFinding, type PersonaConfig, type PolicyEdit, type PolicyEditAdmission, type PolicyEditAdmissionOptions, type PolicyEditAxis, type PolicyEditCandidateRecord, type PolicyEditChange, type PolicyEditExpectedGain, type PolicyEditGainDirection, type PolicyEditGainUnit, type PolicyEditInit, type PolicyEditRisk, type PolicyEditSchemaVersion, type PolicyEditSource, type PolicyEditTarget, type PolicyEditTargetSurface, PolicyEditValidationError, type PreferenceMemoryEntry, type ProducedProposal, type ProducedState, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkValidationReport, ProductClient, type ProductClientConfig, type Scenario as ProductScenario, type ProfileAxisSpec, type ProjectRuntimeTrajectoryEvidenceOptions, type PromptHandle, PromptRegistry, type ProportionInterval, type ProposalEventLike, type ProposalFinding, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type ProposedSideEffect, type QueryTracesPage, REDACTION_VERSION, type RankTestMethod, type RankTestMethodRequest, type RankTestOptions, type RawAnalystFinding, type RawProviderEvent, type RawProviderSink, type RecordRunsOptions, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamReport, type RedactionRule, type ReferenceReplayCaseRun, type ReferenceReplayRun, type ReferenceReplaySplit, type ReflectionContext, type ReflectionProposal, type RegistryRunOpts, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseTraceEvidence, type RepeatedActionOptions, type RequiredPairsForPairedPromotionOptions, type RequiredPairsForPairedPromotionResult, type RequirementCheck, type ResearchReport, type ResearchReportOptions, type Researcher, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type RiskDifferenceResult, type RouteMap, type RoutedField, type RubricDimension, type Run, type RunCampaignOptions, type RunCommandInput, type RunCommandResult, type RunCostProvenance, type RunFilter, RunIntegrityError, type RunIntegrityReport, type RunJudgeMetadata, type RunOutcome, type RunPaidCallInput, type RunRecord, type RunRecordBackend, RunRecordValidationError, type RunScore, type RunScoreWeights, type RunSplitTag, type RunStatus, type RunTaskFailure, type RunTerminalOutcome, type RunTokenUsage, type RuntimeEventLike, type RuntimeTrajectoryEvidenceProjection, type RuntimeTrajectoryEvidenceSummary, type RuntimeTrajectoryHookEvent, type RuntimeTrajectoryRecord, type RuntimeTrajectoryRunRecord, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, type SandboxSdkTransportOpts, type SatisfiedBy, type Scenario$1 as Scenario, type ScenarioCost, type ScenarioFile, type ScenarioResult, type ScorePreference, type ScoreRiskDifferenceResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type SearchSpanResult, type SearchTraceResult, type SelfImproveBudget, type SelfImproveMethodOptions, type SelfImproveMethodProvenance, type SelfImproveMethodResult, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveProposerOptions, type SelfImproveProposerResult, type SelfImproveResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SequentialDecision, type SeriesConvergenceOptions, type SeriesConvergenceResult, type SeriesDistribution, ServedCrossFamilyError, type ServedModelCheck, type ServedModelPolicy, type ServedModelVerdict, type Severity, type SignTestAlternative, type SingleBackendDivergence, type SingleBackendReport, type Span, type SpanFilter, type SpanHandle, type SpanMatchRecord, type SplitCoverage, type SteeringBundle, type SteeringChange, type SteeringOptimizationResult, type SteeringOptimizationRow, type StopDecision, type StrategyChecker, type StreamingDetector, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type SynthesisTarget, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TaskGold, type TaskHeadroom, type ToolCallEventLike, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type TraceAnalysisEngine, type TraceAnalysisEngineResult, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystDefinition, type TraceAnalystFilters, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, type TraceContract, TraceEmitter, type TraceEvent, type TraceInsightReadiness, type TraceInsightSuite, type TraceStore, type TrainSelectionTestOptions, type TrainSelectionTestPartition, type Trajectory, type TrajectoryStep, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, type Turn, type TurnResult, UNKNOWN_MODEL, type UserQuestion, VERIFICATION_STRATEGIES, VERIFICATION_STRATEGY_SOURCES, ValidationError, type VerbosityBiasResult, type Verdict, type VerdictCacheStore, type VerdictCertification, type Verification, type VerificationReport, type VerificationStrategyProfile, type VerificationStrategySource, type VerifyFn, type VerifyOptions, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, type ViteDeployRunnerInput, WILCOXON_EXACT_MAX_N, type WeightedCompositeInput, type WeightedCompositeResult, type WilcoxonSignedRankResult, type WranglerDeployRunnerInput, acquisitionPlansForKnowledgeGaps, admitPolicyEdit, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, aggregateJudgeVerdicts, aggregateRunScore, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeRuns, analyzeSeries, analyzeTraces, applyPolicyEditToSurface, argHash, assertCapabilityHeadroom, assertCrossFamily, assertCrossFamilyServed, assertNoHiddenLeak, assertNoJudgeVerdict, assertProductBenchmarkRun, assertRealBackend, assertRunCaptured, assertServedModel, assertServedModels, assertSingleBackend, assignFeedbackSplit, attest, benjaminiHochberg, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, budgetBreachView, buildAgentProfileCell, buildDefaultAnalystRegistry, buildEquivalenceRecord, buildReflectionPrompt, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, calibrateJudge, calibrateJudgeContinuous, canonicalJson, capabilityHeadroom, captureFetchToRawSink, certificationEvidenceDigest, checkCanaries, checkServedModel, checkTraceContracts, clamp01, classifyFailure, classifyFailureReason, cliffsDelta, cohensD, comparePairedArms, completionVerdict, computeExperimentStats, computeFindingId, computePolicyEditId, computeToolUseMetrics, confidenceInterval, contentHash, continuousAgreement, controlRunToFeedbackTrajectory, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createChatTraceEngine, createDspyRlmTraceEngine, createFeedbackTrajectory, createLlmCorrectnessChecker, createLlmReviewer, createTraceAnalyst, decidePairedPromotion, defaultBlendWeights, defineAgentEval, defineEquivalenceCheck, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, domainEvidencePattern, dominates, eProcess, ensembleJudge, equivalenceVerdict, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, expandProfileAxes, exportProductBenchmark, exportProductBenchmarkRuns, exportRunAsOtlp, extractErrorCount, extractProducedState, extractUsage, extractUsageFromSse, failureBlame, failureClusterView, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToOptimizerRow, fileVerdictCache, formatScorecardDiff, gainHistogram, gateTreatmentApplied, gradeOnHidden, gradeSemanticStatus, groupRunsByAgentProfileCell, harnessAxisOf, hashContent, hashJson, hiddenGrade, holm, improvementVerdict, inMemoryReviewStore, inferDomainKeywords, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isJudgeSpan, isLlmSpan, isModelPriced, isPolicyEdit, isRunRecord, isToolSpan, isTransientLlmError, jsonShape, jsonlReviewStore, jsonlRunRecordBackend, judgeAgreementView, judgeFamily, judgeSpans, knowledgeReadinessTracePayload, leaderboard, llmJudge, loadScorecard, localCommandRunner, makeFinding, makePolicyEdit, makePolicyEditCandidateRecord, makeProposalFinding, manifestContentDigest, mannWhitneyU, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, minimumPairsForPairedDeltaTest, mintRolloutRows, modelHasSnapshot, modelPriceKey, mulberry32, notBlocked, objectiveEval, observeAll, otlpTextToTraceAnalysisStore, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedPromotionPower, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paretoChart, paretoFrontier, parseReflectionResponse, parseRunRecordSafe, partialCredit, partitionHeldOut, partitionTrainSelectionTest, passAtK, pearsonR, policyEditFromFinding, policyEditsFromFindings, preflightModels, productBenchmarkRepoIdentity, index_d_exports as profile, projectRuntimeTrajectoryEvidence, proposeSynthesisTargets, quotaExhaustedUntil, ranks, readProductBenchmarkManifest, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, regexMatches, renderPreferenceMemoryMarkdown, repeatedActionDetector, requiredPairedSampleSize, requiredPairsForPairedPromotion, requiredSampleSize, resolveModelPricing, resolveSeat, roundTripRunRecord, routeFields, runAgentControlLoop, runBoundedProcess, runCampaign, runCanaries, runCounterfactual, runEquivalenceCheck, runEvalCampaign, runIntentMatchJudge, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runProposeReview, runProposeReviewAsControlLoop, runSemanticConceptJudge, runTaskScore, runsForScenario, scoreKnowledgeReadiness, scorePolicyEditReadiness, scoreRedTeamOutput, scoreTraceInsightReadiness, seatPresets, selfImprove, servedModelAcceptable, spearmanR, stripFencedJson, subjectiveEval, summarizeBackendIntegrity, summarizeNumberSeries, summarizePreferenceMemory, summaryTable, textInSnapshot, toAgentProfileJson, tokenizeDomainWords, toolSpansToTraceAnalysisStore, toolWasteView, traceContract, transientDispatchFailure, urlContains, userQuestionsForKnowledgeGaps, validatePolicyEdit, validatePolicyEditCandidateRecord, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, viteDeployRunner, weightedComposite, weightedMean, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, wranglerDeployRunner };
|
|
3650
|
+
export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, type ActionExecutionPolicy, type ActionPolicyDecision, type ActionableSideInfo, type ActiveLearningOptions, type AgentEvalAgent, AgentEvalError, type AgentEvalErrorCode, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentProfile, type AgentProfileCell, type AgentProfileCellInput, type AgentProfileDimensionValue, type AgentProfileJson, type AgentProfileSourceInput, type Analyst, type AnalystContext, type AnalystFeedbackTrajectoryOptions, type AnalystFinding, type AnalystFindingDigest, AnalystRegistry, type AnalystRegistryOptions, type AnalystReviewDecision, type AnalystReviewRequest, type AnalystRunDigest, type AnalystRunEvent, type AnalystRunInputs, type AnalystRunResult, type AnalystRunSummary, type AnalystSeverity, type AnalystUsageReceipt, type AnalyzeRunsOptions, type AnalyzeTracesResult, type ArgumentCheck, type Artifact, type ArtifactCheck, type ArtifactEventLike, type ArtifactResult, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertCrossFamilyServedOptions, type AssertServedModelOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, BOOTSTRAP_GATE_MIN_N, type BackendDescriptor, BackendIntegrityError, type BackendIntegrityReport, type BenchmarkEvaluation, type BlendWeights, type BootstrapOptions, type BootstrapResult, type BoundedProcessInput, type BoundedProcessResult, BudgetBreachError, BudgetGuard, type BudgetLedgerEntry, type BudgetPolicy, type BudgetSpec, CODING_HARNESSES, type CalibrationResult, type CampaignCellFailureReceipt, type CampaignCellRetryPolicy, type CampaignFactoryParams, type CampaignIntegrityPolicy, type CampaignResult, type CampaignRunContext, type CampaignRunOutcome, type CampaignScenario, type CampaignVariant, type CanaryAlert, type CanaryEvaluation, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CandidateScore, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CaptureErrorHandler, type CaptureWrite, type CellVerdict, type ChannelRollup, type ChatCallOpts, type ChatClient, type ChatRequest, type ChatResponse, type ChatTraceEngineOptions, type ChatTransport, type CheckResult, type CheckerIdentity, type CheckerOutcome, type CliffsMagnitude, type CollectedArtifacts, type CommandRunner, type ComparePairedArmsOptions, type CompletionCriterion, type CompletionRequirement, type CompletionVerdict, type ConceptFinding, type ConceptSpec, ConfigError, type ContinuousAgreement, type ContinuousAgreementOptions, type ContinuousCalibrationResult, type ContractCheckResult, type ContractLintFinding, type ContractRule, type ContractRuleExecution, type ContractRun, type ContractSpan, type ContractStatus, type ContractVerdict, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRuntimeConfig, type ControlRuntimeError, type ControlStep, type CorpusAgreementOptions, type CorpusAgreementPerDimension, type CorpusAgreementReport, type CorpusScoreRecord, type CorrectnessChecker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, type CostChannel, type CostEntry, CostLedger, type CostLedgerFilter, type CostLedgerHandle, type CostLedgerOptions, type CostLedgerPersistence, CostLedgerPersistenceError, type CostLedgerSummary, type CostProvenance, type CostReceipt, CostReceiptCaptureError, type CostReceiptInput, CostReservationExceededError, type CostResult, type CostSummary, CostTracker, type CostUsage, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateChatClientOpts, type CreateTraceAnalystOptions, CrossFamilyError, type CustomTokenPricing, type CustomTransportOpts, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS, DEFAULT_FAILURE_REASON_RULES, DEFAULT_PERMUTATIONS, DEFAULT_RED_TEAM_CORPUS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, type DataAcquisitionPlan, type DatasetManifest, type DatasetOverview, type DatasetScenario, type DatasetSplit, type DefaultAnalystRegistryOptions, type DefaultVerdict, type DefineAgentEvalOptions, type DefinedAgentEval, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DetectorEvent, type DetectorSignal, type DiffPolicy, type DiffScorecardOptions, type JudgeScore as DimensionJudgeScore, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, type DriverState, type DspyRlmTraceEngineOptions, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type EquivalenceArm, type EquivalenceCheckDefinition, type EquivalenceCheckSpec, type EquivalenceChecker, type EquivalenceCheckerInput, type EquivalenceCheckerResult, type EquivalenceObligation, type EquivalenceObligationStatus, EquivalenceProtocolError, type EquivalenceRecord, type ErrorCluster, type ErrorCountPattern, type ErrorStreakOptions, type EvalCampaignOptions, type EvalCampaignResult, type EventFilter, type EvidenceRef, type ExactAnalystRunEvent, type ExactAnalystRunResult, type ExactCapableAnalyst, type ExactRegistryRunOpts, type ExactRiskDifferenceResult, type ExperimentPlan, type ExperimentRep, type ExperimentResult, type ExperimentStats, type ExtractOptions, type ExtractResult, type ExtractedUsage, FAILURE_BLAME, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, type FailedRun, type FailureBlame, type FailureClass, type FailureClassification, type FailureContext, type FailureMode, type FailureReasonRule, type FailureRule, type FeedbackArtifactType, type FeedbackAttempt, type FeedbackLabel, type FeedbackLabelKind, type FeedbackLabelSource, type FeedbackOptimizerRow, type FeedbackOutcome, type FeedbackSplitPolicy, type FeedbackTask, type FeedbackTrajectory, type FeedbackTrajectoryFilter, type FeedbackTrajectoryStore, type FieldDestination, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, type Finding, type FindingSubject, type FindingSubjectKind, type FindingToPolicyEditOptions, type FindingsDiff, FindingsStore, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type GateDecision, type GateEvidence, type GenericSpan, type GoldenItem, HARNESS_NATIVE_MODEL, type HarnessType, type HeadroomInput, HeldOutGate, type HeldOutGateConfig, type GateDecision$1 as HeldOutGateDecision, type HeldOutGateRejectionCode, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, INFRA_FAILURE_BLAMES, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, type InsightReport, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InterimReleaseConfidence, type InterimReleaseConfidenceInput, JudgeError, type JudgeFamily, type JudgeFn, type JudgeInput, type JudgeRetryOutcome, type JudgeRetryPolicy, type JudgeRubric, type JudgeScore$1 as JudgeScore, type JudgeScoreInput, type JudgeScoresRecord, type JudgeSpan, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeSensitivity, LOCAL_COMMAND_RUNNER_DEFAULT_CAP_MS, type Layer, type LayerResult, type LayerStatus, type LeaderboardOptions, type LeaderboardRow, LlmCallError, type LlmCallMetadata, type LlmCallRequest, type LlmCallResult, type LlmChargeBounds, type LlmCorrectnessCheckerOpts, type LlmJudgeOptions, type LlmMessage, LlmResponseError, type LlmReviewerConfig, type LlmSpan, type LlmTokenLogprob, type LlmToolCall, type LlmToolChoice, type LlmToolDefinition, type LlmUsage, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, type MannWhitneyResult, type MatchedPair, type MatchedRunRecordPair, type MaximumCharge, type McNemarResult, type MintRolloutOptions, type MintRolloutResult, type MintedRolloutLine, type MockTransportOpts, type ModelEndpointCheck, type ModelEndpointRequest, type ModelSeats, ModelSubstitutionError, MultiLayerVerifier, type NoLeakOptions, NoopRawProviderSink, NotFoundError, OUTPUT_VALUE, type Objective, type OpenAiCompatibleTransportOpts, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrderMode, type OtlpExport, OtlpFileTraceStore, type OtlpFlatLine, type OtlpSpan, POLICY_EDIT_AXES, POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, POLICY_EDIT_TARGET_SURFACES, type PaidCallResult, type PairArmsOptions, type PairArmsResult, type PairRunRecordsResult, type PairedArmRow, type PairedArmsComparison, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedCorrectness, type PairedDecisionMethod, type PairedDecisionShape, type PairedDecisionStatistic, type PairedDeltaTestOptions, type PairedDeltaTestResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedMcNemarEvidence, type PairedMetricDelta, type PairedPromotionAlternative, type PairedPromotionAlternativeCell, type PairedPromotionCallRates, type PairedPromotionDecision, type PairedPromotionDecisionOptions, type PairedPromotionPowerCall, type PairedPromotionPowerOptions, type PairedPromotionPowerResult, type PairedSignTestResult, type PairedTTestResult, type PairedValueLaw, PairwiseSteeringOptimizer, type ParetoPoint, type ParetoResult, type PartitionHeldOutOptions, type PendingCostCall, type PendingCostCallView, type PersistedFinding, type PersonaConfig, type PolicyEdit, type PolicyEditAdmission, type PolicyEditAdmissionOptions, type PolicyEditAxis, type PolicyEditCandidateRecord, type PolicyEditChange, type PolicyEditExpectedGain, type PolicyEditGainDirection, type PolicyEditGainUnit, type PolicyEditInit, type PolicyEditRisk, type PolicyEditSchemaVersion, type PolicyEditSource, type PolicyEditTarget, type PolicyEditTargetSurface, PolicyEditValidationError, type PreferenceMemoryEntry, type ProducedProposal, type ProducedState, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkValidationReport, ProductClient, type ProductClientConfig, type Scenario as ProductScenario, type ProfileAxisSpec, type ProjectRuntimeTrajectoryEvidenceOptions, type PromptHandle, PromptRegistry, type ProportionInterval, type ProposalEventLike, type ProposalFinding, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type ProposedSideEffect, type QueryTracesPage, REDACTION_VERSION, type RankTestMethod, type RankTestMethodRequest, type RankTestOptions, type RawAnalystFinding, type RawProviderEvent, type RawProviderSink, type RecordRunsOptions, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamReport, type RedactOptions, type RedactionProfile, type RedactionReport, type ReferenceReplayCaseRun, type ReferenceReplayRun, type ReferenceReplaySplit, type ReflectionContext, type ReflectionProposal, type RegistryRunOpts, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseTraceEvidence, type RepeatedActionOptions, type RequiredPairsForPairedPromotionOptions, type RequiredPairsForPairedPromotionResult, type RequirementCheck, type ResearchReport, type ResearchReportOptions, type Researcher, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type RiskDifferenceResult, type RouteMap, type RoutedField, type RubricDimension, type Run, type RunCampaignOptions, type RunCaptureReport, type RunCommandInput, type RunCommandResult, type RunCostProvenance, type RunFilter, RunIntegrityError, type RunIntegrityReport, type RunJudgeMetadata, type RunOutcome, type RunPaidCallInput, type RunRecord, type RunRecordBackend, RunRecordValidationError, type RunScore, type RunScoreWeights, type RunSplitTag, type RunStatus, type RunTaskFailure, type RunTerminalOutcome, type RunTokenUsage, type RuntimeEventLike, type RuntimeTrajectoryEvidenceProjection, type RuntimeTrajectoryEvidenceSummary, type RuntimeTrajectoryHookEvent, type RuntimeTrajectoryRecord, type RuntimeTrajectoryRunRecord, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, type SandboxSdkTransportOpts, type SatisfiedBy, type Scenario$1 as Scenario, type ScenarioCost, type ScenarioFile, type ScenarioResult, type ScorePreference, type ScoreRiskDifferenceResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type SearchSpanResult, type SearchTraceResult, type SelfImproveBudget, type SelfImproveMethodOptions, type SelfImproveMethodProvenance, type SelfImproveMethodResult, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveProposerOptions, type SelfImproveProposerResult, type SelfImproveResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SequentialDecision, type SeriesConvergenceOptions, type SeriesConvergenceResult, type SeriesDistribution, ServedCrossFamilyError, type ServedModelCheck, type ServedModelPolicy, type ServedModelVerdict, type Severity, type ShareSafetyStatus, type ShareSafetyVerdict, type SignTestAlternative, type SingleBackendDivergence, type SingleBackendReport, type Span, type SpanFilter, type SpanHandle, type SpanMatchRecord, type SpanPredicate, type SplitCoverage, type SteeringBundle, type SteeringChange, type SteeringOptimizationResult, type SteeringOptimizationRow, type StopDecision, type StrategyChecker, type StreamingDetector, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type SynthesisTarget, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TaskGold, type TaskHeadroom, type ToolCallEventLike, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type TraceAnalysisEngine, type TraceAnalysisEngineResult, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystDefinition, type TraceAnalystFilters, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, type TraceContract, type TraceContractSpec, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceInsightReadiness, type TraceInsightSuite, type TraceStore, type TrainSelectionTestOptions, type TrainSelectionTestPartition, type Trajectory, type TrajectoryStep, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, type Turn, type TurnResult, UNKNOWN_MODEL, type UserQuestion, VERIFICATION_STRATEGIES, VERIFICATION_STRATEGY_SOURCES, ValidationError, type VerbosityBiasResult, type Verdict, type VerdictCacheStore, type VerdictCertification, type Verification, type VerificationReport, type VerificationStrategyProfile, type VerificationStrategySource, type VerifyFn, type VerifyOptions, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, type ViteDeployRunnerInput, WILCOXON_EXACT_MAX_N, type WeightedCompositeInput, type WeightedCompositeResult, type WilcoxonSignedRankResult, type WranglerDeployRunnerInput, acquisitionPlansForKnowledgeGaps, admitPolicyEdit, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, aggregateJudgeVerdicts, aggregateRunScore, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeRuns, analyzeSeries, analyzeTraces, applyPolicyEditToSurface, argHash, assertCapabilityHeadroom, assertCrossFamily, assertCrossFamilyServed, assertNoHiddenLeak, assertNoJudgeVerdict, assertProductBenchmarkRun, assertRealBackend, assertRunCaptured, assertServedModel, assertServedModels, assertSingleBackend, assessShareSafety, assignFeedbackSplit, attest, benjaminiHochberg, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, budgetBreachView, buildAgentProfileCell, buildDefaultAnalystRegistry, buildEquivalenceRecord, buildReflectionPrompt, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, calibrateJudge, calibrateJudgeContinuous, canonicalJson, capabilityHeadroom, captureFetchToRawSink, certificationEvidenceDigest, checkCanaries, checkServedModel, checkTraceContracts, clamp01, classifyFailure, classifyFailureReason, cliffsDelta, cohensD, comparePairedArms, compileTraceContractSpec, completionVerdict, computeExperimentStats, computeFindingId, computePolicyEditId, computeToolUseMetrics, confidenceInterval, contentHash, continuousAgreement, controlRunToFeedbackTrajectory, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createChatTraceEngine, createDspyRlmTraceEngine, createFeedbackTrajectory, createLlmCorrectnessChecker, createLlmReviewer, createTraceAnalyst, decidePairedPromotion, defaultBlendWeights, defineAgentEval, defineEquivalenceCheck, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, domainEvidencePattern, dominates, eProcess, ensembleJudge, equivalenceVerdict, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, expandProfileAxes, explainTraceContract, exportProductBenchmark, exportProductBenchmarkRuns, exportRunAsOtlp, extractErrorCount, extractProducedState, extractUsage, extractUsageFromSse, failureBlame, failureClusterView, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToOptimizerRow, fileVerdictCache, formatScorecardDiff, gainHistogram, gateTreatmentApplied, gradeOnHidden, gradeSemanticStatus, groupRunsByAgentProfileCell, harnessAxisOf, hashContent, hashJson, hiddenGrade, holm, improvementVerdict, inMemoryReviewStore, inferDomainKeywords, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isJudgeSpan, isLlmSpan, isModelPriced, isPolicyEdit, isRunRecord, isToolSpan, isTransientLlmError, jsonShape, jsonlReviewStore, jsonlRunRecordBackend, judgeAgreementView, judgeFamily, judgeSpans, knowledgeReadinessTracePayload, leaderboard, lintTraceContractSpec, llmJudge, loadScorecard, localCommandRunner, makeFinding, makePolicyEdit, makePolicyEditCandidateRecord, makeProposalFinding, manifestContentDigest, mannWhitneyU, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, minimumPairsForPairedDeltaTest, mintRolloutRows, modelHasSnapshot, modelPriceKey, mulberry32, notBlocked, objectiveEval, observeAll, otlpTextToTraceAnalysisStore, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedPromotionPower, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paretoChart, paretoFrontier, parseReflectionResponse, parseRunRecordSafe, partialCredit, partitionHeldOut, partitionTrainSelectionTest, passAtK, pearsonR, policyEditFromFinding, policyEditsFromFindings, preflightModels, productBenchmarkRepoIdentity, index_d_exports as profile, projectRuntimeTrajectoryEvidence, proposeSynthesisTargets, quotaExhaustedUntil, ranks, readProductBenchmarkManifest, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redact, redactForShare, redactText, regexMatches, renderPreferenceMemoryMarkdown, repeatedActionDetector, requiredPairedSampleSize, requiredPairsForPairedPromotion, requiredSampleSize, resolveModelPricing, resolveSeat, roundTripRunRecord, routeFields, runAgentControlLoop, runBoundedProcess, runCampaign, runCanaries, runCounterfactual, runEquivalenceCheck, runEvalCampaign, runIntentMatchJudge, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runProposeReview, runProposeReviewAsControlLoop, runSemanticConceptJudge, runTaskScore, runsForScenario, scoreKnowledgeReadiness, scorePolicyEditReadiness, scoreRedTeamOutput, scoreTraceInsightReadiness, seatPresets, selfImprove, servedModelAcceptable, shareAllowed, spearmanR, stripFencedJson, subjectiveEval, summarizeBackendIntegrity, summarizeNumberSeries, summarizePreferenceMemory, summaryTable, textInSnapshot, toAgentProfileJson, tokenizeDomainWords, toolSpansToTraceAnalysisStore, toolWasteView, traceContract, transientDispatchFailure, urlContains, userQuestionsForKnowledgeGaps, validatePolicyEdit, validatePolicyEditCandidateRecord, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, viteDeployRunner, weightedComposite, weightedMean, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, wranglerDeployRunner };
|
|
3336
3651
|
//# sourceMappingURL=index.d.ts.map
|