@tangle-network/agent-eval 0.200.1 → 0.201.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/adapters/http.d.ts +7 -8
- package/dist/adapters/http.d.ts.map +1 -1
- package/dist/adapters/http.js +2 -2
- package/dist/adapters/http.js.map +1 -1
- package/dist/agent-profile-DDfIjdyS.d.ts.map +1 -1
- package/dist/analyst/index.d.ts +76 -76
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +6 -6
- package/dist/{analyze-runs-DFbp2gB8.js → analyze-runs-BrIibNd0.js} +47 -42
- package/dist/{analyze-runs-DFbp2gB8.js.map → analyze-runs-BrIibNd0.js.map} +1 -1
- package/dist/{ast-CP9ae9B0.js → ast-DJ6Bc7ab.js} +5 -4
- package/dist/{ast-CP9ae9B0.js.map → ast-DJ6Bc7ab.js.map} +1 -1
- package/dist/authenticity/index.d.ts +14 -15
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/{baseline-BC-eBZ7U.js → baseline-CVKsfWTg.js} +2 -2
- package/dist/{baseline-BC-eBZ7U.js.map → baseline-CVKsfWTg.js.map} +1 -1
- package/dist/{benchmark-BeQJyrRC.js → benchmark-BywPeAUo.js} +4 -3
- package/dist/{benchmark-BeQJyrRC.js.map → benchmark-BywPeAUo.js.map} +1 -1
- package/dist/{benchmark-command-EBmuMUBo.js → benchmark-command-DFs_i62N.js} +24 -23
- package/dist/{benchmark-command-EBmuMUBo.js.map → benchmark-command-DFs_i62N.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +12 -12
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/{bounded-process-BBZob7vl.js → bounded-process-Cj9YjXbE.js} +3 -3
- package/dist/{bounded-process-BBZob7vl.js.map → bounded-process-Cj9YjXbE.js.map} +1 -1
- package/dist/builder-eval/index.d.ts +16 -17
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +12 -10
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.js +11 -11
- package/dist/{campaign-L391bmir.js → campaign-PmTWZi72.js} +30 -25
- package/dist/{campaign-L391bmir.js.map → campaign-PmTWZi72.js.map} +1 -1
- package/dist/{campaign-evidence-jL_U7Ikv.js → campaign-evidence-Dt_rWPk2.js} +4 -4
- package/dist/{campaign-evidence-jL_U7Ikv.js.map → campaign-evidence-Dt_rWPk2.js.map} +1 -1
- package/dist/{chat-client-CJORSG_J.js → chat-client-DUhWNQQS.js} +5 -4
- package/dist/{chat-client-CJORSG_J.js.map → chat-client-DUhWNQQS.js.map} +1 -1
- package/dist/{chat-json-call-DnlDjpco.js → chat-json-call-Dx_sHzqm.js} +2 -2
- package/dist/{chat-json-call-DnlDjpco.js.map → chat-json-call-Dx_sHzqm.js.map} +1 -1
- package/dist/{claim-V9Zqbq0J.js → claim-BXcZw-jo.js} +2 -2
- package/dist/{claim-V9Zqbq0J.js.map → claim-BXcZw-jo.js.map} +1 -1
- package/dist/cli.js +12 -11
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.js +22 -19
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-xMLX160E.js → default-registry-D4qXx2aG.js} +2 -2
- package/dist/{default-registry-xMLX160E.js.map → default-registry-D4qXx2aG.js.map} +1 -1
- package/dist/define-agent-eval-1YuvJpOs.d.ts.map +1 -1
- package/dist/{define-agent-eval-7q_UwsTt.js → define-agent-eval-CJN9dTQw.js} +10 -9
- package/dist/{define-agent-eval-7q_UwsTt.js.map → define-agent-eval-CJN9dTQw.js.map} +1 -1
- package/dist/{descriptive-1V17A-qa.js → descriptive-D9LUFk1-.js} +2 -2
- package/dist/{descriptive-1V17A-qa.js.map → descriptive-D9LUFk1-.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +7 -7
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +2 -2
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-B5T43BkZ.js → dspy-rlm-engine-US8YsYGq.js} +5 -5
- package/dist/{dspy-rlm-engine-B5T43BkZ.js.map → dspy-rlm-engine-US8YsYGq.js.map} +1 -1
- package/dist/{eval-campaign-GapmPHIA.js → eval-campaign-ha_aWtk7.js} +14 -12
- package/dist/{eval-campaign-GapmPHIA.js.map → eval-campaign-ha_aWtk7.js.map} +1 -1
- package/dist/evaluation.d.ts +9 -10
- package/dist/evaluation.d.ts.map +1 -1
- package/dist/experiment/index.d.ts +29 -29
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +13 -12
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-DEcXbvgO.js → exporters-Coiv-qO2.js} +2 -2
- package/dist/{exporters-DEcXbvgO.js.map → exporters-Coiv-qO2.js.map} +1 -1
- package/dist/{external-optimizer-process-CEXcCKqd.js → external-optimizer-process-KLcw0Vnc.js} +2 -2
- package/dist/{external-optimizer-process-CEXcCKqd.js.map → external-optimizer-process-KLcw0Vnc.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-laM1-UDV.js → external-optimizer-subprocess-CkyUUV7h.js} +4 -4
- package/dist/{external-optimizer-subprocess-laM1-UDV.js.map → external-optimizer-subprocess-CkyUUV7h.js.map} +1 -1
- package/dist/fuzz.d.ts +15 -15
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/{hf-dataset-CRqtSjnR.js → hf-dataset-kjIfIK_e.js} +3 -3
- package/dist/{hf-dataset-CRqtSjnR.js.map → hf-dataset-kjIfIK_e.js.map} +1 -1
- package/dist/hosted/index.d.ts +8 -8
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-D4pC7Tvd2.d.ts → index-D4pC7Tvd.d.ts} +1 -1
- package/dist/index-D4pC7Tvd.d.ts.map +1 -0
- package/dist/index-DKmrHA02.d.ts.map +1 -1
- package/dist/{index-DNgf5gyG2.d.ts → index-DNgf5gyG.d.ts} +1 -1
- package/dist/index-DNgf5gyG.d.ts.map +1 -0
- package/dist/index.d.ts +124 -124
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +52 -42
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BFP4kjGj.js → integrity-BvnVARVl.js} +30 -29
- package/dist/{integrity-BFP4kjGj.js.map → integrity-BvnVARVl.js.map} +1 -1
- package/dist/{integrity-BvHANvnp.js → integrity-uwK7XC-8.js} +2 -2
- package/dist/{integrity-BvHANvnp.js.map → integrity-uwK7XC-8.js.map} +1 -1
- package/dist/{internal-BMFSR8Ns.js → internal-BOe3zmjQ.js} +2 -2
- package/dist/{internal-BMFSR8Ns.js.map → internal-BOe3zmjQ.js.map} +1 -1
- package/dist/{jev-review-Cu9xtIu4.js → jev-review-DZEjDzlF.js} +7 -5
- package/dist/{jev-review-Cu9xtIu4.js.map → jev-review-DZEjDzlF.js.map} +1 -1
- package/dist/jev.d.ts +10 -10
- package/dist/jev.d.ts.map +1 -1
- package/dist/jev.js +2 -2
- package/dist/{judge-calibration-DYmaBtJr.js → judge-calibration-C5zgtiJT.js} +2 -2
- package/dist/{judge-calibration-DYmaBtJr.js.map → judge-calibration-C5zgtiJT.js.map} +1 -1
- package/dist/kind-factory-CdPPHtGZ.js.map +1 -1
- package/dist/{llm-client-c_bKHTXz.js → llm-client-Bhqoglar.js} +6 -4
- package/dist/{llm-client-c_bKHTXz.js.map → llm-client-Bhqoglar.js.map} +1 -1
- package/dist/{llm-judge-CL6f3utD.js → llm-judge-C-0P1WiT.js} +26 -21
- package/dist/{llm-judge-CL6f3utD.js.map → llm-judge-C-0P1WiT.js.map} +1 -1
- package/dist/matrix/index.d.ts +1 -1
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-_NLSWsKM.d.ts → matrix--IZKROxD.d.ts} +2 -2
- package/dist/{matrix-_NLSWsKM.d.ts.map → matrix--IZKROxD.d.ts.map} +1 -1
- package/dist/{matrix-pWinRMPl.js → matrix-DnlHn5x1.js} +8 -6
- package/dist/{matrix-pWinRMPl.js.map → matrix-DnlHn5x1.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +55 -55
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +4 -4
- package/dist/{mint-iCD-W43s.js → mint-vlRckJEX.js} +2 -2
- package/dist/{mint-iCD-W43s.js.map → mint-vlRckJEX.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +23 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +1 -1
- package/dist/multishot/index.d.ts +9 -9
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-CQmSxtcS.js → otel-spans-C8YIJWy1.js} +9 -4
- package/dist/{otel-spans-CQmSxtcS.js.map → otel-spans-C8YIJWy1.js.map} +1 -1
- package/dist/{outcome-store-BXlkwMPR.js → outcome-store-DUcMj-3D.js} +2 -2
- package/dist/{outcome-store-BXlkwMPR.js.map → outcome-store-DUcMj-3D.js.map} +1 -1
- package/dist/package-version-D7lQHt_-.js.map +1 -1
- package/dist/{paired-arms-DKZFVPt8.js → paired-arms-DpS_h8ON.js} +8 -5
- package/dist/{paired-arms-DKZFVPt8.js.map → paired-arms-DpS_h8ON.js.map} +1 -1
- package/dist/{paired-binary-fzkwENI4.js → paired-binary-OMBVRtc-.js} +2 -2
- package/dist/{paired-binary-fzkwENI4.js.map → paired-binary-OMBVRtc-.js.map} +1 -1
- package/dist/{paired-promotion-decision-D5B1Cxc2.js → paired-promotion-decision-DhrmjdV8.js} +5 -4
- package/dist/{paired-promotion-decision-D5B1Cxc2.js.map → paired-promotion-decision-DhrmjdV8.js.map} +1 -1
- package/dist/{paired-tests-C8iCsioC.js → paired-tests-Ccr16gcW.js} +3 -3
- package/dist/{paired-tests-C8iCsioC.js.map → paired-tests-Ccr16gcW.js.map} +1 -1
- package/dist/pipelines/index.d.ts +19 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +5 -4
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-K_CmO3dc.js → power-and-mde-bie6i965.js} +9 -6
- package/dist/{power-and-mde-K_CmO3dc.js.map → power-and-mde-bie6i965.js.map} +1 -1
- package/dist/{produced-state-DZMW8cuF.js → produced-state-CR2tAPLf.js} +8 -5
- package/dist/{produced-state-DZMW8cuF.js.map → produced-state-CR2tAPLf.js.map} +1 -1
- package/dist/{promotion-policy-C6FAJvmZ.js → promotion-policy-BI1RxzwD.js} +4 -4
- package/dist/{promotion-policy-C6FAJvmZ.js.map → promotion-policy-BI1RxzwD.js.map} +1 -1
- package/dist/{raw-provider-sink-CXuOhLvT.js → raw-provider-sink-rB2DYxr6.js} +2 -2
- package/dist/{raw-provider-sink-CXuOhLvT.js.map → raw-provider-sink-rB2DYxr6.js.map} +1 -1
- package/dist/{release-confidence-XQm_xjhR.js → release-confidence-CiFg1EPq.js} +3 -2
- package/dist/{release-confidence-XQm_xjhR.js.map → release-confidence-CiFg1EPq.js.map} +1 -1
- package/dist/{report-command-DKJ_yAIj.js → report-command-DzPpFgjB.js} +20 -18
- package/dist/{report-command-DKJ_yAIj.js.map → report-command-DzPpFgjB.js.map} +1 -1
- package/dist/reporting.js +5 -5
- package/dist/{reward-hacking-D0XwhVWE.js → reward-hacking-C-a__ShZ.js} +5 -3
- package/dist/{reward-hacking-D0XwhVWE.js.map → reward-hacking-C-a__ShZ.js.map} +1 -1
- package/dist/rl.d.ts +186 -186
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +21 -17
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +5 -5
- package/dist/{rollout-B-XBReTC.js → rollout-BUQixCoc.js} +5 -5
- package/dist/{rollout-B-XBReTC.js.map → rollout-BUQixCoc.js.map} +1 -1
- package/dist/{rubric-predictive-validity-CCK-1B7w.js → rubric-predictive-validity-B0Pg-3I5.js} +3 -3
- package/dist/{rubric-predictive-validity-CCK-1B7w.js.map → rubric-predictive-validity-B0Pg-3I5.js.map} +1 -1
- package/dist/{run-record-tNnlQIrD.js → run-record-BB290Y9A.js} +5 -3
- package/dist/{run-record-tNnlQIrD.js.map → run-record-BB290Y9A.js.map} +1 -1
- package/dist/{schema-C1aaAxTf.js → schema-DAMp77zB.js} +24 -20
- package/dist/{schema-C1aaAxTf.js.map → schema-DAMp77zB.js.map} +1 -1
- package/dist/search/index.d.ts +34 -34
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-AVgIjXaI.js → search-kernel-Dpniloks.js} +33 -26
- package/dist/{search-kernel-AVgIjXaI.js.map → search-kernel-Dpniloks.js.map} +1 -1
- package/dist/{search-ledger-CL0-mXQ8.js → search-ledger-CXQPwdWz.js} +4 -3
- package/dist/{search-ledger-CL0-mXQ8.js.map → search-ledger-CXQPwdWz.js.map} +1 -1
- package/dist/{search-shipper-Cs6K4km9.js → search-shipper-DhhofkDX.js} +3 -6
- package/dist/{search-shipper-Cs6K4km9.js.map → search-shipper-DhhofkDX.js.map} +1 -1
- package/dist/{search-summary-DvWk0hb3.js → search-summary-C1nAHgkB.js} +4 -4
- package/dist/{search-summary-DvWk0hb3.js.map → search-summary-C1nAHgkB.js.map} +1 -1
- package/dist/{semantic-concept-judge-2SE_kXGF.js → semantic-concept-judge-BcjYg37z.js} +4 -4
- package/dist/{semantic-concept-judge-2SE_kXGF.js.map → semantic-concept-judge-BcjYg37z.js.map} +1 -1
- package/dist/{sequential-Yz3OSskl.js → sequential-DWn_np6P.js} +2 -3
- package/dist/{sequential-Yz3OSskl.js.map → sequential-DWn_np6P.js.map} +1 -1
- package/dist/{server-D56r1lhp.js → server-B4jMhqQn.js} +2 -2
- package/dist/{server-D56r1lhp.js.map → server-B4jMhqQn.js.map} +1 -1
- package/dist/{skillopt-optimization-method-B2syNik0.js → skillopt-optimization-method-Bls9t-G_.js} +9 -9
- package/dist/{skillopt-optimization-method-B2syNik0.js.map → skillopt-optimization-method-Bls9t-G_.js.map} +1 -1
- package/dist/{statistical-heldout-BAFDCo07.js → statistical-heldout-CNLOvRq_.js} +3 -3
- package/dist/{statistical-heldout-BAFDCo07.js.map → statistical-heldout-CNLOvRq_.js.map} +1 -1
- package/dist/statistical-heldout-HswMvzbP.d.ts.map +1 -1
- package/dist/{store-DNe_Uv1Q.js → store-BhNuLdJw.js} +3 -5
- package/dist/{store-DNe_Uv1Q.js.map → store-BhNuLdJw.js.map} +1 -1
- package/dist/store-C4avFjIi.d.ts.map +1 -1
- package/dist/{store-otlp-BPowgPm6.js → store-otlp-yd9lDziN.js} +2 -2
- package/dist/{store-otlp-BPowgPm6.js.map → store-otlp-yd9lDziN.js.map} +1 -1
- package/dist/{store-tool-spans-BaGakFc2.js → store-tool-spans-Biio4_Nw.js} +4 -4
- package/dist/{store-tool-spans-BaGakFc2.js.map → store-tool-spans-Biio4_Nw.js.map} +1 -1
- package/dist/storyboard/index.d.ts +20 -21
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/{student-t-BA-Uy51p.js → student-t-BBZYka0E.js} +4 -3
- package/dist/{student-t-BA-Uy51p.js.map → student-t-BBZYka0E.js.map} +1 -1
- package/dist/{summary-report-BeYnvddy.js → summary-report-DLzptLSA.js} +8 -7
- package/dist/{summary-report-BeYnvddy.js.map → summary-report-DLzptLSA.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +25 -25
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +2 -2
- package/dist/{tool-waste-CJDfxH45.js → tool-waste-B-u3rzCz.js} +11 -6
- package/dist/{tool-waste-CJDfxH45.js.map → tool-waste-B-u3rzCz.js.map} +1 -1
- package/dist/trace-repair/index.d.ts +96 -96
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +1 -1
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +29 -29
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -7
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +35 -35
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +5 -5
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-Ki_gW7z8.js → tree-D48IJUK0.js} +39 -31
- package/dist/{tree-Ki_gW7z8.js.map → tree-D48IJUK0.js.map} +1 -1
- package/dist/wire/index.d.ts +48 -48
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -1
- package/package.json +18 -36
- package/dist/index-D4pC7Tvd2.d.ts.map +0 -1
- package/dist/index-DNgf5gyG2.d.ts.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as LlmClient } from "./llm-client-
|
|
1
|
+
import { n as LlmClient } from "./llm-client-Bhqoglar.js";
|
|
2
2
|
//#region src/analyst/chat-client.ts
|
|
3
3
|
/**
|
|
4
4
|
* Provider-neutral chat contract for every model call made by agent-eval.
|
|
@@ -78,7 +78,7 @@ function openAiCompatibleClient(opts) {
|
|
|
78
78
|
if (!baseUrl) throw new Error("createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.");
|
|
79
79
|
if (/\/chat\/completions$/.test(baseUrl)) throw new Error(`createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; this package appends /chat/completions. Got ${baseUrl}`);
|
|
80
80
|
if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) throw new Error("createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader is required. agent-eval reads no environment variable for a credential.");
|
|
81
|
-
const
|
|
81
|
+
const clientOptions = {
|
|
82
82
|
baseUrl,
|
|
83
83
|
...opts.apiKey ? { apiKey: opts.apiKey } : {},
|
|
84
84
|
...opts.bearer ? { bearer: opts.bearer } : {},
|
|
@@ -95,7 +95,8 @@ function openAiCompatibleClient(opts) {
|
|
|
95
95
|
...opts.rawSink === void 0 ? {} : { rawSink: opts.rawSink },
|
|
96
96
|
...opts.signal === void 0 ? {} : { signal: opts.signal },
|
|
97
97
|
...opts.thinking === void 0 ? {} : { thinking: opts.thinking }
|
|
98
|
-
}
|
|
98
|
+
};
|
|
99
|
+
const client = new LlmClient(clientOptions);
|
|
99
100
|
return {
|
|
100
101
|
transport: "openai-compatible",
|
|
101
102
|
...opts.defaultModel ? { defaultModel: opts.defaultModel } : {},
|
|
@@ -112,4 +113,4 @@ function openAiCompatibleClient(opts) {
|
|
|
112
113
|
//#endregion
|
|
113
114
|
export { createChatClient as t };
|
|
114
115
|
|
|
115
|
-
//# sourceMappingURL=chat-client-
|
|
116
|
+
//# sourceMappingURL=chat-client-DUhWNQQS.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat-client-CJORSG_J.js","names":[],"sources":["../src/analyst/chat-client.ts"],"sourcesContent":["/**\n * Provider-neutral chat contract for every model call made by agent-eval.\n *\n * The caller owns model execution. Evaluation code receives canonical requests\n * and results without importing a provider SDK, and agent-eval reads no\n * environment variable to find a provider credential: every transport is bound\n * at the call site.\n *\n * Two ways to bind one. `custom` and `sandbox-sdk` take a `chat` function the\n * caller wrote. `openai-compatible` takes an endpoint and a credential the\n * caller passes as values, and agent-eval issues the HTTP request itself.\n *\n * WHY THE THIRD ONE EXISTS. Every consumer that wanted a plain OpenAI-style\n * endpoint had to hand-write the same fetch loop — retry on 429 and 5xx,\n * `json_schema` degrade, fenced-JSON stripping, and, load-bearing here, the\n * `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed`\n * read. A hand-written transport that omits `servedModel` makes both identity\n * checks read `unreported`, so the two-vendor judge rule they enforce silently\n * measures nothing. Measured motive: agent-eval 0.172.1 exported no such\n * transport, so a two-model judge panel in discovery-lab reached\n * `createChatClient({ transport: 'custom', baseUrl, apiKey })` — fields that\n * exist on no transport option — and every call threw\n * `TypeError: opts.chat is not a function`, which surfaced as\n * `aggregateJudgeVerdicts: all 2 judges failed`.\n *\n * The credential still never comes from agent-eval's environment. `baseUrl`\n * and the bearer are required arguments with no default and no fallback, so a\n * misconfigured caller gets a refusal at construction, never a call to an\n * endpoint it did not name.\n */\n\nimport {\n type LlmCallRequest,\n type LlmCallResult,\n LlmClient,\n type LlmClientOptions,\n} from '../llm-client'\n\n/**\n * Unified chat interface using the package's canonical LLM request and result.\n */\nexport interface ChatClient {\n /** Display name of the bound transport, included in telemetry. */\n readonly transport: ChatTransport\n /** Default model when the caller omits one. */\n readonly defaultModel?: string\n /** Total provider attempts this transport can make for one chat call. */\n readonly maximumAttempts?: number\n\n /** Implementations must enforce `req.maxTokens` when it is present. */\n chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>\n}\n\nexport type ChatTransport =\n | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK\n | 'custom' // caller-adapted SDK or transport\n | 'openai-compatible' // caller-named /v1 endpoint, driven by this package\n | 'mock' // test-time injection\n\nexport interface ChatRequest extends Omit<LlmCallRequest, 'model'> {\n /** Optional — falls back to ChatClient.defaultModel. */\n model?: string\n}\n\nexport type ChatResponse = LlmCallResult\n\nexport interface ChatCallOpts {\n /** Cancel the in-flight request. */\n signal?: AbortSignal\n /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */\n maxCostUsd?: number\n /** Correlation tag carried into request headers when the transport allows. */\n correlationId?: string\n /** Stable provider idempotency key for retries/redrives of one paid call. */\n idempotencyKey?: string\n}\n\n// ── Factory ─────────────────────────────────────────────────────────\n\nexport type CreateChatClientOpts =\n | SandboxSdkTransportOpts\n | CustomTransportOpts\n | OpenAiCompatibleTransportOpts\n | MockTransportOpts\n\ninterface BaseTransportOpts {\n defaultModel?: string\n /** Total provider attempts. Required for opaque transports used in capped runs. */\n maximumAttempts?: number\n}\n\n/**\n * Sandbox-SDK transport. The caller supplies a canonical chat function for an\n * already-configured Sandbox handle, so agent-eval does not import the SDK.\n */\nexport interface SandboxSdkTransportOpts extends BaseTransportOpts {\n transport: 'sandbox-sdk'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */\nexport interface CustomTransportOpts extends BaseTransportOpts {\n transport: 'custom'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and\n * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.\n *\n * Use this instead of hand-rolling a `custom` transport around `fetch`. The\n * result carries `servedModel` — the model id the provider echoed, verbatim —\n * which is the only field `assertServedModel` and `assertCrossFamilyServed`\n * can read to witness a gateway answering from a different model than the one\n * requested.\n *\n * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this\n * package's to append. A `baseUrl` that already carries the path is a\n * construction error, not a request to a doubled URL.\n *\n * Exactly one credential form is required — `apiKey`, `bearer`, or\n * `authHeader`. There is no environment fallback: agent-eval never goes\n * looking for a key.\n */\nexport interface OpenAiCompatibleTransportOpts\n extends BaseTransportOpts,\n Pick<\n LlmClientOptions,\n | 'assertServedModel'\n | 'customTokenPricing'\n | 'deadlineMs'\n | 'defaultTimeoutMs'\n | 'fetch'\n | 'jsonPayloadMode'\n | 'jsonSchemaTransport'\n | 'provider'\n | 'rawSink'\n | 'signal'\n | 'thinking'\n > {\n transport: 'openai-compatible'\n /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */\n baseUrl: string\n /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */\n apiKey?: string\n /** Bearer credential, alternate spelling. */\n bearer?: string\n /** Non-bearer authorization header, for endpoints that want their own scheme. */\n authHeader?: { name: string; value: string }\n}\n\n/**\n * Mock transport for tests. The handler receives the request and returns\n * whatever the test wants. No retries, no JSON-schema degrade.\n */\nexport interface MockTransportOpts extends BaseTransportOpts {\n transport: 'mock'\n handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * Build a ChatClient bound to a specific transport. The returned client\n * is safe to share across analysts in a single registry run.\n */\nexport function createChatClient(opts: CreateChatClientOpts): ChatClient {\n switch (opts.transport) {\n case 'sandbox-sdk':\n return {\n transport: 'sandbox-sdk',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'custom':\n return {\n transport: 'custom',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'openai-compatible':\n return openAiCompatibleClient(opts)\n case 'mock':\n return {\n transport: 'mock',\n defaultModel: opts.defaultModel,\n maximumAttempts: 1,\n chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts),\n }\n }\n}\n\nfunction resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {\n if (req.model) return req\n if (!defaultModel) {\n throw new Error(\n 'ChatClient.chat: no model on request and no defaultModel on the client. ' +\n 'Either pass req.model or bind defaultModel at createChatClient().',\n )\n }\n return { ...req, model: defaultModel }\n}\n\n/**\n * Build the OpenAI-compatible client, refusing a half-configured one first.\n *\n * Both refusals are construction-time on purpose. A missing credential or a\n * doubled path is a caller mistake that a request would turn into a 401 or a\n * 404 from an endpoint the caller did not intend to reach — the failure then\n * reads as a provider problem instead of a configuration one.\n */\nfunction openAiCompatibleClient(opts: OpenAiCompatibleTransportOpts): ChatClient {\n const baseUrl = opts.baseUrl?.trim().replace(/\\/+$/, '') ?? ''\n if (!baseUrl) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.\",\n )\n }\n if (/\\/chat\\/completions$/.test(baseUrl)) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; \" +\n `this package appends /chat/completions. Got ${baseUrl}`,\n )\n }\n if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader \" +\n 'is required. agent-eval reads no environment variable for a credential.',\n )\n }\n\n const clientOptions: LlmClientOptions = {\n baseUrl,\n ...(opts.apiKey ? { apiKey: opts.apiKey } : {}),\n ...(opts.bearer ? { bearer: opts.bearer } : {}),\n ...(opts.authHeader ? { authHeader: opts.authHeader } : {}),\n ...(opts.maximumAttempts === undefined ? {} : { maximumAttempts: opts.maximumAttempts }),\n ...(opts.assertServedModel === undefined ? {} : { assertServedModel: opts.assertServedModel }),\n ...(opts.customTokenPricing === undefined\n ? {}\n : { customTokenPricing: opts.customTokenPricing }),\n ...(opts.deadlineMs === undefined ? {} : { deadlineMs: opts.deadlineMs }),\n ...(opts.defaultTimeoutMs === undefined ? {} : { defaultTimeoutMs: opts.defaultTimeoutMs }),\n ...(opts.fetch === undefined ? {} : { fetch: opts.fetch }),\n ...(opts.jsonPayloadMode === undefined ? {} : { jsonPayloadMode: opts.jsonPayloadMode }),\n ...(opts.jsonSchemaTransport === undefined\n ? {}\n : { jsonSchemaTransport: opts.jsonSchemaTransport }),\n ...(opts.provider === undefined ? {} : { provider: opts.provider }),\n ...(opts.rawSink === undefined ? {} : { rawSink: opts.rawSink }),\n ...(opts.signal === undefined ? {} : { signal: opts.signal }),\n ...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),\n }\n const client = new LlmClient(clientOptions)\n\n return {\n transport: 'openai-compatible',\n ...(opts.defaultModel ? { defaultModel: opts.defaultModel } : {}),\n maximumAttempts: client.maximumAttempts,\n chat: async (req, callOpts) => {\n const resolved = resolveModel(req, opts.defaultModel)\n return client.call(resolved as LlmCallRequest, {\n ...(callOpts?.signal ? { signal: callOpts.signal } : {}),\n ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}),\n })\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoKA,SAAgB,iBAAiB,MAAwC;CACvE,QAAQ,KAAK,WAAb;EACE,KAAK,eACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,UACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,qBACH,OAAO,uBAAuB,IAAI;EACpC,KAAK,QACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB;GACjB,MAAM,OAAO,KAAK,aAAa,KAAK,QAAQ,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EAC5F;CACJ;AACF;AAEA,SAAS,aAAa,KAAkB,cAA+C;CACrF,IAAI,IAAI,OAAO,OAAO;CACtB,IAAI,CAAC,cACH,MAAM,IAAI,MACR,2IAEF;CAEF,OAAO;EAAE,GAAG;EAAK,OAAO;CAAa;AACvC;;;;;;;;;AAUA,SAAS,uBAAuB,MAAiD;CAC/E,MAAM,UAAU,KAAK,SAAS,KAAK,CAAC,CAAC,QAAQ,QAAQ,EAAE,KAAK;CAC5D,IAAI,CAAC,SACH,MAAM,IAAI,MACR,+FACF;CAEF,IAAI,uBAAuB,KAAK,OAAO,GACrC,MAAM,IAAI,MACR,qIACiD,SACnD;CAEF,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,YACxD,MAAM,IAAI,MACR,oKAEF;CAyBF,MAAM,SAAS,IAAI,UAAU;EArB3B;EACA,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,aAAa,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;EACzD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,sBAAsB,KAAA,IAAY,CAAC,IAAI,EAAE,mBAAmB,KAAK,kBAAkB;EAC5F,GAAI,KAAK,uBAAuB,KAAA,IAC5B,CAAC,IACD,EAAE,oBAAoB,KAAK,mBAAmB;EAClD,GAAI,KAAK,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,YAAY,KAAK,WAAW;EACvE,GAAI,KAAK,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,kBAAkB,KAAK,iBAAiB;EACzF,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,wBAAwB,KAAA,IAC7B,CAAC,IACD,EAAE,qBAAqB,KAAK,oBAAoB;EACpD,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;EAC3D,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;CAE1B,CAAC;CAE1C,OAAO;EACL,WAAW;EACX,GAAI,KAAK,eAAe,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;EAC/D,iBAAiB,OAAO;EACxB,MAAM,OAAO,KAAK,aAAa;GAC7B,MAAM,WAAW,aAAa,KAAK,KAAK,YAAY;GACpD,OAAO,OAAO,KAAK,UAA4B;IAC7C,GAAI,UAAU,SAAS,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;IACtD,GAAI,UAAU,iBAAiB,EAAE,gBAAgB,SAAS,eAAe,IAAI,CAAC;GAChF,CAAC;EACH;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"chat-client-DUhWNQQS.js","names":[],"sources":["../src/analyst/chat-client.ts"],"sourcesContent":["/**\n * Provider-neutral chat contract for every model call made by agent-eval.\n *\n * The caller owns model execution. Evaluation code receives canonical requests\n * and results without importing a provider SDK, and agent-eval reads no\n * environment variable to find a provider credential: every transport is bound\n * at the call site.\n *\n * Two ways to bind one. `custom` and `sandbox-sdk` take a `chat` function the\n * caller wrote. `openai-compatible` takes an endpoint and a credential the\n * caller passes as values, and agent-eval issues the HTTP request itself.\n *\n * WHY THE THIRD ONE EXISTS. Every consumer that wanted a plain OpenAI-style\n * endpoint had to hand-write the same fetch loop — retry on 429 and 5xx,\n * `json_schema` degrade, fenced-JSON stripping, and, load-bearing here, the\n * `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed`\n * read. A hand-written transport that omits `servedModel` makes both identity\n * checks read `unreported`, so the two-vendor judge rule they enforce silently\n * measures nothing. Measured motive: agent-eval 0.172.1 exported no such\n * transport, so a two-model judge panel in discovery-lab reached\n * `createChatClient({ transport: 'custom', baseUrl, apiKey })` — fields that\n * exist on no transport option — and every call threw\n * `TypeError: opts.chat is not a function`, which surfaced as\n * `aggregateJudgeVerdicts: all 2 judges failed`.\n *\n * The credential still never comes from agent-eval's environment. `baseUrl`\n * and the bearer are required arguments with no default and no fallback, so a\n * misconfigured caller gets a refusal at construction, never a call to an\n * endpoint it did not name.\n */\n\nimport {\n type LlmCallRequest,\n type LlmCallResult,\n LlmClient,\n type LlmClientOptions,\n} from '../llm-client'\n\n/**\n * Unified chat interface using the package's canonical LLM request and result.\n */\nexport interface ChatClient {\n /** Display name of the bound transport, included in telemetry. */\n readonly transport: ChatTransport\n /** Default model when the caller omits one. */\n readonly defaultModel?: string\n /** Total provider attempts this transport can make for one chat call. */\n readonly maximumAttempts?: number\n\n /** Implementations must enforce `req.maxTokens` when it is present. */\n chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>\n}\n\nexport type ChatTransport =\n | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK\n | 'custom' // caller-adapted SDK or transport\n | 'openai-compatible' // caller-named /v1 endpoint, driven by this package\n | 'mock' // test-time injection\n\nexport interface ChatRequest extends Omit<LlmCallRequest, 'model'> {\n /** Optional — falls back to ChatClient.defaultModel. */\n model?: string\n}\n\nexport type ChatResponse = LlmCallResult\n\nexport interface ChatCallOpts {\n /** Cancel the in-flight request. */\n signal?: AbortSignal\n /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */\n maxCostUsd?: number\n /** Correlation tag carried into request headers when the transport allows. */\n correlationId?: string\n /** Stable provider idempotency key for retries/redrives of one paid call. */\n idempotencyKey?: string\n}\n\n// ── Factory ─────────────────────────────────────────────────────────\n\nexport type CreateChatClientOpts =\n | SandboxSdkTransportOpts\n | CustomTransportOpts\n | OpenAiCompatibleTransportOpts\n | MockTransportOpts\n\ninterface BaseTransportOpts {\n defaultModel?: string\n /** Total provider attempts. Required for opaque transports used in capped runs. */\n maximumAttempts?: number\n}\n\n/**\n * Sandbox-SDK transport. The caller supplies a canonical chat function for an\n * already-configured Sandbox handle, so agent-eval does not import the SDK.\n */\nexport interface SandboxSdkTransportOpts extends BaseTransportOpts {\n transport: 'sandbox-sdk'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */\nexport interface CustomTransportOpts extends BaseTransportOpts {\n transport: 'custom'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and\n * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.\n *\n * Use this instead of hand-rolling a `custom` transport around `fetch`. The\n * result carries `servedModel` — the model id the provider echoed, verbatim —\n * which is the only field `assertServedModel` and `assertCrossFamilyServed`\n * can read to witness a gateway answering from a different model than the one\n * requested.\n *\n * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this\n * package's to append. A `baseUrl` that already carries the path is a\n * construction error, not a request to a doubled URL.\n *\n * Exactly one credential form is required — `apiKey`, `bearer`, or\n * `authHeader`. There is no environment fallback: agent-eval never goes\n * looking for a key.\n */\nexport interface OpenAiCompatibleTransportOpts\n extends BaseTransportOpts,\n Pick<\n LlmClientOptions,\n | 'assertServedModel'\n | 'customTokenPricing'\n | 'deadlineMs'\n | 'defaultTimeoutMs'\n | 'fetch'\n | 'jsonPayloadMode'\n | 'jsonSchemaTransport'\n | 'provider'\n | 'rawSink'\n | 'signal'\n | 'thinking'\n > {\n transport: 'openai-compatible'\n /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */\n baseUrl: string\n /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */\n apiKey?: string\n /** Bearer credential, alternate spelling. */\n bearer?: string\n /** Non-bearer authorization header, for endpoints that want their own scheme. */\n authHeader?: { name: string; value: string }\n}\n\n/**\n * Mock transport for tests. The handler receives the request and returns\n * whatever the test wants. No retries, no JSON-schema degrade.\n */\nexport interface MockTransportOpts extends BaseTransportOpts {\n transport: 'mock'\n handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * Build a ChatClient bound to a specific transport. The returned client\n * is safe to share across analysts in a single registry run.\n */\nexport function createChatClient(opts: CreateChatClientOpts): ChatClient {\n switch (opts.transport) {\n case 'sandbox-sdk':\n return {\n transport: 'sandbox-sdk',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'custom':\n return {\n transport: 'custom',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'openai-compatible':\n return openAiCompatibleClient(opts)\n case 'mock':\n return {\n transport: 'mock',\n defaultModel: opts.defaultModel,\n maximumAttempts: 1,\n chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts),\n }\n }\n}\n\nfunction resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {\n if (req.model) return req\n if (!defaultModel) {\n throw new Error(\n 'ChatClient.chat: no model on request and no defaultModel on the client. ' +\n 'Either pass req.model or bind defaultModel at createChatClient().',\n )\n }\n return { ...req, model: defaultModel }\n}\n\n/**\n * Build the OpenAI-compatible client, refusing a half-configured one first.\n *\n * Both refusals are construction-time on purpose. A missing credential or a\n * doubled path is a caller mistake that a request would turn into a 401 or a\n * 404 from an endpoint the caller did not intend to reach — the failure then\n * reads as a provider problem instead of a configuration one.\n */\nfunction openAiCompatibleClient(opts: OpenAiCompatibleTransportOpts): ChatClient {\n const baseUrl = opts.baseUrl?.trim().replace(/\\/+$/, '') ?? ''\n if (!baseUrl) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.\",\n )\n }\n if (/\\/chat\\/completions$/.test(baseUrl)) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; \" +\n `this package appends /chat/completions. Got ${baseUrl}`,\n )\n }\n if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader \" +\n 'is required. agent-eval reads no environment variable for a credential.',\n )\n }\n\n const clientOptions: LlmClientOptions = {\n baseUrl,\n ...(opts.apiKey ? { apiKey: opts.apiKey } : {}),\n ...(opts.bearer ? { bearer: opts.bearer } : {}),\n ...(opts.authHeader ? { authHeader: opts.authHeader } : {}),\n ...(opts.maximumAttempts === undefined ? {} : { maximumAttempts: opts.maximumAttempts }),\n ...(opts.assertServedModel === undefined ? {} : { assertServedModel: opts.assertServedModel }),\n ...(opts.customTokenPricing === undefined\n ? {}\n : { customTokenPricing: opts.customTokenPricing }),\n ...(opts.deadlineMs === undefined ? {} : { deadlineMs: opts.deadlineMs }),\n ...(opts.defaultTimeoutMs === undefined ? {} : { defaultTimeoutMs: opts.defaultTimeoutMs }),\n ...(opts.fetch === undefined ? {} : { fetch: opts.fetch }),\n ...(opts.jsonPayloadMode === undefined ? {} : { jsonPayloadMode: opts.jsonPayloadMode }),\n ...(opts.jsonSchemaTransport === undefined\n ? {}\n : { jsonSchemaTransport: opts.jsonSchemaTransport }),\n ...(opts.provider === undefined ? {} : { provider: opts.provider }),\n ...(opts.rawSink === undefined ? {} : { rawSink: opts.rawSink }),\n ...(opts.signal === undefined ? {} : { signal: opts.signal }),\n ...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),\n }\n const client = new LlmClient(clientOptions)\n\n return {\n transport: 'openai-compatible',\n ...(opts.defaultModel ? { defaultModel: opts.defaultModel } : {}),\n maximumAttempts: client.maximumAttempts,\n chat: async (req, callOpts) => {\n const resolved = resolveModel(req, opts.defaultModel)\n return client.call(resolved as LlmCallRequest, {\n ...(callOpts?.signal ? { signal: callOpts.signal } : {}),\n ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}),\n })\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoKA,SAAgB,iBAAiB,MAAwC;CACvE,QAAQ,KAAK,WAAb;EACE,KAAK,eACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,UACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,qBACH,OAAO,uBAAuB,IAAI;EACpC,KAAK,QACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB;GACjB,MAAM,OAAO,KAAK,aAAa,KAAK,QAAQ,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EAC5F;CACJ;AACF;AAEA,SAAS,aAAa,KAAkB,cAA+C;CACrF,IAAI,IAAI,OAAO,OAAO;CACtB,IAAI,CAAC,cACH,MAAM,IAAI,MACR,2IAEF;CAEF,OAAO;EAAE,GAAG;EAAK,OAAO;CAAa;AACvC;;;;;;;;;AAUA,SAAS,uBAAuB,MAAiD;CAC/E,MAAM,UAAU,KAAK,SAAS,KAAK,CAAC,CAAC,QAAQ,QAAQ,EAAE,KAAK;CAC5D,IAAI,CAAC,SACH,MAAM,IAAI,MACR,+FACF;CAEF,IAAI,uBAAuB,KAAK,OAAO,GACrC,MAAM,IAAI,MACR,qIACiD,SACnD;CAEF,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,YACxD,MAAM,IAAI,MACR,oKAEF;CAGF,MAAM,gBAAkC;EACtC;EACA,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,aAAa,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;EACzD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,sBAAsB,KAAA,IAAY,CAAC,IAAI,EAAE,mBAAmB,KAAK,kBAAkB;EAC5F,GAAI,KAAK,uBAAuB,KAAA,IAC5B,CAAC,IACD,EAAE,oBAAoB,KAAK,mBAAmB;EAClD,GAAI,KAAK,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,YAAY,KAAK,WAAW;EACvE,GAAI,KAAK,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,kBAAkB,KAAK,iBAAiB;EACzF,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,wBAAwB,KAAA,IAC7B,CAAC,IACD,EAAE,qBAAqB,KAAK,oBAAoB;EACpD,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;EAC3D,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;CACnE;CACA,MAAM,SAAS,IAAI,UAAU,aAAa;CAE1C,OAAO;EACL,WAAW;EACX,GAAI,KAAK,eAAe,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;EAC/D,iBAAiB,OAAO;EACxB,MAAM,OAAO,KAAK,aAAa;GAC7B,MAAM,WAAW,aAAa,KAAK,KAAK,YAAY;GACpD,OAAO,OAAO,KAAK,UAA4B;IAC7C,GAAI,UAAU,SAAS,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;IACtD,GAAI,UAAU,iBAAiB,EAAE,gBAAgB,SAAS,eAAe,IAAI,CAAC;GAChF,CAAC;EACH;CACF;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { c as extractJsonPayload, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-
|
|
1
|
+
import { c as extractJsonPayload, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-Bhqoglar.js";
|
|
2
2
|
//#region src/chat-json-call.ts
|
|
3
3
|
/** Parse a JSON answer out of a model response. The transport may fence it. */
|
|
4
4
|
function parseJsonAnswer(response, actor) {
|
|
@@ -60,4 +60,4 @@ async function paidJsonChat(input) {
|
|
|
60
60
|
//#endregion
|
|
61
61
|
export { paidJsonChat as n, paidChat as t };
|
|
62
62
|
|
|
63
|
-
//# sourceMappingURL=chat-json-call-
|
|
63
|
+
//# sourceMappingURL=chat-json-call-Dx_sHzqm.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat-json-call-
|
|
1
|
+
{"version":3,"file":"chat-json-call-Dx_sHzqm.js","names":[],"sources":["../src/chat-json-call.ts"],"sourcesContent":["/**\n * One paid model call through a caller-owned `ChatClient`, metered by the\n * cost ledger.\n *\n * agent-eval executes no paid model: the transport is supplied by the caller\n * and the credential never enters this package. What stays here is the\n * accounting around the call — the priced maximum reserved before it runs, the\n * stable call id forwarded as the provider idempotency key, the receipt\n * settled from the response, and an honest unknown-usage receipt when the\n * transport failed.\n *\n * The judges, the wire judge endpoint, and the in-process trace-analysis\n * engine all make the same call in the same order; this is the one copy of\n * that sequence. `paidChat` is the sequence; `paidJsonChat` adds the JSON\n * answer parse that a structured caller needs.\n */\n\nimport type { ChatClient, ChatResponse } from './analyst/chat-client'\nimport type { CostChannel, CostLedgerHandle, CostReceipt, CustomTokenPricing } from './cost-ledger'\nimport {\n costReceiptFromLlm,\n costReceiptFromLlmError,\n extractJsonPayload,\n type LlmCallRequest,\n maximumChargeForLlmRequest,\n} from './llm-client'\n\nexport interface PaidChatInput {\n /** Caller-owned transport. One `chat()` call. */\n chat: ChatClient\n /** The exact canonical request, including its JSON-mode or schema fields. */\n request: LlmCallRequest\n ledger: CostLedgerHandle\n channel: CostChannel\n phase: string\n actor: string\n tags?: Record<string, string>\n signal?: AbortSignal\n /** Endpoint rates used when the transport reports no billed amount. */\n pricing?: CustomTokenPricing\n}\n\nexport type PaidJsonChatInput = PaidChatInput\n\nexport type PaidChatResult =\n | { succeeded: true; response: ChatResponse; receipt: CostReceipt }\n | { succeeded: false; error: Error; receipt?: CostReceipt }\n\nexport type PaidJsonChatResult<T> =\n | { succeeded: true; value: T; response: ChatResponse; receipt: CostReceipt }\n | { succeeded: false; error: Error; receipt?: CostReceipt }\n\n/** Parse a JSON answer out of a model response. The transport may fence it. */\nfunction parseJsonAnswer<T>(response: ChatResponse, actor: string): T {\n try {\n return JSON.parse(extractJsonPayload(response.content)) as T\n } catch (error) {\n throw new Error(\n `${actor}: model answer was not JSON — ${error instanceof Error ? error.message : String(error)}`,\n )\n }\n}\n\n/** Reserve, execute, and settle one caller-owned chat call. */\nexport async function paidChat(input: PaidChatInput): Promise<PaidChatResult> {\n const paid = await input.ledger.runPaidCall({\n channel: input.channel,\n phase: input.phase,\n actor: input.actor,\n model: input.request.model,\n ...(input.tags && Object.keys(input.tags).length > 0 ? { tags: input.tags } : {}),\n maximumCharge: maximumChargeForLlmRequest(input.request, {\n ...(input.chat.maximumAttempts === undefined\n ? {}\n : { maximumAttempts: input.chat.maximumAttempts }),\n ...(input.pricing ? { customTokenPricing: input.pricing } : {}),\n }),\n ...(input.signal ? { signal: input.signal } : {}),\n execute: (signal, callId) => input.chat.chat(input.request, { signal, idempotencyKey: callId }),\n receipt: (response) => costReceiptFromLlm(response, input.pricing),\n receiptFromError: (error) => costReceiptFromLlmError(error, input.pricing),\n })\n if (!paid.succeeded) {\n return {\n succeeded: false,\n error: paid.error,\n ...(paid.receipt ? { receipt: paid.receipt } : {}),\n }\n }\n return { succeeded: true, response: paid.value, receipt: paid.receipt }\n}\n\nexport async function paidJsonChat<T>(input: PaidJsonChatInput): Promise<PaidJsonChatResult<T>> {\n const paid = await paidChat(input)\n if (!paid.succeeded) return paid\n // The call completed and was billed. A malformed answer is a contract\n // failure AFTER the money was spent, so it keeps the settled receipt instead\n // of reporting the spend as unknown.\n try {\n return {\n succeeded: true,\n value: parseJsonAnswer<T>(paid.response, input.actor),\n response: paid.response,\n receipt: paid.receipt,\n }\n } catch (error) {\n return {\n succeeded: false,\n error: error instanceof Error ? error : new Error(String(error)),\n receipt: paid.receipt,\n }\n }\n}\n"],"mappings":";;;AAqDA,SAAS,gBAAmB,UAAwB,OAAkB;CACpE,IAAI;EACF,OAAO,KAAK,MAAM,mBAAmB,SAAS,OAAO,CAAC;CACxD,SAAS,OAAO;EACd,MAAM,IAAI,MACR,GAAG,MAAM,gCAAgC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAChG;CACF;AACF;;AAGA,eAAsB,SAAS,OAA+C;CAC5E,MAAM,OAAO,MAAM,MAAM,OAAO,YAAY;EAC1C,SAAS,MAAM;EACf,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM,QAAQ;EACrB,GAAI,MAAM,QAAQ,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;EAC/E,eAAe,2BAA2B,MAAM,SAAS;GACvD,GAAI,MAAM,KAAK,oBAAoB,KAAA,IAC/B,CAAC,IACD,EAAE,iBAAiB,MAAM,KAAK,gBAAgB;GAClD,GAAI,MAAM,UAAU,EAAE,oBAAoB,MAAM,QAAQ,IAAI,CAAC;EAC/D,CAAC;EACD,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;EAC/C,UAAU,QAAQ,WAAW,MAAM,KAAK,KAAK,MAAM,SAAS;GAAE;GAAQ,gBAAgB;EAAO,CAAC;EAC9F,UAAU,aAAa,mBAAmB,UAAU,MAAM,OAAO;EACjE,mBAAmB,UAAU,wBAAwB,OAAO,MAAM,OAAO;CAC3E,CAAC;CACD,IAAI,CAAC,KAAK,WACR,OAAO;EACL,WAAW;EACX,OAAO,KAAK;EACZ,GAAI,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,IAAI,CAAC;CAClD;CAEF,OAAO;EAAE,WAAW;EAAM,UAAU,KAAK;EAAO,SAAS,KAAK;CAAQ;AACxE;AAEA,eAAsB,aAAgB,OAA0D;CAC9F,MAAM,OAAO,MAAM,SAAS,KAAK;CACjC,IAAI,CAAC,KAAK,WAAW,OAAO;CAI5B,IAAI;EACF,OAAO;GACL,WAAW;GACX,OAAO,gBAAmB,KAAK,UAAU,MAAM,KAAK;GACpD,UAAU,KAAK;GACf,SAAS,KAAK;EAChB;CACF,SAAS,OAAO;EACd,OAAO;GACL,WAAW;GACX,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GAC/D,SAAS,KAAK;EAChB;CACF;AACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
2
|
import { i as compareCodeUnits } from "./canonical-BsZVgBhW.js";
|
|
3
|
-
import { g as readField } from "./ast-
|
|
3
|
+
import { g as readField } from "./ast-DJ6Bc7ab.js";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
//#region src/experiment/claim.ts
|
|
6
6
|
const nonEmpty = z.string().min(1).refine((value) => value.trim() === value);
|
|
@@ -51,4 +51,4 @@ function summarizeEvaluationUnits(claim, rows) {
|
|
|
51
51
|
//#endregion
|
|
52
52
|
export { evaluationClaimSchema as n, summarizeEvaluationUnits as r, defineEvaluationClaim as t };
|
|
53
53
|
|
|
54
|
-
//# sourceMappingURL=claim-
|
|
54
|
+
//# sourceMappingURL=claim-BXcZw-jo.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claim-
|
|
1
|
+
{"version":3,"file":"claim-BXcZw-jo.js","names":[],"sources":["../src/experiment/claim.ts"],"sourcesContent":["import { z } from 'zod'\nimport { ValidationError } from '../errors'\nimport { compareCodeUnits } from '../ledger-core/canonical'\nimport { type EvidenceRecord, readField } from './ast'\n\nconst nonEmpty = z\n .string()\n .min(1)\n .refine((value) => value.trim() === value)\n\n/** The claim's schema, shared by every record that binds a claim. */\nexport const evaluationClaimSchema = z\n .object({\n use: z.enum(['development', 'comparison', 'certification']),\n population: z.object({ id: nonEmpty, description: nonEmpty }).strict(),\n samplingFrame: nonEmpty,\n independentUnit: nonEmpty,\n generalization: z.enum(['fixed-roster', 'new-units']),\n minimumEffect: z.number().finite().positive().optional(),\n })\n .strict()\n\n/** The population and independent observations a measured result can describe. */\nexport type EvaluationClaim = z.infer<typeof evaluationClaimSchema>\n\n/** Validate a claim before binding it to a sealed experiment or final evidence. */\nexport function defineEvaluationClaim(input: EvaluationClaim): EvaluationClaim {\n const parsed = evaluationClaimSchema.safeParse(input)\n if (!parsed.success)\n throw new ValidationError(`invalid evaluation claim: ${parsed.error.message}`)\n return Object.freeze({ ...parsed.data, population: Object.freeze(parsed.data.population) })\n}\n\nexport interface EvaluationUnitSummary {\n observations: number\n independentUnits: number\n units: Array<{ id: string; observations: number }>\n}\n\n/** Repetitions retain their denominator without becoming additional independent units. */\nexport function summarizeEvaluationUnits(\n claim: EvaluationClaim,\n rows: readonly object[],\n): EvaluationUnitSummary {\n const validated = defineEvaluationClaim(claim)\n const counts = new Map<string, number>()\n for (const [index, row] of rows.entries()) {\n if (row === null || typeof row !== 'object' || Array.isArray(row)) {\n throw new ValidationError(`evaluation claim: row ${index} must be an object`)\n }\n const value = readField(row as EvidenceRecord, validated.independentUnit)\n if (typeof value !== 'string' || !value.trim() || value.trim() !== value) {\n throw new ValidationError(\n `evaluation claim: row ${index} needs a nonempty string at '${validated.independentUnit}'`,\n )\n }\n counts.set(value, (counts.get(value) ?? 0) + 1)\n }\n return {\n observations: rows.length,\n independentUnits: counts.size,\n units: [...counts]\n .sort(([left], [right]) => compareCodeUnits(left, right))\n .map(([id, observations]) => ({ id, observations })),\n }\n}\n"],"mappings":";;;;;AAKA,MAAM,WAAW,EACd,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,QAAQ,UAAU,MAAM,KAAK,MAAM,KAAK;;AAG3C,MAAa,wBAAwB,EAClC,OAAO;CACN,KAAK,EAAE,KAAK;EAAC;EAAe;EAAc;CAAe,CAAC;CAC1D,YAAY,EAAE,OAAO;EAAE,IAAI;EAAU,aAAa;CAAS,CAAC,CAAC,CAAC,OAAO;CACrE,eAAe;CACf,iBAAiB;CACjB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,WAAW,CAAC;CACpD,eAAe,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;AACzD,CAAC,CAAC,CACD,OAAO;;AAMV,SAAgB,sBAAsB,OAAyC;CAC7E,MAAM,SAAS,sBAAsB,UAAU,KAAK;CACpD,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,gBAAgB,6BAA6B,OAAO,MAAM,SAAS;CAC/E,OAAO,OAAO,OAAO;EAAE,GAAG,OAAO;EAAM,YAAY,OAAO,OAAO,OAAO,KAAK,UAAU;CAAE,CAAC;AAC5F;;AASA,SAAgB,yBACd,OACA,MACuB;CACvB,MAAM,YAAY,sBAAsB,KAAK;CAC7C,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GAAG;EACzC,IAAI,QAAQ,QAAQ,OAAO,QAAQ,YAAY,MAAM,QAAQ,GAAG,GAC9D,MAAM,IAAI,gBAAgB,yBAAyB,MAAM,mBAAmB;EAE9E,MAAM,QAAQ,UAAU,KAAuB,UAAU,eAAe;EACxE,IAAI,OAAO,UAAU,YAAY,CAAC,MAAM,KAAK,KAAK,MAAM,KAAK,MAAM,OACjE,MAAM,IAAI,gBACR,yBAAyB,MAAM,+BAA+B,UAAU,gBAAgB,EAC1F;EAEF,OAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAChD;CACA,OAAO;EACL,cAAc,KAAK;EACnB,kBAAkB,OAAO;EACzB,OAAO,CAAC,GAAG,MAAM,CAAC,CACf,MAAM,CAAC,OAAO,CAAC,WAAW,iBAAiB,MAAM,KAAK,CAAC,CAAC,CACxD,KAAK,CAAC,IAAI,mBAAmB;GAAE;GAAI;EAAa,EAAE;CACvD;AACF"}
|
package/dist/cli.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { g as INSUFFICIENT_FROM, h as operatorYield } from "./search-kernel-
|
|
3
|
-
import { i as replaySearchLedgerText } from "./search-ledger-
|
|
4
|
-
import { t as renderSearchSummary } from "./search-summary-
|
|
5
|
-
import { c as hostedTenantFromEnv, g as SearchRunKindSchema, r as shipSearchLedger, t as SearchShipConflictError, u as SEARCH_LEDGER_BATCH_MAX_BYTES } from "./search-shipper-
|
|
6
|
-
import { i as runRolloutReleaseCli } from "./hf-dataset-
|
|
7
|
-
import { t as createChatClient } from "./chat-client-
|
|
8
|
-
import { t as runAnalystBenchmarkCommand } from "./benchmark-command-
|
|
9
|
-
import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-
|
|
10
|
-
import { A as editCreditText, E as front, f as renderMetaSearchText, h as landscape, k as editCredit, l as metaSearch, m as formatLandscape, n as taskMatrix, o as skillManifold, r as formatSkillManifold, t as tree, x as surfaceTextEdits } from "./tree-
|
|
11
|
-
import { t as runSupervisorRunCommand } from "./report-command-
|
|
2
|
+
import { g as INSUFFICIENT_FROM, h as operatorYield } from "./search-kernel-Dpniloks.js";
|
|
3
|
+
import { i as replaySearchLedgerText } from "./search-ledger-CXQPwdWz.js";
|
|
4
|
+
import { t as renderSearchSummary } from "./search-summary-C1nAHgkB.js";
|
|
5
|
+
import { c as hostedTenantFromEnv, g as SearchRunKindSchema, r as shipSearchLedger, t as SearchShipConflictError, u as SEARCH_LEDGER_BATCH_MAX_BYTES } from "./search-shipper-DhhofkDX.js";
|
|
6
|
+
import { i as runRolloutReleaseCli } from "./hf-dataset-kjIfIK_e.js";
|
|
7
|
+
import { t as createChatClient } from "./chat-client-DUhWNQQS.js";
|
|
8
|
+
import { t as runAnalystBenchmarkCommand } from "./benchmark-command-DFs_i62N.js";
|
|
9
|
+
import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-B4jMhqQn.js";
|
|
10
|
+
import { A as editCreditText, E as front, f as renderMetaSearchText, h as landscape, k as editCredit, l as metaSearch, m as formatLandscape, n as taskMatrix, o as skillManifold, r as formatSkillManifold, t as tree, x as surfaceTextEdits } from "./tree-D48IJUK0.js";
|
|
11
|
+
import { t as runSupervisorRunCommand } from "./report-command-DzPpFgjB.js";
|
|
12
12
|
import { createHash } from "node:crypto";
|
|
13
13
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
14
14
|
import { dirname, join } from "node:path";
|
|
@@ -207,7 +207,8 @@ async function runShowCommand(argv) {
|
|
|
207
207
|
}
|
|
208
208
|
const path = positional[0];
|
|
209
209
|
const searchId = await firstLineSearchId(path);
|
|
210
|
-
const
|
|
210
|
+
const text = await readFile(path, "utf8");
|
|
211
|
+
const state = replaySearchLedgerText(text, searchId, path);
|
|
211
212
|
const split = state.header && state.header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
212
213
|
const lenses = showLenses(state, path, flags);
|
|
213
214
|
if (flags.has("--json")) {
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","names":[],"sources":["../src/cli-config.ts","../src/search-command.ts","../src/cli.ts"],"sourcesContent":["/**\n * Provider configuration for the `agent-eval` binary.\n *\n * The binary reads endpoint and credential values from its environment.\n * TypeScript library callers bind a transport explicitly. The maintained\n * HTTP transport accepts their credential and sends provider requests.\n */\n\nimport { type ChatClient, createChatClient } from './analyst/chat-client'\n\nexport interface CliLlmConfig {\n /** Judge transport for the wire handlers. Absent when no provider is configured. */\n chat?: ChatClient\n model?: string\n}\n\n/** The endpoint the binary resolved from its environment. */\nexport interface CliProviderRoute {\n baseUrl: string\n apiKey: string\n model?: string\n}\n\n/**\n * Environment precedence for the binary's provider route, with no client built.\n *\n * Both halves are required: a base URL without a key, or a key without an\n * endpoint, is a half-configured server. `/v1/judge` then refuses with\n * `llm_not_configured` instead of calling an unintended endpoint.\n */\nexport function resolveCliProviderRoute(\n env: NodeJS.ProcessEnv = process.env,\n): CliProviderRoute | undefined {\n const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n const baseUrl =\n explicitBaseUrl ??\n nonEmpty(env.OPENAI_BASE_URL) ??\n nonEmpty(env.TANGLE_ROUTER_URL) ??\n (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n const model =\n nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n if (!baseUrl || !apiKey) return undefined\n return { baseUrl, apiKey, ...(model ? { model } : {}) }\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n const route = resolveCliProviderRoute(env)\n if (!route) return {}\n return {\n chat: cliChatClient({ baseUrl: route.baseUrl, apiKey: route.apiKey }, route.model),\n ...(route.model ? { model: route.model } : {}),\n }\n}\n\n/**\n * The binary's transport IS the published one — `createChatClient({ transport:\n * 'openai-compatible' })` — so the server and a library consumer cannot drift\n * onto two different clients for the same endpoint.\n *\n * The one thing kept local is the error text. A generic \"no model on the\n * request\" tells a server operator nothing; this names the variable to set.\n */\nfunction cliChatClient(\n route: { baseUrl: string; apiKey: string },\n defaultModel: string | undefined,\n): ChatClient {\n const client = createChatClient({\n transport: 'openai-compatible',\n baseUrl: route.baseUrl,\n apiKey: route.apiKey,\n ...(defaultModel ? { defaultModel } : {}),\n })\n return {\n ...client,\n chat: (req, callOpts) => {\n if (!req.model && !defaultModel) {\n throw new Error(\n 'agent-eval: no model on the request and no AGENT_EVAL_LLM_MODEL configured',\n )\n }\n return client.chat(req, callOpts)\n },\n }\n}\n\nfunction nonEmpty(value: string | undefined): string | undefined {\n const trimmed = value?.trim()\n return trimmed ? trimmed : undefined\n}\n","/**\n * `agent-eval search <subcommand>`: work with a search ledger from a terminal.\n *\n * agent-eval search ship <search-ledger.jsonl> --run-kind optimization|eval [--content full|digests]\n * agent-eval search show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n * [--edit-credit] [--landscape] [--skill-manifold] [--json]\n * agent-eval search show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n *\n * `ship` sends the ledger to the hosted store named by `TANGLE_INGEST_URL`,\n * `TANGLE_INGEST_API_KEY` and `TANGLE_TENANT_ID`, starting from the store's\n * head, so it finishes or resumes a ship that a loop could not complete.\n *\n * `show` verifies the ledger and prints `renderSearchSummary`: the same\n * compact text a proposer reads, on the search's own ranking split (the\n * selection split when the search declares one, else train). There is no\n * local HTML renderer; the hosted store's page is the visual view (§5 of the\n * search-tree design). A lens flag adds that lens's text form after the\n * summary, so an agent reads what a person sees; `--json` prints the lens\n * results as JSON instead, the shape a view renders.\n *\n * `show --meta` reads every ledger it is given and prints the `metaSearch`\n * lens over them instead of one search's summary: each search as one node\n * scored by its held-out lift per known dollar, the configurations best first,\n * and the searches arranged by derivation and containment.\n */\n\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { open, readFile } from 'node:fs/promises'\nimport { dirname, join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { replaySearchLedgerText } from './campaign/search-ledger'\nimport type { SearchArtifactRef } from './campaign/search-ledger-types'\nimport type { SearchStateView } from './campaign/search-state'\nimport { renderSearchSummary } from './campaign/search-summary'\nimport { hostedTenantFromEnv } from './hosted/client'\nimport { SEARCH_LEDGER_BATCH_MAX_BYTES, SearchRunKindSchema } from './hosted/search-ledger-wire'\nimport { SearchShipConflictError, shipSearchLedger } from './hosted/search-shipper'\nimport { editCredit, editCreditText } from './search/lenses/edit-credit'\nimport { type FrontData, front } from './search/lenses/front'\nimport { formatLandscape, landscape, surfaceTextEdits } from './search/lenses/landscape'\nimport { metaSearch, renderMetaSearchText } from './search/lenses/meta-search'\nimport { type OperatorYieldData, operatorYield } from './search/lenses/operator-yield'\nimport { INSUFFICIENT_FROM } from './search/lenses/shared'\nimport { formatSkillManifold, skillManifold } from './search/lenses/skill-manifold'\nimport { type TaskMatrixData, taskMatrix } from './search/lenses/task-matrix'\nimport { type TreeData, type TreeNode, tree } from './search/lenses/tree'\n\nconst USAGE = `usage: agent-eval search <subcommand> ...\n\n ship <search-ledger.jsonl> --run-kind optimization|eval [--content full|digests]\n Ships the ledger's entries, and the blobs they name, to the hosted store in\n TANGLE_INGEST_URL (or TANGLE_ORCHESTRATOR_URL) as tenant TANGLE_TENANT_ID with\n TANGLE_INGEST_API_KEY (or TANGLE_API_KEY). It starts from the store's head, so\n running it again sends only what the store lacks. Prints the result as JSON.\n Exits 1 when the store holds a different chain for the search.\n\n show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n [--edit-credit] [--landscape] [--skill-manifold] [--json]\n Verifies the ledger and prints its search summary: the leading nodes\n against the root, the most recently discarded nodes and why, and a log\n of recent proposals. The same text a proposer reads as context, on the\n search's own ranking split. Each lens flag adds that lens's text form\n (search-tree-design §12) below the summary — the same JSON\n Intelligence, discovery lab, VerticalBench and agent-runtime read.\n --tree the tidy tree of nodes and edges.\n --operator-yield outcome counts and yield per known dollar, by edge operator.\n --front the Pareto front over score and known cost per cell.\n --task-matrix nodes × units, clustered, with specialist gain per cluster.\n --edit-credit every edit as a gene followed down the lineage, its\n credit, interacting pairs, and skill candidates. It\n reads node artifacts from the blobs the ledger names,\n verified by digest.\n --landscape nodes placed by line edits between their profile\n surfaces (read from the blobs, verified by digest), the\n kriged score surface, its basins, and the plateau score\n a policy drafts on.\n --skill-manifold the node × unit score matrix factored into skill axes,\n its cross-validated intrinsic dimension, and the unit\n that best separates the leaders.\n --json prints the requested lenses as JSON instead of text.\n\n show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n Verifies every ledger and prints the meta-search lens over them: each\n search as one node, scored by its claim's held-out lift per known\n dollar (unscored searches say why), configurations best first with\n their interval, method and n, and the searches arranged by derivation\n and containment. --objective names the objective the best\n configuration is chosen within when the searches span several.`\n\nexport async function runSearchCommand(argv: string[]): Promise<number> {\n const [subcommand, ...rest] = argv\n if (subcommand === undefined || subcommand === '--help' || subcommand === '-h') {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (subcommand === 'show') return await runShowCommand(rest)\n if (subcommand !== 'ship') {\n process.stderr.write(`unknown search subcommand: ${subcommand}\\n${USAGE}\\n`)\n return 1\n }\n if (rest.some((arg) => arg === undefined || arg === '--help' || arg === '-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n const { path, flags } = parseShipArgs(rest)\n const runKind = SearchRunKindSchema.safeParse(flags['run-kind'])\n if (!runKind.success) throw new Error(`--run-kind must be optimization or eval\\n${USAGE}`)\n const content = flags.content ?? 'full'\n if (content !== 'full' && content !== 'digests') {\n throw new Error(`--content must be full or digests\\n${USAGE}`)\n }\n const tenant = hostedTenantFromEnv()\n if (!tenant) {\n throw new Error(\n 'set TANGLE_INGEST_URL, TANGLE_INGEST_API_KEY and TANGLE_TENANT_ID to name the store',\n )\n }\n const searchId = await firstLineSearchId(path)\n try {\n const shipped = await shipSearchLedger({\n tenant,\n ledger: { path, searchId },\n runKind: runKind.data,\n content,\n })\n process.stdout.write(`${JSON.stringify(shipped, null, 2)}\\n`)\n return shipped.head.nextSequence === shipped.localLines ? 0 : 1\n } catch (error) {\n if (!(error instanceof SearchShipConflictError)) throw error\n process.stderr.write(`${error.message}\\n`)\n return 1\n }\n}\n\nasync function runShowCommand(argv: string[]): Promise<number> {\n if (argv.includes('--help') || argv.includes('-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (argv.includes('--meta')) return await runShowMetaCommand(argv)\n const positional = argv.filter((arg) => !arg.startsWith('--'))\n const flags = new Set(argv.filter((arg) => arg.startsWith('--')))\n const unknown = [...flags].filter((flag) => !SHOW_FLAGS.has(flag))\n if (positional.length !== 1 || unknown.length > 0) {\n if (unknown.length > 0) process.stderr.write(`unknown flag ${unknown.join(', ')}\\n`)\n process.stdout.write(`${USAGE}\\n`)\n return 1\n }\n const path = positional[0]!\n const searchId = await firstLineSearchId(path)\n const text = await readFile(path, 'utf8')\n const state = replaySearchLedgerText(text, searchId, path)\n const split =\n state.header && state.header.splits.selection.tasks.length > 0 ? 'selection' : 'train'\n const lenses = showLenses(state, path, flags)\n if (flags.has('--json')) {\n if (lenses.length === 0) {\n process.stderr.write('--json needs a lens flag\\n')\n return 1\n }\n const json = Object.fromEntries(lenses.map((lens) => [lens.name, lens.result]))\n process.stdout.write(`${JSON.stringify(json, null, 2)}\\n`)\n return 0\n }\n const sections = [renderSearchSummary(state, { split }), ...lenses.map((lens) => lens.text)]\n process.stdout.write(`${sections.join('\\n\\n')}\\n`)\n return 0\n}\n\nasync function runShowMetaCommand(argv: string[]): Promise<number> {\n const paths: string[] = []\n let objective: string | undefined\n let json = false\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (token === '--meta') continue\n if (token === '--json') {\n json = true\n continue\n }\n if (token === '--objective') {\n objective = argv[++index]\n if (objective === undefined) throw new Error(`--objective needs a value\\n${USAGE}`)\n continue\n }\n if (token.startsWith('--')) throw new Error(`unknown flag ${token}\\n${USAGE}`)\n paths.push(token)\n }\n if (paths.length === 0) throw new Error(`show --meta needs at least one ledger\\n${USAGE}`)\n const states = []\n for (const path of paths) {\n const searchId = await firstLineSearchId(path)\n states.push(replaySearchLedgerText(await readFile(path, 'utf8'), searchId, path))\n }\n const lens = metaSearch(states, objective === undefined ? {} : { objective })\n process.stdout.write(\n json ? `${JSON.stringify({ metaSearch: lens }, null, 2)}\\n` : `${renderMetaSearchText(lens)}\\n`,\n )\n return 0\n}\n\nconst SHOW_FLAGS = new Set([\n '--edit-credit',\n '--json',\n '--tree',\n '--operator-yield',\n '--front',\n '--task-matrix',\n '--landscape',\n '--skill-manifold',\n])\n\nfunction showLenses(\n state: SearchStateView,\n ledgerPath: string,\n flags: ReadonlySet<string>,\n): Array<{ name: string; result: unknown; text: string }> {\n const lenses: Array<{ name: string; result: unknown; text: string }> = []\n if (flags.has('--tree')) {\n const result = tree(state)\n lenses.push({ name: 'tree', result, text: renderTreeText(result.data) })\n }\n if (flags.has('--operator-yield')) {\n const result = operatorYield(state)\n lenses.push({ name: 'operatorYield', result, text: renderOperatorYieldText(result.data) })\n }\n if (flags.has('--front')) {\n const result = front(state)\n lenses.push({ name: 'front', result, text: renderFrontText(result.data) })\n }\n if (flags.has('--task-matrix')) {\n const result = taskMatrix(state)\n lenses.push({ name: 'taskMatrix', result, text: renderTaskMatrixText(result.data) })\n }\n if (flags.has('--edit-credit')) {\n const result = editCredit(state, { readArtifact: ledgerBlobReader(ledgerPath) })\n lenses.push({ name: 'editCredit', result, text: editCreditText(result) })\n }\n if (flags.has('--landscape')) {\n const read = ledgerBlobReader(ledgerPath)\n const result = landscape(\n state,\n surfaceTextEdits((surface) => read(surface.artifact) ?? null),\n )\n lenses.push({ name: 'landscape', result, text: formatLandscape(result) })\n }\n if (flags.has('--skill-manifold')) {\n const result = skillManifold(state)\n lenses.push({ name: 'skillManifold', result, text: formatSkillManifold(result) })\n }\n return lenses\n}\n\n/**\n * Reads a blob a ledger names: at its `file:` URI, else under `blobs/` beside\n * the ledger, where `SearchRecorder` writes them. Bytes that do not hash to the\n * reference's digest and length read as unavailable, never as content.\n */\nfunction ledgerBlobReader(ledgerPath: string): (ref: SearchArtifactRef) => unknown | undefined {\n const cache = new Map<string, unknown>()\n return (ref) => {\n if (cache.has(ref.sha256)) return cache.get(ref.sha256)\n const hex = ref.sha256.slice('sha256:'.length)\n const candidates = [join(dirname(ledgerPath), 'blobs', `${hex}.json`)]\n if (ref.uri.startsWith('file:')) candidates.unshift(fileURLToPath(ref.uri))\n let value: unknown\n for (const candidate of candidates) {\n if (!existsSync(candidate)) continue\n const bytes = readFileSync(candidate)\n const digest = createHash('sha256').update(bytes).digest('hex')\n if (digest !== hex || bytes.byteLength !== ref.byteLength) continue\n try {\n value = JSON.parse(bytes.toString('utf8'))\n } catch {\n // Verified bytes that are not JSON are not content a lens can read.\n }\n break\n }\n cache.set(ref.sha256, value)\n return value\n }\n}\n\nfunction renderTreeText(data: TreeData): string {\n const lines = [\n `tree: ${data.nodeCount} node${data.nodeCount === 1 ? '' : 's'}, ${data.edgeCount} edge${data.edgeCount === 1 ? '' : 's'}`,\n ]\n const walk = (node: TreeNode, prefix: string, isLast: boolean): void => {\n const branch = prefix === '' ? '' : isLast ? '└─ ' : '├─ '\n const op = node.operator ? `${node.operator} → ` : ''\n const rung = node.rung === null ? '' : ` rung ${node.rung}`\n const depth = node.depth === null ? ' depth unknown' : ''\n const cost =\n node.unknownCostCells === 0\n ? `$${node.knownCostUsd.toFixed(2)}`\n : `$${node.knownCostUsd.toFixed(2)} known + at least $${node.floorUsd.toFixed(2)} over ${node.unknownCostCells} unknown-cost cell${node.unknownCostCells === 1 ? '' : 's'}`\n lines.push(\n `${prefix}${branch}${op}${node.nodeId} (${node.status ?? 'undecided'}${rung}${depth}) ${cost}`,\n )\n const childPrefix = prefix + (prefix === '' ? '' : isLast ? ' ' : '│ ')\n node.children.forEach((child, index) => {\n walk(child, childPrefix, index === node.children.length - 1)\n })\n }\n data.roots.forEach((root, index) => {\n walk(root, '', index === data.roots.length - 1)\n })\n return lines.join('\\n')\n}\n\nfunction renderOperatorYieldText(data: OperatorYieldData): string {\n const lines = [`operator yield (${data.split} split, gain per known $ of proposal + screen):`]\n if (data.rows.length === 0) lines.push(' no edges yet')\n for (const row of data.rows) {\n const y = row.yield\n const outcomes = Object.entries(row.outcomes)\n .filter(([, count]) => count > 0)\n .map(([status, count]) => `${status}=${count}`)\n .join(' ')\n const yieldText =\n y.method === 'none'\n ? y.n === 0\n ? 'yield: no measured children'\n : `yield unknown (${y.n} of ${INSUFFICIENT_FROM}, none)`\n : y.method === 'insufficient'\n ? `yield≈${y.mean!.toFixed(4)}/$ (${y.n} of ${INSUFFICIENT_FROM}, insufficient)`\n : `yield≈${y.mean!.toFixed(4)}/$ [${y.interval![0].toFixed(4)}, ${y.interval![1].toFixed(4)}] (${y.method}, ${y.intervalMethod}, n=${y.n})`\n const nodes = Object.values(row.outcomes).reduce((sum, count) => sum + count, 0)\n const reproposals = row.reproposals > 0 ? `, ${row.reproposals} re-proposals` : ''\n lines.push(\n ` ${row.operator}: ${row.proposals} proposals${reproposals} → ${nodes} node${nodes === 1 ? '' : 's'} (${outcomes || 'no decisions yet'}); ${yieldText}; ${row.excluded} excluded`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderFrontText(data: FrontData): string {\n const onFront = data.rows.filter((row) => row.onFront)\n const reasons = Object.entries(data.exclusions)\n .filter(([, count]) => count > 0)\n .map(([reason, count]) => `${reason} ${count}`)\n .join(', ')\n const lines = [\n `front (${data.split} split, axes: ${data.axes.join(', ')}): ${data.frontierSize} of ${data.rows.length} node(s) on the frontier, ${data.excludedCount} excluded${reasons ? ` (${reasons})` : ''}`,\n ]\n for (const row of onFront) {\n lines.push(\n ` ${row.nodeId}: score=${row.score!.toFixed(4)} on ${row.units} units (${row.method}) costPerCell=$${row.costPerCellUsd!.toFixed(4)}`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderTaskMatrixText(data: TaskMatrixData): string {\n const lines = [\n `task matrix (${data.split} split, ${data.direction}): ${data.nodeIds.length} node(s) × ${data.unitIds.length} unit(s), ${data.nodeClusters.length} node cluster(s), ${data.unitClusters.length} unit cluster(s)`,\n ]\n for (const row of data.specialistGain) {\n const gain =\n row.gain === null\n ? `insufficient (${row.insufficient})`\n : `${row.gain.toFixed(4)} (best ${row.bestNodeId}, ${row.nodesContributing} nodes on ${row.unitIds.length} units)`\n lines.push(` cluster ${row.clusterId} [${row.unitIds.join(', ')}]: specialist gain ${gain}`)\n }\n return lines.join('\\n')\n}\n\nfunction parseShipArgs(argv: string[]): { path: string; flags: Record<string, string> } {\n const flags: Record<string, string> = {}\n const positional: string[] = []\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (!token.startsWith('--')) {\n positional.push(token)\n continue\n }\n const [name, inline] = token.slice(2).split('=', 2) as [string, string | undefined]\n if (name !== 'run-kind' && name !== 'content') throw new Error(`unknown flag --${name}`)\n const value = inline ?? argv[++index]\n if (value === undefined) throw new Error(`--${name} needs a value`)\n flags[name] = value\n }\n if (positional.length !== 1) throw new Error(USAGE)\n return { path: positional[0]!, flags }\n}\n\n/** The ledger names its search on every line; read the first. */\nasync function firstLineSearchId(path: string): Promise<string> {\n const handle = await open(path, 'r')\n try {\n // A line longer than one request's limit cannot ship anyway.\n const bytes = Buffer.alloc(SEARCH_LEDGER_BATCH_MAX_BYTES + 1)\n const { bytesRead } = await handle.read(bytes, 0, bytes.byteLength, 0)\n const newline = bytes.subarray(0, bytesRead).indexOf(0x0a)\n if (newline < 0) throw new Error(`${path} holds no complete ledger line`)\n const first = JSON.parse(bytes.toString('utf8', 0, newline)) as { searchId?: unknown }\n if (typeof first.searchId !== 'string') throw new Error(`${path} line 1 names no searchId`)\n return first.searchId\n } finally {\n await handle.close()\n }\n}\n","#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n * agent-eval serve [--port 5005] [--host 127.0.0.1]\n * agent-eval rpc <method> # one request from stdin → one response on stdout\n * agent-eval rpc-batch <method> # JSONL stdin → JSONL stdout\n * agent-eval analyst-benchmark ... # benchmark a real-model trace analyst\n * agent-eval supervisor-run report <runDir> [--format headline|markdown|json]\n * agent-eval search ship <search-ledger.jsonl> --run-kind optimization|eval\n * agent-eval search show <search-ledger.jsonl>\n * agent-eval openapi [--out path] # write OpenAPI spec\n * agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { runAnalystBenchmarkCommand } from './analyst/benchmark-command'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { runRolloutReleaseCli } from './rollout/release/hf-dataset'\nimport { runSearchCommand } from './search-command'\nimport { runSupervisorRunCommand } from './supervisor-run/report-command'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n command: string\n positional: string[]\n flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n const [command, ...rest] = argv\n const positional: string[] = []\n const flags: Record<string, string> = {}\n for (let i = 0; i < rest.length; i++) {\n const tok = rest[i]!\n if (tok.startsWith('--')) {\n const raw = tok.slice(2)\n const equalsAt = raw.indexOf('=')\n if (equalsAt >= 0) {\n flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n continue\n }\n const key = raw\n if (key === 'help') {\n flags[key] = 'true'\n continue\n }\n const next = rest[i + 1]\n if (next != null && !next.startsWith('--')) {\n flags[key] = next\n i++\n } else {\n flags[key] = 'true'\n }\n } else if (tok === '-h') {\n flags.help = 'true'\n } else {\n positional.push(tok)\n }\n }\n return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n serve [--port 5005] [--host 127.0.0.1]\n Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n rpc <method>\n Read one JSON object from stdin (the params for <method>), write one\n JSON object to stdout. Methods: judge, listRubrics, version.\n rpc-batch <method>\n Like 'rpc' but JSONL in / JSONL out.\n openapi [--out openapi.json]\n Write the OpenAPI 3.1 spec.\n rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]\n Build a HuggingFace-ready dataset dir from tangle.rollout.v1 ledgers:\n validate, fail-closed split filter, scrub, export formats + card.\n analyst-benchmark --dataset <agentrx|codetracebench> --labels <path> --trace-dir <path> [--artifact-dir <path>] --out <dir> ...\n Compare an empty baseline with a real-model trace analyst on public labels.\n Run with --help for all required inputs and controls.\n supervisor-run report <runDir> [--format headline|markdown|json]\n Report one Runtime or loops supervisor run directory. The status comes\n from Runtime's own result.json / failure.json; every missing measurement\n stays named. Exits 1 when the directory cannot be read.\n search ship <search-ledger.jsonl> --run-kind optimization|eval [--content full|digests]\n Ship a search ledger and its blobs to the hosted store in TANGLE_INGEST_URL,\n starting from the store's head. Run with --help for the environment it reads.\n search show <search-ledger.jsonl>\n Verify a search ledger and print its summary: leading nodes, recently\n discarded nodes and recent proposals, the same text a proposer reads.\n version\n Print server + wire-protocol version JSON.\n\nJudge provider:\n Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n OPENAI_* and TANGLE_* equivalents are also accepted. This binary is the only\n place that reads a provider credential; the library never does. Without both\n a base URL and a key, /v1/judge refuses with llm_not_configured.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n const { command, positional, flags } = parseArgs(process.argv.slice(2))\n assertKnownFlags(command, flags)\n\n if (command === 'analyst-benchmark' && flags.help === 'true') {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n if (command === 'search') return await runSearchCommand(process.argv.slice(3))\n if (flags.help === 'true') {\n process.stdout.write(`${HELP}\\n`)\n return 0\n }\n\n switch (command) {\n case 'serve': {\n const port = parsePort(flags.port ?? '5005')\n const host = flags.host ?? '127.0.0.1'\n const llm = resolveCliLlmConfig()\n const { server } = await startServerAsync({\n port,\n host,\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n // Keep process alive on SIGINT/SIGTERM\n const shutdown = (sig: string) => {\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] received ${sig}, shutting down`)\n server.close(() => process.exit(0))\n // Force exit after 5s if close hangs\n setTimeout(() => process.exit(1), 5000).unref()\n }\n process.on('SIGINT', () => shutdown('SIGINT'))\n process.on('SIGTERM', () => shutdown('SIGTERM'))\n // Block forever\n await new Promise(() => {})\n return 0\n }\n case 'rpc': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcOnce(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'rpc-batch': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcBatch(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'openapi': {\n const out = flags.out ?? 'openapi.json'\n const spec = buildOpenApi(handleVersion().version)\n writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n return 0\n }\n case 'rollout-release': {\n // The subcommand owns its own flag grammar (multi-value --formats,\n // boolean --include-proposers) — pass raw argv through untouched.\n return await runRolloutReleaseCli(process.argv.slice(3))\n }\n case 'analyst-benchmark': {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n case 'supervisor-run': {\n // The subcommand owns its own flag grammar; pass raw argv through.\n return await runSupervisorRunCommand(process.argv.slice(3))\n }\n case 'version': {\n process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n return 0\n }\n case '--version': {\n process.stdout.write(`${handleVersion().version}\\n`)\n return 0\n }\n case 'help':\n case '--help':\n case '-h':\n case '':\n process.stdout.write(`${HELP}\\n`)\n return 0\n default:\n process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n return 1\n }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n serve: new Set(['help', 'host', 'port']),\n rpc: new Set(['help']),\n 'rpc-batch': new Set(['help']),\n openapi: new Set(['help', 'out']),\n version: new Set(['help']),\n help: new Set(),\n '--help': new Set(),\n '-h': new Set(),\n '--version': new Set(),\n '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n // These subcommands parse their own argv.\n if (\n command === 'rollout-release' ||\n command === 'analyst-benchmark' ||\n command === 'supervisor-run' ||\n command === 'search'\n ) {\n return\n }\n const allowed = FLAGS_BY_COMMAND[command]\n if (!allowed) return\n const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n if (unknown.length > 0) {\n throw new Error(`unknown flag for ${command || 'help'}: --${unknown[0]}`)\n }\n}\n\nfunction parsePort(raw: string): number {\n const port = Number(raw)\n if (!Number.isInteger(port) || port < 0 || port > 65_535) {\n throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n }\n return port\n}\n\nmain()\n .then((code) => {\n process.exitCode = code\n })\n .catch((err) => {\n // eslint-disable-next-line no-console\n console.error('[agent-eval] cli error:', err)\n process.exitCode = 1\n })\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,wBACd,MAAyB,QAAQ,KACH;CAC9B,MAAM,kBAAkB,SAAS,IAAI,uBAAuB;CAC5D,MAAM,iBAAiB,SAAS,IAAI,sBAAsB;CAC1D,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,KAAA,OAC7C,eAAe,mCAAmC,KAAA;CACrD,MAAM,SAAS,kBAAkB,gBAAgB;CACjD,MAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;CAE/F,IAAI,CAAC,WAAW,CAAC,QAAQ,OAAO,KAAA;CAChC,OAAO;EAAE;EAAS;EAAQ,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;CAAG;AACxD;AAEA,SAAgB,oBAAoB,MAAyB,QAAQ,KAAmB;CACtF,MAAM,QAAQ,wBAAwB,GAAG;CACzC,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,OAAO;EACL,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,QAAQ,MAAM;EAAO,GAAG,MAAM,KAAK;EACjF,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;CAC9C;AACF;;;;;;;;;AAUA,SAAS,cACP,OACA,cACY;CACZ,MAAM,SAAS,iBAAiB;EAC9B,WAAW;EACX,SAAS,MAAM;EACf,QAAQ,MAAM;EACd,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;CACzC,CAAC;CACD,OAAO;EACL,GAAG;EACH,OAAO,KAAK,aAAa;GACvB,IAAI,CAAC,IAAI,SAAS,CAAC,cACjB,MAAM,IAAI,MACR,4EACF;GAEF,OAAO,OAAO,KAAK,KAAK,QAAQ;EAClC;CACF;AACF;AAEA,SAAS,SAAS,OAA+C;CAC/D,MAAM,UAAU,OAAO,KAAK;CAC5B,OAAO,UAAU,UAAU,KAAA;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC9CA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0Cd,eAAsB,iBAAiB,MAAiC;CACtE,MAAM,CAAC,YAAY,GAAG,QAAQ;CAC9B,IAAI,eAAe,KAAA,KAAa,eAAe,YAAY,eAAe,MAAM;EAC9E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,eAAe,QAAQ,OAAO,MAAM,eAAe,IAAI;CAC3D,IAAI,eAAe,QAAQ;EACzB,QAAQ,OAAO,MAAM,8BAA8B,WAAW,IAAI,MAAM,GAAG;EAC3E,OAAO;CACT;CACA,IAAI,KAAK,MAAM,QAAQ,QAAQ,KAAA,KAAa,QAAQ,YAAY,QAAQ,IAAI,GAAG;EAC7E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,EAAE,MAAM,UAAU,cAAc,IAAI;CAC1C,MAAM,UAAU,oBAAoB,UAAU,MAAM,WAAW;CAC/D,IAAI,CAAC,QAAQ,SAAS,MAAM,IAAI,MAAM,4CAA4C,OAAO;CACzF,MAAM,UAAU,MAAM,WAAW;CACjC,IAAI,YAAY,UAAU,YAAY,WACpC,MAAM,IAAI,MAAM,sCAAsC,OAAO;CAE/D,MAAM,SAAS,oBAAoB;CACnC,IAAI,CAAC,QACH,MAAM,IAAI,MACR,qFACF;CAEF,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,IAAI;EACF,MAAM,UAAU,MAAM,iBAAiB;GACrC;GACA,QAAQ;IAAE;IAAM;GAAS;GACzB,SAAS,QAAQ;GACjB;EACF,CAAC;EACD,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,EAAE,GAAG;EAC5D,OAAO,QAAQ,KAAK,iBAAiB,QAAQ,aAAa,IAAI;CAChE,SAAS,OAAO;EACd,IAAI,EAAE,iBAAiB,0BAA0B,MAAM;EACvD,QAAQ,OAAO,MAAM,GAAG,MAAM,QAAQ,GAAG;EACzC,OAAO;CACT;AACF;AAEA,eAAe,eAAe,MAAiC;CAC7D,IAAI,KAAK,SAAS,QAAQ,KAAK,KAAK,SAAS,IAAI,GAAG;EAClD,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,KAAK,SAAS,QAAQ,GAAG,OAAO,MAAM,mBAAmB,IAAI;CACjE,MAAM,aAAa,KAAK,QAAQ,QAAQ,CAAC,IAAI,WAAW,IAAI,CAAC;CAC7D,MAAM,QAAQ,IAAI,IAAI,KAAK,QAAQ,QAAQ,IAAI,WAAW,IAAI,CAAC,CAAC;CAChE,MAAM,UAAU,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC;CACjE,IAAI,WAAW,WAAW,KAAK,QAAQ,SAAS,GAAG;EACjD,IAAI,QAAQ,SAAS,GAAG,QAAQ,OAAO,MAAM,gBAAgB,QAAQ,KAAK,IAAI,EAAE,GAAG;EACnF,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,OAAO,WAAW;CACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAE7C,MAAM,QAAQ,uBAAuB,MADlB,SAAS,MAAM,MAAM,GACG,UAAU,IAAI;CACzD,MAAM,QACJ,MAAM,UAAU,MAAM,OAAO,OAAO,UAAU,MAAM,SAAS,IAAI,cAAc;CACjF,MAAM,SAAS,WAAW,OAAO,MAAM,KAAK;CAC5C,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,IAAI,OAAO,WAAW,GAAG;GACvB,QAAQ,OAAO,MAAM,4BAA4B;GACjD,OAAO;EACT;EACA,MAAM,OAAO,OAAO,YAAY,OAAO,KAAK,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;EAC9E,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,GAAG;EACzD,OAAO;CACT;CACA,MAAM,WAAW,CAAC,oBAAoB,OAAO,EAAE,MAAM,CAAC,GAAG,GAAG,OAAO,KAAK,SAAS,KAAK,IAAI,CAAC;CAC3F,QAAQ,OAAO,MAAM,GAAG,SAAS,KAAK,MAAM,EAAE,GAAG;CACjD,OAAO;AACT;AAEA,eAAe,mBAAmB,MAAiC;CACjE,MAAM,QAAkB,CAAC;CACzB,IAAI;CACJ,IAAI,OAAO;CACX,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,UAAU,UAAU;EACxB,IAAI,UAAU,UAAU;GACtB,OAAO;GACP;EACF;EACA,IAAI,UAAU,eAAe;GAC3B,YAAY,KAAK,EAAE;GACnB,IAAI,cAAc,KAAA,GAAW,MAAM,IAAI,MAAM,8BAA8B,OAAO;GAClF;EACF;EACA,IAAI,MAAM,WAAW,IAAI,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,IAAI,OAAO;EAC7E,MAAM,KAAK,KAAK;CAClB;CACA,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,0CAA0C,OAAO;CACzF,MAAM,SAAS,CAAC;CAChB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;EAC7C,OAAO,KAAK,uBAAuB,MAAM,SAAS,MAAM,MAAM,GAAG,UAAU,IAAI,CAAC;CAClF;CACA,MAAM,OAAO,WAAW,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,CAAC;CAC5E,QAAQ,OAAO,MACb,OAAO,GAAG,KAAK,UAAU,EAAE,YAAY,KAAK,GAAG,MAAM,CAAC,EAAE,MAAM,GAAG,qBAAqB,IAAI,EAAE,GAC9F;CACA,OAAO;AACT;AAEA,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,SAAS,WACP,OACA,YACA,OACwD;CACxD,MAAM,SAAiE,CAAC;CACxE,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,MAAM,SAAS,KAAK,KAAK;EACzB,OAAO,KAAK;GAAE,MAAM;GAAQ;GAAQ,MAAM,eAAe,OAAO,IAAI;EAAE,CAAC;CACzE;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,wBAAwB,OAAO,IAAI;EAAE,CAAC;CAC3F;CACA,IAAI,MAAM,IAAI,SAAS,GAAG;EACxB,MAAM,SAAS,MAAM,KAAK;EAC1B,OAAO,KAAK;GAAE,MAAM;GAAS;GAAQ,MAAM,gBAAgB,OAAO,IAAI;EAAE,CAAC;CAC3E;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,KAAK;EAC/B,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,qBAAqB,OAAO,IAAI;EAAE,CAAC;CACrF;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,OAAO,EAAE,cAAc,iBAAiB,UAAU,EAAE,CAAC;EAC/E,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,eAAe,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,aAAa,GAAG;EAC5B,MAAM,OAAO,iBAAiB,UAAU;EACxC,MAAM,SAAS,UACb,OACA,kBAAkB,YAAY,KAAK,QAAQ,QAAQ,KAAK,IAAI,CAC9D;EACA,OAAO,KAAK;GAAE,MAAM;GAAa;GAAQ,MAAM,gBAAgB,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,oBAAoB,MAAM;EAAE,CAAC;CAClF;CACA,OAAO;AACT;;;;;;AAOA,SAAS,iBAAiB,YAAqE;CAC7F,MAAM,wBAAQ,IAAI,IAAqB;CACvC,QAAQ,QAAQ;EACd,IAAI,MAAM,IAAI,IAAI,MAAM,GAAG,OAAO,MAAM,IAAI,IAAI,MAAM;EACtD,MAAM,MAAM,IAAI,OAAO,MAAM,CAAgB;EAC7C,MAAM,aAAa,CAAC,KAAK,QAAQ,UAAU,GAAG,SAAS,GAAG,IAAI,MAAM,CAAC;EACrE,IAAI,IAAI,IAAI,WAAW,OAAO,GAAG,WAAW,QAAQ,cAAc,IAAI,GAAG,CAAC;EAC1E,IAAI;EACJ,KAAK,MAAM,aAAa,YAAY;GAClC,IAAI,CAAC,WAAW,SAAS,GAAG;GAC5B,MAAM,QAAQ,aAAa,SAAS;GAEpC,IADe,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAChD,MAAM,OAAO,MAAM,eAAe,IAAI,YAAY;GAC3D,IAAI;IACF,QAAQ,KAAK,MAAM,MAAM,SAAS,MAAM,CAAC;GAC3C,QAAQ,CAER;GACA;EACF;EACA,MAAM,IAAI,IAAI,QAAQ,KAAK;EAC3B,OAAO;CACT;AACF;AAEA,SAAS,eAAe,MAAwB;CAC9C,MAAM,QAAQ,CACZ,SAAS,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,IAAI,IAAI,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,KACvH;CACA,MAAM,QAAQ,MAAgB,QAAgB,WAA0B;EACtE,MAAM,SAAS,WAAW,KAAK,KAAK,SAAS,QAAQ;EACrD,MAAM,KAAK,KAAK,WAAW,GAAG,KAAK,SAAS,OAAO;EACnD,MAAM,OAAO,KAAK,SAAS,OAAO,KAAK,SAAS,KAAK;EACrD,MAAM,QAAQ,KAAK,UAAU,OAAO,mBAAmB;EACvD,MAAM,OACJ,KAAK,qBAAqB,IACtB,IAAI,KAAK,aAAa,QAAQ,CAAC,MAC/B,IAAI,KAAK,aAAa,QAAQ,CAAC,EAAE,qBAAqB,KAAK,SAAS,QAAQ,CAAC,EAAE,QAAQ,KAAK,iBAAiB,oBAAoB,KAAK,qBAAqB,IAAI,KAAK;EAC1K,MAAM,KACJ,GAAG,SAAS,SAAS,KAAK,KAAK,OAAO,IAAI,KAAK,UAAU,cAAc,OAAO,MAAM,IAAI,MAC1F;EACA,MAAM,cAAc,UAAU,WAAW,KAAK,KAAK,SAAS,QAAQ;EACpE,KAAK,SAAS,SAAS,OAAO,UAAU;GACtC,KAAK,OAAO,aAAa,UAAU,KAAK,SAAS,SAAS,CAAC;EAC7D,CAAC;CACH;CACA,KAAK,MAAM,SAAS,MAAM,UAAU;EAClC,KAAK,MAAM,IAAI,UAAU,KAAK,MAAM,SAAS,CAAC;CAChD,CAAC;CACD,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,wBAAwB,MAAiC;CAChE,MAAM,QAAQ,CAAC,mBAAmB,KAAK,MAAM,gDAAgD;CAC7F,IAAI,KAAK,KAAK,WAAW,GAAG,MAAM,KAAK,gBAAgB;CACvD,KAAK,MAAM,OAAO,KAAK,MAAM;EAC3B,MAAM,IAAI,IAAI;EACd,MAAM,WAAW,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAC1C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,GAAG;EACX,MAAM,YACJ,EAAE,WAAW,SACT,EAAE,MAAM,IACN,gCACA,kBAAkB,EAAE,EAAE,MAAM,kBAAkB,WAChD,EAAE,WAAW,iBACX,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,kBAAkB,mBAC9D,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,eAAe,MAAM,EAAE,EAAE;EAC/I,MAAM,QAAQ,OAAO,OAAO,IAAI,QAAQ,CAAC,CAAC,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC;EAC/E,MAAM,cAAc,IAAI,cAAc,IAAI,KAAK,IAAI,YAAY,iBAAiB;EAChF,MAAM,KACJ,KAAK,IAAI,SAAS,IAAI,IAAI,UAAU,YAAY,YAAY,KAAK,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,YAAY,mBAAmB,KAAK,UAAU,IAAI,IAAI,SAAS,UAC1K;CACF;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,gBAAgB,MAAyB;CAChD,MAAM,UAAU,KAAK,KAAK,QAAQ,QAAQ,IAAI,OAAO;CACrD,MAAM,UAAU,OAAO,QAAQ,KAAK,UAAU,CAAC,CAC5C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,IAAI;CACZ,MAAM,QAAQ,CACZ,UAAU,KAAK,MAAM,gBAAgB,KAAK,KAAK,KAAK,IAAI,EAAE,KAAK,KAAK,aAAa,MAAM,KAAK,KAAK,OAAO,4BAA4B,KAAK,cAAc,WAAW,UAAU,KAAK,QAAQ,KAAK,IAChM;CACA,KAAK,MAAM,OAAO,SAChB,MAAM,KACJ,KAAK,IAAI,OAAO,UAAU,IAAI,MAAO,QAAQ,CAAC,EAAE,MAAM,IAAI,MAAM,UAAU,IAAI,OAAO,iBAAiB,IAAI,eAAgB,QAAQ,CAAC,GACrI;CAEF,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,qBAAqB,MAA8B;CAC1D,MAAM,QAAQ,CACZ,gBAAgB,KAAK,MAAM,UAAU,KAAK,UAAU,KAAK,KAAK,QAAQ,OAAO,aAAa,KAAK,QAAQ,OAAO,YAAY,KAAK,aAAa,OAAO,oBAAoB,KAAK,aAAa,OAAO,iBAClM;CACA,KAAK,MAAM,OAAO,KAAK,gBAAgB;EACrC,MAAM,OACJ,IAAI,SAAS,OACT,iBAAiB,IAAI,aAAa,KAClC,GAAG,IAAI,KAAK,QAAQ,CAAC,EAAE,SAAS,IAAI,WAAW,IAAI,IAAI,kBAAkB,YAAY,IAAI,QAAQ,OAAO;EAC9G,MAAM,KAAK,aAAa,IAAI,UAAU,IAAI,IAAI,QAAQ,KAAK,IAAI,EAAE,qBAAqB,MAAM;CAC9F;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,cAAc,MAAiE;CACtF,MAAM,QAAgC,CAAC;CACvC,MAAM,aAAuB,CAAC;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,CAAC,MAAM,WAAW,IAAI,GAAG;GAC3B,WAAW,KAAK,KAAK;GACrB;EACF;EACA,MAAM,CAAC,MAAM,UAAU,MAAM,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;EAClD,IAAI,SAAS,cAAc,SAAS,WAAW,MAAM,IAAI,MAAM,kBAAkB,MAAM;EACvF,MAAM,QAAQ,UAAU,KAAK,EAAE;EAC/B,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,KAAK,KAAK,eAAe;EAClE,MAAM,QAAQ;CAChB;CACA,IAAI,WAAW,WAAW,GAAG,MAAM,IAAI,MAAM,KAAK;CAClD,OAAO;EAAE,MAAM,WAAW;EAAK;CAAM;AACvC;;AAGA,eAAe,kBAAkB,MAA+B;CAC9D,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;CACnC,IAAI;EAEF,MAAM,QAAQ,OAAO,MAAM,gCAAgC,CAAC;EAC5D,MAAM,EAAE,cAAc,MAAM,OAAO,KAAK,OAAO,GAAG,MAAM,YAAY,CAAC;EACrE,MAAM,UAAU,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,QAAQ,EAAI;EACzD,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,GAAG,KAAK,+BAA+B;EACxE,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,QAAQ,GAAG,OAAO,CAAC;EAC3D,IAAI,OAAO,MAAM,aAAa,UAAU,MAAM,IAAI,MAAM,GAAG,KAAK,0BAA0B;EAC1F,OAAO,MAAM;CACf,UAAU;EACR,MAAM,OAAO,MAAM;CACrB;AACF;;;;;;;;;;;;;;;;;;;AChXA,SAAS,UAAU,MAAsB;CACvC,MAAM,CAAC,SAAS,GAAG,QAAQ;CAC3B,MAAM,aAAuB,CAAC;CAC9B,MAAM,QAAgC,CAAC;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,IAAI,WAAW,IAAI,GAAG;GACxB,MAAM,MAAM,IAAI,MAAM,CAAC;GACvB,MAAM,WAAW,IAAI,QAAQ,GAAG;GAChC,IAAI,YAAY,GAAG;IACjB,MAAM,IAAI,MAAM,GAAG,QAAQ,KAAK,IAAI,MAAM,WAAW,CAAC;IACtD;GACF;GACA,MAAM,MAAM;GACZ,IAAI,QAAQ,QAAQ;IAClB,MAAM,OAAO;IACb;GACF;GACA,MAAM,OAAO,KAAK,IAAI;GACtB,IAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;IAC1C,MAAM,OAAO;IACb;GACF,OACE,MAAM,OAAO;EAEjB,OAAO,IAAI,QAAQ,MACjB,MAAM,OAAO;OAEb,WAAW,KAAK,GAAG;CAEvB;CACA,OAAO;EAAE,SAAS,WAAW;EAAQ;EAAY;CAAM;AACzD;AAEA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCb,eAAe,OAAwB;CACrC,MAAM,EAAE,SAAS,YAAY,UAAU,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;CACtE,iBAAiB,SAAS,KAAK;CAE/B,IAAI,YAAY,uBAAuB,MAAM,SAAS,QACpD,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;CAE/D,IAAI,YAAY,UAAU,OAAO,MAAM,iBAAiB,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7E,IAAI,MAAM,SAAS,QAAQ;EACzB,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;EAChC,OAAO;CACT;CAEA,QAAQ,SAAR;EACE,KAAK,SAAS;GACZ,MAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;GAC3C,MAAM,OAAO,MAAM,QAAQ;GAC3B,MAAM,MAAM,oBAAoB;GAChC,MAAM,EAAE,WAAW,MAAM,iBAAiB;IACxC;IACA;IACA,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;GAED,MAAM,YAAY,QAAgB;IAEhC,QAAQ,IAAI,yBAAyB,IAAI,gBAAgB;IACzD,OAAO,YAAY,QAAQ,KAAK,CAAC,CAAC;IAElC,iBAAiB,QAAQ,KAAK,CAAC,GAAG,GAAI,CAAC,CAAC,MAAM;GAChD;GACA,QAAQ,GAAG,gBAAgB,SAAS,QAAQ,CAAC;GAC7C,QAAQ,GAAG,iBAAiB,SAAS,SAAS,CAAC;GAE/C,MAAM,IAAI,cAAc,CAAC,CAAC;GAC1B,OAAO;EACT;EACA,KAAK,OAAO;GACV,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,WAAW,QAAQ;IAC9B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,aAAa;GAChB,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,YAAY,QAAQ;IAC/B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,WAAW;GACd,MAAM,MAAM,MAAM,OAAO;GACzB,MAAM,OAAO,aAAa,cAAc,CAAC,CAAC,OAAO;GACjD,cAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,KAAK,OAAO;GAEhE,QAAQ,IAAI,0CAA0C,KAAK;GAC3D,OAAO;EACT;EACA,KAAK,mBAGH,OAAO,MAAM,qBAAqB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAEzD,KAAK,qBACH,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE/D,KAAK,kBAEH,OAAO,MAAM,wBAAwB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE5D,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,EAAE,GAAG;GACpE,OAAO;EAET,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,cAAc,CAAC,CAAC,QAAQ,GAAG;GACnD,OAAO;EAET,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;GAChC,OAAO;EACT;GACE,QAAQ,OAAO,MAAM,oBAAoB,QAAQ,IAAI,KAAK,GAAG;GAC7D,OAAO;CACX;AACF;AAEA,MAAM,mBAAwD;CAC5D,uBAAO,IAAI,IAAI;EAAC;EAAQ;EAAQ;CAAM,CAAC;CACvC,qBAAK,IAAI,IAAI,CAAC,MAAM,CAAC;CACrB,6BAAa,IAAI,IAAI,CAAC,MAAM,CAAC;CAC7B,yBAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;CAChC,yBAAS,IAAI,IAAI,CAAC,MAAM,CAAC;CACzB,sBAAM,IAAI,IAAI;CACd,0BAAU,IAAI,IAAI;CAClB,sBAAM,IAAI,IAAI;CACd,6BAAa,IAAI,IAAI;CACrB,oBAAI,IAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;CAE9E,IACE,YAAY,qBACZ,YAAY,uBACZ,YAAY,oBACZ,YAAY,UAEZ;CAEF,MAAM,UAAU,iBAAiB;CACjC,IAAI,CAAC,SAAS;CACd,MAAM,UAAU,OAAO,KAAK,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;CACtE,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MAAM,oBAAoB,WAAW,OAAO,MAAM,QAAQ,IAAI;AAE5E;AAEA,SAAS,UAAU,KAAqB;CACtC,MAAM,OAAO,OAAO,GAAG;CACvB,IAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAChD,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,GAAG;CAE9F,OAAO;AACT;AAEA,KAAK,CAAC,CACH,MAAM,SAAS;CACd,QAAQ,WAAW;AACrB,CAAC,CAAC,CACD,OAAO,QAAQ;CAEd,QAAQ,MAAM,2BAA2B,GAAG;CAC5C,QAAQ,WAAW;AACrB,CAAC"}
|
|
1
|
+
{"version":3,"file":"cli.js","names":[],"sources":["../src/cli-config.ts","../src/search-command.ts","../src/cli.ts"],"sourcesContent":["/**\n * Provider configuration for the `agent-eval` binary.\n *\n * The binary reads endpoint and credential values from its environment.\n * TypeScript library callers bind a transport explicitly. The maintained\n * HTTP transport accepts their credential and sends provider requests.\n */\n\nimport { type ChatClient, createChatClient } from './analyst/chat-client'\n\nexport interface CliLlmConfig {\n /** Judge transport for the wire handlers. Absent when no provider is configured. */\n chat?: ChatClient\n model?: string\n}\n\n/** The endpoint the binary resolved from its environment. */\nexport interface CliProviderRoute {\n baseUrl: string\n apiKey: string\n model?: string\n}\n\n/**\n * Environment precedence for the binary's provider route, with no client built.\n *\n * Both halves are required: a base URL without a key, or a key without an\n * endpoint, is a half-configured server. `/v1/judge` then refuses with\n * `llm_not_configured` instead of calling an unintended endpoint.\n */\nexport function resolveCliProviderRoute(\n env: NodeJS.ProcessEnv = process.env,\n): CliProviderRoute | undefined {\n const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n const baseUrl =\n explicitBaseUrl ??\n nonEmpty(env.OPENAI_BASE_URL) ??\n nonEmpty(env.TANGLE_ROUTER_URL) ??\n (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n const model =\n nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n if (!baseUrl || !apiKey) return undefined\n return { baseUrl, apiKey, ...(model ? { model } : {}) }\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n const route = resolveCliProviderRoute(env)\n if (!route) return {}\n return {\n chat: cliChatClient({ baseUrl: route.baseUrl, apiKey: route.apiKey }, route.model),\n ...(route.model ? { model: route.model } : {}),\n }\n}\n\n/**\n * The binary's transport IS the published one — `createChatClient({ transport:\n * 'openai-compatible' })` — so the server and a library consumer cannot drift\n * onto two different clients for the same endpoint.\n *\n * The one thing kept local is the error text. A generic \"no model on the\n * request\" tells a server operator nothing; this names the variable to set.\n */\nfunction cliChatClient(\n route: { baseUrl: string; apiKey: string },\n defaultModel: string | undefined,\n): ChatClient {\n const client = createChatClient({\n transport: 'openai-compatible',\n baseUrl: route.baseUrl,\n apiKey: route.apiKey,\n ...(defaultModel ? { defaultModel } : {}),\n })\n return {\n ...client,\n chat: (req, callOpts) => {\n if (!req.model && !defaultModel) {\n throw new Error(\n 'agent-eval: no model on the request and no AGENT_EVAL_LLM_MODEL configured',\n )\n }\n return client.chat(req, callOpts)\n },\n }\n}\n\nfunction nonEmpty(value: string | undefined): string | undefined {\n const trimmed = value?.trim()\n return trimmed ? trimmed : undefined\n}\n","/**\n * `agent-eval search <subcommand>`: work with a search ledger from a terminal.\n *\n * agent-eval search ship <search-ledger.jsonl> --run-kind optimization|eval [--content full|digests]\n * agent-eval search show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n * [--edit-credit] [--landscape] [--skill-manifold] [--json]\n * agent-eval search show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n *\n * `ship` sends the ledger to the hosted store named by `TANGLE_INGEST_URL`,\n * `TANGLE_INGEST_API_KEY` and `TANGLE_TENANT_ID`, starting from the store's\n * head, so it finishes or resumes a ship that a loop could not complete.\n *\n * `show` verifies the ledger and prints `renderSearchSummary`: the same\n * compact text a proposer reads, on the search's own ranking split (the\n * selection split when the search declares one, else train). There is no\n * local HTML renderer; the hosted store's page is the visual view (§5 of the\n * search-tree design). A lens flag adds that lens's text form after the\n * summary, so an agent reads what a person sees; `--json` prints the lens\n * results as JSON instead, the shape a view renders.\n *\n * `show --meta` reads every ledger it is given and prints the `metaSearch`\n * lens over them instead of one search's summary: each search as one node\n * scored by its held-out lift per known dollar, the configurations best first,\n * and the searches arranged by derivation and containment.\n */\n\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { open, readFile } from 'node:fs/promises'\nimport { dirname, join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { replaySearchLedgerText } from './campaign/search-ledger'\nimport type { SearchArtifactRef } from './campaign/search-ledger-types'\nimport type { SearchStateView } from './campaign/search-state'\nimport { renderSearchSummary } from './campaign/search-summary'\nimport { hostedTenantFromEnv } from './hosted/client'\nimport { SEARCH_LEDGER_BATCH_MAX_BYTES, SearchRunKindSchema } from './hosted/search-ledger-wire'\nimport { SearchShipConflictError, shipSearchLedger } from './hosted/search-shipper'\nimport { editCredit, editCreditText } from './search/lenses/edit-credit'\nimport { type FrontData, front } from './search/lenses/front'\nimport { formatLandscape, landscape, surfaceTextEdits } from './search/lenses/landscape'\nimport { metaSearch, renderMetaSearchText } from './search/lenses/meta-search'\nimport { type OperatorYieldData, operatorYield } from './search/lenses/operator-yield'\nimport { INSUFFICIENT_FROM } from './search/lenses/shared'\nimport { formatSkillManifold, skillManifold } from './search/lenses/skill-manifold'\nimport { type TaskMatrixData, taskMatrix } from './search/lenses/task-matrix'\nimport { type TreeData, type TreeNode, tree } from './search/lenses/tree'\n\nconst USAGE = `usage: agent-eval search <subcommand> ...\n\n ship <search-ledger.jsonl> --run-kind optimization|eval [--content full|digests]\n Ships the ledger's entries, and the blobs they name, to the hosted store in\n TANGLE_INGEST_URL (or TANGLE_ORCHESTRATOR_URL) as tenant TANGLE_TENANT_ID with\n TANGLE_INGEST_API_KEY (or TANGLE_API_KEY). It starts from the store's head, so\n running it again sends only what the store lacks. Prints the result as JSON.\n Exits 1 when the store holds a different chain for the search.\n\n show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n [--edit-credit] [--landscape] [--skill-manifold] [--json]\n Verifies the ledger and prints its search summary: the leading nodes\n against the root, the most recently discarded nodes and why, and a log\n of recent proposals. The same text a proposer reads as context, on the\n search's own ranking split. Each lens flag adds that lens's text form\n (search-tree-design §12) below the summary — the same JSON\n Intelligence, discovery lab, VerticalBench and agent-runtime read.\n --tree the tidy tree of nodes and edges.\n --operator-yield outcome counts and yield per known dollar, by edge operator.\n --front the Pareto front over score and known cost per cell.\n --task-matrix nodes × units, clustered, with specialist gain per cluster.\n --edit-credit every edit as a gene followed down the lineage, its\n credit, interacting pairs, and skill candidates. It\n reads node artifacts from the blobs the ledger names,\n verified by digest.\n --landscape nodes placed by line edits between their profile\n surfaces (read from the blobs, verified by digest), the\n kriged score surface, its basins, and the plateau score\n a policy drafts on.\n --skill-manifold the node × unit score matrix factored into skill axes,\n its cross-validated intrinsic dimension, and the unit\n that best separates the leaders.\n --json prints the requested lenses as JSON instead of text.\n\n show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n Verifies every ledger and prints the meta-search lens over them: each\n search as one node, scored by its claim's held-out lift per known\n dollar (unscored searches say why), configurations best first with\n their interval, method and n, and the searches arranged by derivation\n and containment. --objective names the objective the best\n configuration is chosen within when the searches span several.`\n\nexport async function runSearchCommand(argv: string[]): Promise<number> {\n const [subcommand, ...rest] = argv\n if (subcommand === undefined || subcommand === '--help' || subcommand === '-h') {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (subcommand === 'show') return await runShowCommand(rest)\n if (subcommand !== 'ship') {\n process.stderr.write(`unknown search subcommand: ${subcommand}\\n${USAGE}\\n`)\n return 1\n }\n if (rest.some((arg) => arg === undefined || arg === '--help' || arg === '-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n const { path, flags } = parseShipArgs(rest)\n const runKind = SearchRunKindSchema.safeParse(flags['run-kind'])\n if (!runKind.success) throw new Error(`--run-kind must be optimization or eval\\n${USAGE}`)\n const content = flags.content ?? 'full'\n if (content !== 'full' && content !== 'digests') {\n throw new Error(`--content must be full or digests\\n${USAGE}`)\n }\n const tenant = hostedTenantFromEnv()\n if (!tenant) {\n throw new Error(\n 'set TANGLE_INGEST_URL, TANGLE_INGEST_API_KEY and TANGLE_TENANT_ID to name the store',\n )\n }\n const searchId = await firstLineSearchId(path)\n try {\n const shipped = await shipSearchLedger({\n tenant,\n ledger: { path, searchId },\n runKind: runKind.data,\n content,\n })\n process.stdout.write(`${JSON.stringify(shipped, null, 2)}\\n`)\n return shipped.head.nextSequence === shipped.localLines ? 0 : 1\n } catch (error) {\n if (!(error instanceof SearchShipConflictError)) throw error\n process.stderr.write(`${error.message}\\n`)\n return 1\n }\n}\n\nasync function runShowCommand(argv: string[]): Promise<number> {\n if (argv.includes('--help') || argv.includes('-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (argv.includes('--meta')) return await runShowMetaCommand(argv)\n const positional = argv.filter((arg) => !arg.startsWith('--'))\n const flags = new Set(argv.filter((arg) => arg.startsWith('--')))\n const unknown = [...flags].filter((flag) => !SHOW_FLAGS.has(flag))\n if (positional.length !== 1 || unknown.length > 0) {\n if (unknown.length > 0) process.stderr.write(`unknown flag ${unknown.join(', ')}\\n`)\n process.stdout.write(`${USAGE}\\n`)\n return 1\n }\n const path = positional[0]!\n const searchId = await firstLineSearchId(path)\n const text = await readFile(path, 'utf8')\n const state = replaySearchLedgerText(text, searchId, path)\n const split =\n state.header && state.header.splits.selection.tasks.length > 0 ? 'selection' : 'train'\n const lenses = showLenses(state, path, flags)\n if (flags.has('--json')) {\n if (lenses.length === 0) {\n process.stderr.write('--json needs a lens flag\\n')\n return 1\n }\n const json = Object.fromEntries(lenses.map((lens) => [lens.name, lens.result]))\n process.stdout.write(`${JSON.stringify(json, null, 2)}\\n`)\n return 0\n }\n const sections = [renderSearchSummary(state, { split }), ...lenses.map((lens) => lens.text)]\n process.stdout.write(`${sections.join('\\n\\n')}\\n`)\n return 0\n}\n\nasync function runShowMetaCommand(argv: string[]): Promise<number> {\n const paths: string[] = []\n let objective: string | undefined\n let json = false\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (token === '--meta') continue\n if (token === '--json') {\n json = true\n continue\n }\n if (token === '--objective') {\n objective = argv[++index]\n if (objective === undefined) throw new Error(`--objective needs a value\\n${USAGE}`)\n continue\n }\n if (token.startsWith('--')) throw new Error(`unknown flag ${token}\\n${USAGE}`)\n paths.push(token)\n }\n if (paths.length === 0) throw new Error(`show --meta needs at least one ledger\\n${USAGE}`)\n const states = []\n for (const path of paths) {\n const searchId = await firstLineSearchId(path)\n states.push(replaySearchLedgerText(await readFile(path, 'utf8'), searchId, path))\n }\n const lens = metaSearch(states, objective === undefined ? {} : { objective })\n process.stdout.write(\n json ? `${JSON.stringify({ metaSearch: lens }, null, 2)}\\n` : `${renderMetaSearchText(lens)}\\n`,\n )\n return 0\n}\n\nconst SHOW_FLAGS = new Set([\n '--edit-credit',\n '--json',\n '--tree',\n '--operator-yield',\n '--front',\n '--task-matrix',\n '--landscape',\n '--skill-manifold',\n])\n\nfunction showLenses(\n state: SearchStateView,\n ledgerPath: string,\n flags: ReadonlySet<string>,\n): Array<{ name: string; result: unknown; text: string }> {\n const lenses: Array<{ name: string; result: unknown; text: string }> = []\n if (flags.has('--tree')) {\n const result = tree(state)\n lenses.push({ name: 'tree', result, text: renderTreeText(result.data) })\n }\n if (flags.has('--operator-yield')) {\n const result = operatorYield(state)\n lenses.push({ name: 'operatorYield', result, text: renderOperatorYieldText(result.data) })\n }\n if (flags.has('--front')) {\n const result = front(state)\n lenses.push({ name: 'front', result, text: renderFrontText(result.data) })\n }\n if (flags.has('--task-matrix')) {\n const result = taskMatrix(state)\n lenses.push({ name: 'taskMatrix', result, text: renderTaskMatrixText(result.data) })\n }\n if (flags.has('--edit-credit')) {\n const result = editCredit(state, { readArtifact: ledgerBlobReader(ledgerPath) })\n lenses.push({ name: 'editCredit', result, text: editCreditText(result) })\n }\n if (flags.has('--landscape')) {\n const read = ledgerBlobReader(ledgerPath)\n const result = landscape(\n state,\n surfaceTextEdits((surface) => read(surface.artifact) ?? null),\n )\n lenses.push({ name: 'landscape', result, text: formatLandscape(result) })\n }\n if (flags.has('--skill-manifold')) {\n const result = skillManifold(state)\n lenses.push({ name: 'skillManifold', result, text: formatSkillManifold(result) })\n }\n return lenses\n}\n\n/**\n * Reads a blob a ledger names: at its `file:` URI, else under `blobs/` beside\n * the ledger, where `SearchRecorder` writes them. Bytes that do not hash to the\n * reference's digest and length read as unavailable, never as content.\n */\nfunction ledgerBlobReader(ledgerPath: string): (ref: SearchArtifactRef) => unknown | undefined {\n const cache = new Map<string, unknown>()\n return (ref) => {\n if (cache.has(ref.sha256)) return cache.get(ref.sha256)\n const hex = ref.sha256.slice('sha256:'.length)\n const candidates = [join(dirname(ledgerPath), 'blobs', `${hex}.json`)]\n if (ref.uri.startsWith('file:')) candidates.unshift(fileURLToPath(ref.uri))\n let value: unknown\n for (const candidate of candidates) {\n if (!existsSync(candidate)) continue\n const bytes = readFileSync(candidate)\n const digest = createHash('sha256').update(bytes).digest('hex')\n if (digest !== hex || bytes.byteLength !== ref.byteLength) continue\n try {\n value = JSON.parse(bytes.toString('utf8'))\n } catch {\n // Verified bytes that are not JSON are not content a lens can read.\n }\n break\n }\n cache.set(ref.sha256, value)\n return value\n }\n}\n\nfunction renderTreeText(data: TreeData): string {\n const lines = [\n `tree: ${data.nodeCount} node${data.nodeCount === 1 ? '' : 's'}, ${data.edgeCount} edge${data.edgeCount === 1 ? '' : 's'}`,\n ]\n const walk = (node: TreeNode, prefix: string, isLast: boolean): void => {\n const branch = prefix === '' ? '' : isLast ? '└─ ' : '├─ '\n const op = node.operator ? `${node.operator} → ` : ''\n const rung = node.rung === null ? '' : ` rung ${node.rung}`\n const depth = node.depth === null ? ' depth unknown' : ''\n const cost =\n node.unknownCostCells === 0\n ? `$${node.knownCostUsd.toFixed(2)}`\n : `$${node.knownCostUsd.toFixed(2)} known + at least $${node.floorUsd.toFixed(2)} over ${node.unknownCostCells} unknown-cost cell${node.unknownCostCells === 1 ? '' : 's'}`\n lines.push(\n `${prefix}${branch}${op}${node.nodeId} (${node.status ?? 'undecided'}${rung}${depth}) ${cost}`,\n )\n const childPrefix = prefix + (prefix === '' ? '' : isLast ? ' ' : '│ ')\n node.children.forEach((child, index) => {\n walk(child, childPrefix, index === node.children.length - 1)\n })\n }\n data.roots.forEach((root, index) => {\n walk(root, '', index === data.roots.length - 1)\n })\n return lines.join('\\n')\n}\n\nfunction renderOperatorYieldText(data: OperatorYieldData): string {\n const lines = [`operator yield (${data.split} split, gain per known $ of proposal + screen):`]\n if (data.rows.length === 0) lines.push(' no edges yet')\n for (const row of data.rows) {\n const y = row.yield\n const outcomes = Object.entries(row.outcomes)\n .filter(([, count]) => count > 0)\n .map(([status, count]) => `${status}=${count}`)\n .join(' ')\n const yieldText =\n y.method === 'none'\n ? y.n === 0\n ? 'yield: no measured children'\n : `yield unknown (${y.n} of ${INSUFFICIENT_FROM}, none)`\n : y.method === 'insufficient'\n ? `yield≈${y.mean!.toFixed(4)}/$ (${y.n} of ${INSUFFICIENT_FROM}, insufficient)`\n : `yield≈${y.mean!.toFixed(4)}/$ [${y.interval![0].toFixed(4)}, ${y.interval![1].toFixed(4)}] (${y.method}, ${y.intervalMethod}, n=${y.n})`\n const nodes = Object.values(row.outcomes).reduce((sum, count) => sum + count, 0)\n const reproposals = row.reproposals > 0 ? `, ${row.reproposals} re-proposals` : ''\n lines.push(\n ` ${row.operator}: ${row.proposals} proposals${reproposals} → ${nodes} node${nodes === 1 ? '' : 's'} (${outcomes || 'no decisions yet'}); ${yieldText}; ${row.excluded} excluded`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderFrontText(data: FrontData): string {\n const onFront = data.rows.filter((row) => row.onFront)\n const reasons = Object.entries(data.exclusions)\n .filter(([, count]) => count > 0)\n .map(([reason, count]) => `${reason} ${count}`)\n .join(', ')\n const lines = [\n `front (${data.split} split, axes: ${data.axes.join(', ')}): ${data.frontierSize} of ${data.rows.length} node(s) on the frontier, ${data.excludedCount} excluded${reasons ? ` (${reasons})` : ''}`,\n ]\n for (const row of onFront) {\n lines.push(\n ` ${row.nodeId}: score=${row.score!.toFixed(4)} on ${row.units} units (${row.method}) costPerCell=$${row.costPerCellUsd!.toFixed(4)}`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderTaskMatrixText(data: TaskMatrixData): string {\n const lines = [\n `task matrix (${data.split} split, ${data.direction}): ${data.nodeIds.length} node(s) × ${data.unitIds.length} unit(s), ${data.nodeClusters.length} node cluster(s), ${data.unitClusters.length} unit cluster(s)`,\n ]\n for (const row of data.specialistGain) {\n const gain =\n row.gain === null\n ? `insufficient (${row.insufficient})`\n : `${row.gain.toFixed(4)} (best ${row.bestNodeId}, ${row.nodesContributing} nodes on ${row.unitIds.length} units)`\n lines.push(` cluster ${row.clusterId} [${row.unitIds.join(', ')}]: specialist gain ${gain}`)\n }\n return lines.join('\\n')\n}\n\nfunction parseShipArgs(argv: string[]): { path: string; flags: Record<string, string> } {\n const flags: Record<string, string> = {}\n const positional: string[] = []\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (!token.startsWith('--')) {\n positional.push(token)\n continue\n }\n const [name, inline] = token.slice(2).split('=', 2) as [string, string | undefined]\n if (name !== 'run-kind' && name !== 'content') throw new Error(`unknown flag --${name}`)\n const value = inline ?? argv[++index]\n if (value === undefined) throw new Error(`--${name} needs a value`)\n flags[name] = value\n }\n if (positional.length !== 1) throw new Error(USAGE)\n return { path: positional[0]!, flags }\n}\n\n/** The ledger names its search on every line; read the first. */\nasync function firstLineSearchId(path: string): Promise<string> {\n const handle = await open(path, 'r')\n try {\n // A line longer than one request's limit cannot ship anyway.\n const bytes = Buffer.alloc(SEARCH_LEDGER_BATCH_MAX_BYTES + 1)\n const { bytesRead } = await handle.read(bytes, 0, bytes.byteLength, 0)\n const newline = bytes.subarray(0, bytesRead).indexOf(0x0a)\n if (newline < 0) throw new Error(`${path} holds no complete ledger line`)\n const first = JSON.parse(bytes.toString('utf8', 0, newline)) as { searchId?: unknown }\n if (typeof first.searchId !== 'string') throw new Error(`${path} line 1 names no searchId`)\n return first.searchId\n } finally {\n await handle.close()\n }\n}\n","#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n * agent-eval serve [--port 5005] [--host 127.0.0.1]\n * agent-eval rpc <method> # one request from stdin → one response on stdout\n * agent-eval rpc-batch <method> # JSONL stdin → JSONL stdout\n * agent-eval analyst-benchmark ... # benchmark a real-model trace analyst\n * agent-eval supervisor-run report <runDir> [--format headline|markdown|json]\n * agent-eval search ship <search-ledger.jsonl> --run-kind optimization|eval\n * agent-eval search show <search-ledger.jsonl>\n * agent-eval openapi [--out path] # write OpenAPI spec\n * agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { runAnalystBenchmarkCommand } from './analyst/benchmark-command'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { runRolloutReleaseCli } from './rollout/release/hf-dataset'\nimport { runSearchCommand } from './search-command'\nimport { runSupervisorRunCommand } from './supervisor-run/report-command'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n command: string\n positional: string[]\n flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n const [command, ...rest] = argv\n const positional: string[] = []\n const flags: Record<string, string> = {}\n for (let i = 0; i < rest.length; i++) {\n const tok = rest[i]!\n if (tok.startsWith('--')) {\n const raw = tok.slice(2)\n const equalsAt = raw.indexOf('=')\n if (equalsAt >= 0) {\n flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n continue\n }\n const key = raw\n if (key === 'help') {\n flags[key] = 'true'\n continue\n }\n const next = rest[i + 1]\n if (next != null && !next.startsWith('--')) {\n flags[key] = next\n i++\n } else {\n flags[key] = 'true'\n }\n } else if (tok === '-h') {\n flags.help = 'true'\n } else {\n positional.push(tok)\n }\n }\n return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n serve [--port 5005] [--host 127.0.0.1]\n Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n rpc <method>\n Read one JSON object from stdin (the params for <method>), write one\n JSON object to stdout. Methods: judge, listRubrics, version.\n rpc-batch <method>\n Like 'rpc' but JSONL in / JSONL out.\n openapi [--out openapi.json]\n Write the OpenAPI 3.1 spec.\n rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]\n Build a HuggingFace-ready dataset dir from tangle.rollout.v1 ledgers:\n validate, fail-closed split filter, scrub, export formats + card.\n analyst-benchmark --dataset <agentrx|codetracebench> --labels <path> --trace-dir <path> [--artifact-dir <path>] --out <dir> ...\n Compare an empty baseline with a real-model trace analyst on public labels.\n Run with --help for all required inputs and controls.\n supervisor-run report <runDir> [--format headline|markdown|json]\n Report one Runtime or loops supervisor run directory. The status comes\n from Runtime's own result.json / failure.json; every missing measurement\n stays named. Exits 1 when the directory cannot be read.\n search ship <search-ledger.jsonl> --run-kind optimization|eval [--content full|digests]\n Ship a search ledger and its blobs to the hosted store in TANGLE_INGEST_URL,\n starting from the store's head. Run with --help for the environment it reads.\n search show <search-ledger.jsonl>\n Verify a search ledger and print its summary: leading nodes, recently\n discarded nodes and recent proposals, the same text a proposer reads.\n version\n Print server + wire-protocol version JSON.\n\nJudge provider:\n Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n OPENAI_* and TANGLE_* equivalents are also accepted. This binary is the only\n place that reads a provider credential; the library never does. Without both\n a base URL and a key, /v1/judge refuses with llm_not_configured.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n const { command, positional, flags } = parseArgs(process.argv.slice(2))\n assertKnownFlags(command, flags)\n\n if (command === 'analyst-benchmark' && flags.help === 'true') {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n if (command === 'search') return await runSearchCommand(process.argv.slice(3))\n if (flags.help === 'true') {\n process.stdout.write(`${HELP}\\n`)\n return 0\n }\n\n switch (command) {\n case 'serve': {\n const port = parsePort(flags.port ?? '5005')\n const host = flags.host ?? '127.0.0.1'\n const llm = resolveCliLlmConfig()\n const { server } = await startServerAsync({\n port,\n host,\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n // Keep process alive on SIGINT/SIGTERM\n const shutdown = (sig: string) => {\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] received ${sig}, shutting down`)\n server.close(() => process.exit(0))\n // Force exit after 5s if close hangs\n setTimeout(() => process.exit(1), 5000).unref()\n }\n process.on('SIGINT', () => shutdown('SIGINT'))\n process.on('SIGTERM', () => shutdown('SIGTERM'))\n // Block forever\n await new Promise(() => {})\n return 0\n }\n case 'rpc': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcOnce(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'rpc-batch': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcBatch(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'openapi': {\n const out = flags.out ?? 'openapi.json'\n const spec = buildOpenApi(handleVersion().version)\n writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n return 0\n }\n case 'rollout-release': {\n // The subcommand owns its own flag grammar (multi-value --formats,\n // boolean --include-proposers) — pass raw argv through untouched.\n return await runRolloutReleaseCli(process.argv.slice(3))\n }\n case 'analyst-benchmark': {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n case 'supervisor-run': {\n // The subcommand owns its own flag grammar; pass raw argv through.\n return await runSupervisorRunCommand(process.argv.slice(3))\n }\n case 'version': {\n process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n return 0\n }\n case '--version': {\n process.stdout.write(`${handleVersion().version}\\n`)\n return 0\n }\n case 'help':\n case '--help':\n case '-h':\n case '':\n process.stdout.write(`${HELP}\\n`)\n return 0\n default:\n process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n return 1\n }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n serve: new Set(['help', 'host', 'port']),\n rpc: new Set(['help']),\n 'rpc-batch': new Set(['help']),\n openapi: new Set(['help', 'out']),\n version: new Set(['help']),\n help: new Set(),\n '--help': new Set(),\n '-h': new Set(),\n '--version': new Set(),\n '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n // These subcommands parse their own argv.\n if (\n command === 'rollout-release' ||\n command === 'analyst-benchmark' ||\n command === 'supervisor-run' ||\n command === 'search'\n ) {\n return\n }\n const allowed = FLAGS_BY_COMMAND[command]\n if (!allowed) return\n const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n if (unknown.length > 0) {\n throw new Error(`unknown flag for ${command || 'help'}: --${unknown[0]}`)\n }\n}\n\nfunction parsePort(raw: string): number {\n const port = Number(raw)\n if (!Number.isInteger(port) || port < 0 || port > 65_535) {\n throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n }\n return port\n}\n\nmain()\n .then((code) => {\n process.exitCode = code\n })\n .catch((err) => {\n // eslint-disable-next-line no-console\n console.error('[agent-eval] cli error:', err)\n process.exitCode = 1\n })\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,wBACd,MAAyB,QAAQ,KACH;CAC9B,MAAM,kBAAkB,SAAS,IAAI,uBAAuB;CAC5D,MAAM,iBAAiB,SAAS,IAAI,sBAAsB;CAC1D,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,KAAA,OAC7C,eAAe,mCAAmC,KAAA;CACrD,MAAM,SAAS,kBAAkB,gBAAgB;CACjD,MAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;CAE/F,IAAI,CAAC,WAAW,CAAC,QAAQ,OAAO,KAAA;CAChC,OAAO;EAAE;EAAS;EAAQ,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;CAAG;AACxD;AAEA,SAAgB,oBAAoB,MAAyB,QAAQ,KAAmB;CACtF,MAAM,QAAQ,wBAAwB,GAAG;CACzC,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,OAAO;EACL,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,QAAQ,MAAM;EAAO,GAAG,MAAM,KAAK;EACjF,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;CAC9C;AACF;;;;;;;;;AAUA,SAAS,cACP,OACA,cACY;CACZ,MAAM,SAAS,iBAAiB;EAC9B,WAAW;EACX,SAAS,MAAM;EACf,QAAQ,MAAM;EACd,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;CACzC,CAAC;CACD,OAAO;EACL,GAAG;EACH,OAAO,KAAK,aAAa;GACvB,IAAI,CAAC,IAAI,SAAS,CAAC,cACjB,MAAM,IAAI,MACR,4EACF;GAEF,OAAO,OAAO,KAAK,KAAK,QAAQ;EAClC;CACF;AACF;AAEA,SAAS,SAAS,OAA+C;CAC/D,MAAM,UAAU,OAAO,KAAK;CAC5B,OAAO,UAAU,UAAU,KAAA;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC9CA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0Cd,eAAsB,iBAAiB,MAAiC;CACtE,MAAM,CAAC,YAAY,GAAG,QAAQ;CAC9B,IAAI,eAAe,KAAA,KAAa,eAAe,YAAY,eAAe,MAAM;EAC9E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,eAAe,QAAQ,OAAO,MAAM,eAAe,IAAI;CAC3D,IAAI,eAAe,QAAQ;EACzB,QAAQ,OAAO,MAAM,8BAA8B,WAAW,IAAI,MAAM,GAAG;EAC3E,OAAO;CACT;CACA,IAAI,KAAK,MAAM,QAAQ,QAAQ,KAAA,KAAa,QAAQ,YAAY,QAAQ,IAAI,GAAG;EAC7E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,EAAE,MAAM,UAAU,cAAc,IAAI;CAC1C,MAAM,UAAU,oBAAoB,UAAU,MAAM,WAAW;CAC/D,IAAI,CAAC,QAAQ,SAAS,MAAM,IAAI,MAAM,4CAA4C,OAAO;CACzF,MAAM,UAAU,MAAM,WAAW;CACjC,IAAI,YAAY,UAAU,YAAY,WACpC,MAAM,IAAI,MAAM,sCAAsC,OAAO;CAE/D,MAAM,SAAS,oBAAoB;CACnC,IAAI,CAAC,QACH,MAAM,IAAI,MACR,qFACF;CAEF,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,IAAI;EACF,MAAM,UAAU,MAAM,iBAAiB;GACrC;GACA,QAAQ;IAAE;IAAM;GAAS;GACzB,SAAS,QAAQ;GACjB;EACF,CAAC;EACD,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,EAAE,GAAG;EAC5D,OAAO,QAAQ,KAAK,iBAAiB,QAAQ,aAAa,IAAI;CAChE,SAAS,OAAO;EACd,IAAI,EAAE,iBAAiB,0BAA0B,MAAM;EACvD,QAAQ,OAAO,MAAM,GAAG,MAAM,QAAQ,GAAG;EACzC,OAAO;CACT;AACF;AAEA,eAAe,eAAe,MAAiC;CAC7D,IAAI,KAAK,SAAS,QAAQ,KAAK,KAAK,SAAS,IAAI,GAAG;EAClD,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,KAAK,SAAS,QAAQ,GAAG,OAAO,MAAM,mBAAmB,IAAI;CACjE,MAAM,aAAa,KAAK,QAAQ,QAAQ,CAAC,IAAI,WAAW,IAAI,CAAC;CAC7D,MAAM,QAAQ,IAAI,IAAI,KAAK,QAAQ,QAAQ,IAAI,WAAW,IAAI,CAAC,CAAC;CAChE,MAAM,UAAU,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC;CACjE,IAAI,WAAW,WAAW,KAAK,QAAQ,SAAS,GAAG;EACjD,IAAI,QAAQ,SAAS,GAAG,QAAQ,OAAO,MAAM,gBAAgB,QAAQ,KAAK,IAAI,EAAE,GAAG;EACnF,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,OAAO,WAAW;CACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,MAAM,OAAO,MAAM,SAAS,MAAM,MAAM;CACxC,MAAM,QAAQ,uBAAuB,MAAM,UAAU,IAAI;CACzD,MAAM,QACJ,MAAM,UAAU,MAAM,OAAO,OAAO,UAAU,MAAM,SAAS,IAAI,cAAc;CACjF,MAAM,SAAS,WAAW,OAAO,MAAM,KAAK;CAC5C,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,IAAI,OAAO,WAAW,GAAG;GACvB,QAAQ,OAAO,MAAM,4BAA4B;GACjD,OAAO;EACT;EACA,MAAM,OAAO,OAAO,YAAY,OAAO,KAAK,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;EAC9E,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,GAAG;EACzD,OAAO;CACT;CACA,MAAM,WAAW,CAAC,oBAAoB,OAAO,EAAE,MAAM,CAAC,GAAG,GAAG,OAAO,KAAK,SAAS,KAAK,IAAI,CAAC;CAC3F,QAAQ,OAAO,MAAM,GAAG,SAAS,KAAK,MAAM,EAAE,GAAG;CACjD,OAAO;AACT;AAEA,eAAe,mBAAmB,MAAiC;CACjE,MAAM,QAAkB,CAAC;CACzB,IAAI;CACJ,IAAI,OAAO;CACX,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,UAAU,UAAU;EACxB,IAAI,UAAU,UAAU;GACtB,OAAO;GACP;EACF;EACA,IAAI,UAAU,eAAe;GAC3B,YAAY,KAAK,EAAE;GACnB,IAAI,cAAc,KAAA,GAAW,MAAM,IAAI,MAAM,8BAA8B,OAAO;GAClF;EACF;EACA,IAAI,MAAM,WAAW,IAAI,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,IAAI,OAAO;EAC7E,MAAM,KAAK,KAAK;CAClB;CACA,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,0CAA0C,OAAO;CACzF,MAAM,SAAS,CAAC;CAChB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;EAC7C,OAAO,KAAK,uBAAuB,MAAM,SAAS,MAAM,MAAM,GAAG,UAAU,IAAI,CAAC;CAClF;CACA,MAAM,OAAO,WAAW,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,CAAC;CAC5E,QAAQ,OAAO,MACb,OAAO,GAAG,KAAK,UAAU,EAAE,YAAY,KAAK,GAAG,MAAM,CAAC,EAAE,MAAM,GAAG,qBAAqB,IAAI,EAAE,GAC9F;CACA,OAAO;AACT;AAEA,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,SAAS,WACP,OACA,YACA,OACwD;CACxD,MAAM,SAAiE,CAAC;CACxE,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,MAAM,SAAS,KAAK,KAAK;EACzB,OAAO,KAAK;GAAE,MAAM;GAAQ;GAAQ,MAAM,eAAe,OAAO,IAAI;EAAE,CAAC;CACzE;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,wBAAwB,OAAO,IAAI;EAAE,CAAC;CAC3F;CACA,IAAI,MAAM,IAAI,SAAS,GAAG;EACxB,MAAM,SAAS,MAAM,KAAK;EAC1B,OAAO,KAAK;GAAE,MAAM;GAAS;GAAQ,MAAM,gBAAgB,OAAO,IAAI;EAAE,CAAC;CAC3E;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,KAAK;EAC/B,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,qBAAqB,OAAO,IAAI;EAAE,CAAC;CACrF;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,OAAO,EAAE,cAAc,iBAAiB,UAAU,EAAE,CAAC;EAC/E,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,eAAe,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,aAAa,GAAG;EAC5B,MAAM,OAAO,iBAAiB,UAAU;EACxC,MAAM,SAAS,UACb,OACA,kBAAkB,YAAY,KAAK,QAAQ,QAAQ,KAAK,IAAI,CAC9D;EACA,OAAO,KAAK;GAAE,MAAM;GAAa;GAAQ,MAAM,gBAAgB,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,oBAAoB,MAAM;EAAE,CAAC;CAClF;CACA,OAAO;AACT;;;;;;AAOA,SAAS,iBAAiB,YAAqE;CAC7F,MAAM,wBAAQ,IAAI,IAAqB;CACvC,QAAQ,QAAQ;EACd,IAAI,MAAM,IAAI,IAAI,MAAM,GAAG,OAAO,MAAM,IAAI,IAAI,MAAM;EACtD,MAAM,MAAM,IAAI,OAAO,MAAM,CAAgB;EAC7C,MAAM,aAAa,CAAC,KAAK,QAAQ,UAAU,GAAG,SAAS,GAAG,IAAI,MAAM,CAAC;EACrE,IAAI,IAAI,IAAI,WAAW,OAAO,GAAG,WAAW,QAAQ,cAAc,IAAI,GAAG,CAAC;EAC1E,IAAI;EACJ,KAAK,MAAM,aAAa,YAAY;GAClC,IAAI,CAAC,WAAW,SAAS,GAAG;GAC5B,MAAM,QAAQ,aAAa,SAAS;GAEpC,IADe,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAChD,MAAM,OAAO,MAAM,eAAe,IAAI,YAAY;GAC3D,IAAI;IACF,QAAQ,KAAK,MAAM,MAAM,SAAS,MAAM,CAAC;GAC3C,QAAQ,CAER;GACA;EACF;EACA,MAAM,IAAI,IAAI,QAAQ,KAAK;EAC3B,OAAO;CACT;AACF;AAEA,SAAS,eAAe,MAAwB;CAC9C,MAAM,QAAQ,CACZ,SAAS,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,IAAI,IAAI,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,KACvH;CACA,MAAM,QAAQ,MAAgB,QAAgB,WAA0B;EACtE,MAAM,SAAS,WAAW,KAAK,KAAK,SAAS,QAAQ;EACrD,MAAM,KAAK,KAAK,WAAW,GAAG,KAAK,SAAS,OAAO;EACnD,MAAM,OAAO,KAAK,SAAS,OAAO,KAAK,SAAS,KAAK;EACrD,MAAM,QAAQ,KAAK,UAAU,OAAO,mBAAmB;EACvD,MAAM,OACJ,KAAK,qBAAqB,IACtB,IAAI,KAAK,aAAa,QAAQ,CAAC,MAC/B,IAAI,KAAK,aAAa,QAAQ,CAAC,EAAE,qBAAqB,KAAK,SAAS,QAAQ,CAAC,EAAE,QAAQ,KAAK,iBAAiB,oBAAoB,KAAK,qBAAqB,IAAI,KAAK;EAC1K,MAAM,KACJ,GAAG,SAAS,SAAS,KAAK,KAAK,OAAO,IAAI,KAAK,UAAU,cAAc,OAAO,MAAM,IAAI,MAC1F;EACA,MAAM,cAAc,UAAU,WAAW,KAAK,KAAK,SAAS,QAAQ;EACpE,KAAK,SAAS,SAAS,OAAO,UAAU;GACtC,KAAK,OAAO,aAAa,UAAU,KAAK,SAAS,SAAS,CAAC;EAC7D,CAAC;CACH;CACA,KAAK,MAAM,SAAS,MAAM,UAAU;EAClC,KAAK,MAAM,IAAI,UAAU,KAAK,MAAM,SAAS,CAAC;CAChD,CAAC;CACD,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,wBAAwB,MAAiC;CAChE,MAAM,QAAQ,CAAC,mBAAmB,KAAK,MAAM,gDAAgD;CAC7F,IAAI,KAAK,KAAK,WAAW,GAAG,MAAM,KAAK,gBAAgB;CACvD,KAAK,MAAM,OAAO,KAAK,MAAM;EAC3B,MAAM,IAAI,IAAI;EACd,MAAM,WAAW,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAC1C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,GAAG;EACX,MAAM,YACJ,EAAE,WAAW,SACT,EAAE,MAAM,IACN,gCACA,kBAAkB,EAAE,EAAE,MAAM,kBAAkB,WAChD,EAAE,WAAW,iBACX,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,kBAAkB,mBAC9D,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,eAAe,MAAM,EAAE,EAAE;EAC/I,MAAM,QAAQ,OAAO,OAAO,IAAI,QAAQ,CAAC,CAAC,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC;EAC/E,MAAM,cAAc,IAAI,cAAc,IAAI,KAAK,IAAI,YAAY,iBAAiB;EAChF,MAAM,KACJ,KAAK,IAAI,SAAS,IAAI,IAAI,UAAU,YAAY,YAAY,KAAK,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,YAAY,mBAAmB,KAAK,UAAU,IAAI,IAAI,SAAS,UAC1K;CACF;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,gBAAgB,MAAyB;CAChD,MAAM,UAAU,KAAK,KAAK,QAAQ,QAAQ,IAAI,OAAO;CACrD,MAAM,UAAU,OAAO,QAAQ,KAAK,UAAU,CAAC,CAC5C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,IAAI;CACZ,MAAM,QAAQ,CACZ,UAAU,KAAK,MAAM,gBAAgB,KAAK,KAAK,KAAK,IAAI,EAAE,KAAK,KAAK,aAAa,MAAM,KAAK,KAAK,OAAO,4BAA4B,KAAK,cAAc,WAAW,UAAU,KAAK,QAAQ,KAAK,IAChM;CACA,KAAK,MAAM,OAAO,SAChB,MAAM,KACJ,KAAK,IAAI,OAAO,UAAU,IAAI,MAAO,QAAQ,CAAC,EAAE,MAAM,IAAI,MAAM,UAAU,IAAI,OAAO,iBAAiB,IAAI,eAAgB,QAAQ,CAAC,GACrI;CAEF,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,qBAAqB,MAA8B;CAC1D,MAAM,QAAQ,CACZ,gBAAgB,KAAK,MAAM,UAAU,KAAK,UAAU,KAAK,KAAK,QAAQ,OAAO,aAAa,KAAK,QAAQ,OAAO,YAAY,KAAK,aAAa,OAAO,oBAAoB,KAAK,aAAa,OAAO,iBAClM;CACA,KAAK,MAAM,OAAO,KAAK,gBAAgB;EACrC,MAAM,OACJ,IAAI,SAAS,OACT,iBAAiB,IAAI,aAAa,KAClC,GAAG,IAAI,KAAK,QAAQ,CAAC,EAAE,SAAS,IAAI,WAAW,IAAI,IAAI,kBAAkB,YAAY,IAAI,QAAQ,OAAO;EAC9G,MAAM,KAAK,aAAa,IAAI,UAAU,IAAI,IAAI,QAAQ,KAAK,IAAI,EAAE,qBAAqB,MAAM;CAC9F;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,cAAc,MAAiE;CACtF,MAAM,QAAgC,CAAC;CACvC,MAAM,aAAuB,CAAC;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,CAAC,MAAM,WAAW,IAAI,GAAG;GAC3B,WAAW,KAAK,KAAK;GACrB;EACF;EACA,MAAM,CAAC,MAAM,UAAU,MAAM,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;EAClD,IAAI,SAAS,cAAc,SAAS,WAAW,MAAM,IAAI,MAAM,kBAAkB,MAAM;EACvF,MAAM,QAAQ,UAAU,KAAK,EAAE;EAC/B,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,KAAK,KAAK,eAAe;EAClE,MAAM,QAAQ;CAChB;CACA,IAAI,WAAW,WAAW,GAAG,MAAM,IAAI,MAAM,KAAK;CAClD,OAAO;EAAE,MAAM,WAAW;EAAK;CAAM;AACvC;;AAGA,eAAe,kBAAkB,MAA+B;CAC9D,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;CACnC,IAAI;EAEF,MAAM,QAAQ,OAAO,MAAM,gCAAgC,CAAC;EAC5D,MAAM,EAAE,cAAc,MAAM,OAAO,KAAK,OAAO,GAAG,MAAM,YAAY,CAAC;EACrE,MAAM,UAAU,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,QAAQ,EAAI;EACzD,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,GAAG,KAAK,+BAA+B;EACxE,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,QAAQ,GAAG,OAAO,CAAC;EAC3D,IAAI,OAAO,MAAM,aAAa,UAAU,MAAM,IAAI,MAAM,GAAG,KAAK,0BAA0B;EAC1F,OAAO,MAAM;CACf,UAAU;EACR,MAAM,OAAO,MAAM;CACrB;AACF;;;;;;;;;;;;;;;;;;;AChXA,SAAS,UAAU,MAAsB;CACvC,MAAM,CAAC,SAAS,GAAG,QAAQ;CAC3B,MAAM,aAAuB,CAAC;CAC9B,MAAM,QAAgC,CAAC;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,IAAI,WAAW,IAAI,GAAG;GACxB,MAAM,MAAM,IAAI,MAAM,CAAC;GACvB,MAAM,WAAW,IAAI,QAAQ,GAAG;GAChC,IAAI,YAAY,GAAG;IACjB,MAAM,IAAI,MAAM,GAAG,QAAQ,KAAK,IAAI,MAAM,WAAW,CAAC;IACtD;GACF;GACA,MAAM,MAAM;GACZ,IAAI,QAAQ,QAAQ;IAClB,MAAM,OAAO;IACb;GACF;GACA,MAAM,OAAO,KAAK,IAAI;GACtB,IAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;IAC1C,MAAM,OAAO;IACb;GACF,OACE,MAAM,OAAO;EAEjB,OAAO,IAAI,QAAQ,MACjB,MAAM,OAAO;OAEb,WAAW,KAAK,GAAG;CAEvB;CACA,OAAO;EAAE,SAAS,WAAW;EAAQ;EAAY;CAAM;AACzD;AAEA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCb,eAAe,OAAwB;CACrC,MAAM,EAAE,SAAS,YAAY,UAAU,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;CACtE,iBAAiB,SAAS,KAAK;CAE/B,IAAI,YAAY,uBAAuB,MAAM,SAAS,QACpD,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;CAE/D,IAAI,YAAY,UAAU,OAAO,MAAM,iBAAiB,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7E,IAAI,MAAM,SAAS,QAAQ;EACzB,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;EAChC,OAAO;CACT;CAEA,QAAQ,SAAR;EACE,KAAK,SAAS;GACZ,MAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;GAC3C,MAAM,OAAO,MAAM,QAAQ;GAC3B,MAAM,MAAM,oBAAoB;GAChC,MAAM,EAAE,WAAW,MAAM,iBAAiB;IACxC;IACA;IACA,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;GAED,MAAM,YAAY,QAAgB;IAEhC,QAAQ,IAAI,yBAAyB,IAAI,gBAAgB;IACzD,OAAO,YAAY,QAAQ,KAAK,CAAC,CAAC;IAElC,iBAAiB,QAAQ,KAAK,CAAC,GAAG,GAAI,CAAC,CAAC,MAAM;GAChD;GACA,QAAQ,GAAG,gBAAgB,SAAS,QAAQ,CAAC;GAC7C,QAAQ,GAAG,iBAAiB,SAAS,SAAS,CAAC;GAE/C,MAAM,IAAI,cAAc,CAAC,CAAC;GAC1B,OAAO;EACT;EACA,KAAK,OAAO;GACV,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,WAAW,QAAQ;IAC9B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,aAAa;GAChB,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,YAAY,QAAQ;IAC/B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,WAAW;GACd,MAAM,MAAM,MAAM,OAAO;GACzB,MAAM,OAAO,aAAa,cAAc,CAAC,CAAC,OAAO;GACjD,cAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,KAAK,OAAO;GAEhE,QAAQ,IAAI,0CAA0C,KAAK;GAC3D,OAAO;EACT;EACA,KAAK,mBAGH,OAAO,MAAM,qBAAqB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAEzD,KAAK,qBACH,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE/D,KAAK,kBAEH,OAAO,MAAM,wBAAwB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE5D,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,EAAE,GAAG;GACpE,OAAO;EAET,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,cAAc,CAAC,CAAC,QAAQ,GAAG;GACnD,OAAO;EAET,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;GAChC,OAAO;EACT;GACE,QAAQ,OAAO,MAAM,oBAAoB,QAAQ,IAAI,KAAK,GAAG;GAC7D,OAAO;CACX;AACF;AAEA,MAAM,mBAAwD;CAC5D,uBAAO,IAAI,IAAI;EAAC;EAAQ;EAAQ;CAAM,CAAC;CACvC,qBAAK,IAAI,IAAI,CAAC,MAAM,CAAC;CACrB,6BAAa,IAAI,IAAI,CAAC,MAAM,CAAC;CAC7B,yBAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;CAChC,yBAAS,IAAI,IAAI,CAAC,MAAM,CAAC;CACzB,sBAAM,IAAI,IAAI;CACd,0BAAU,IAAI,IAAI;CAClB,sBAAM,IAAI,IAAI;CACd,6BAAa,IAAI,IAAI;CACrB,oBAAI,IAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;CAE9E,IACE,YAAY,qBACZ,YAAY,uBACZ,YAAY,oBACZ,YAAY,UAEZ;CAEF,MAAM,UAAU,iBAAiB;CACjC,IAAI,CAAC,SAAS;CACd,MAAM,UAAU,OAAO,KAAK,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;CACtE,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MAAM,oBAAoB,WAAW,OAAO,MAAM,QAAQ,IAAI;AAE5E;AAEA,SAAS,UAAU,KAAqB;CACtC,MAAM,OAAO,OAAO,GAAG;CACvB,IAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAChD,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,GAAG;CAE9F,OAAO;AACT;AAEA,KAAK,CAAC,CACH,MAAM,SAAS;CACd,QAAQ,WAAW;AACrB,CAAC,CAAC,CACD,OAAO,QAAQ;CAEd,QAAQ,MAAM,2BAA2B,GAAG;CAC5C,QAAQ,WAAW;AACrB,CAAC"}
|
package/dist/contract/index.js
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
import { a as hashCanonical } from "../canonical-BsZVgBhW.js";
|
|
2
|
-
import { r as pairedBootstrap } from "../paired-tests-
|
|
3
|
-
import { t as decidePairedPromotion } from "../paired-promotion-decision-
|
|
4
|
-
import { a as heldoutSignificance } from "../statistical-heldout-
|
|
5
|
-
import { n as summarizeExecution, t as analyzeRuns } from "../analyze-runs-
|
|
2
|
+
import { r as pairedBootstrap } from "../paired-tests-Ccr16gcW.js";
|
|
3
|
+
import { t as decidePairedPromotion } from "../paired-promotion-decision-DhrmjdV8.js";
|
|
4
|
+
import { a as heldoutSignificance } from "../statistical-heldout-CNLOvRq_.js";
|
|
5
|
+
import { n as summarizeExecution, t as analyzeRuns } from "../analyze-runs-BrIibNd0.js";
|
|
6
6
|
import { a as parseRunRecordSafe } from "../run-record-BRlM99rX.js";
|
|
7
|
-
import { K as defaultProductionGate, a as transientDispatchFailure, c as compareOptimizationMethods, h as runImprovementLoop, nt as runEval, rt as runCampaign, t as llmJudge } from "../llm-judge-
|
|
7
|
+
import { K as defaultProductionGate, a as transientDispatchFailure, c as compareOptimizationMethods, h as runImprovementLoop, nt as runEval, rt as runCampaign, t as llmJudge } from "../llm-judge-C-0P1WiT.js";
|
|
8
8
|
import { i as isModelPriced, n as estimateCost } from "../metrics-DOMpkoeb.js";
|
|
9
9
|
import { i as CostLedger } from "../cost-ledger-_IbkejCF.js";
|
|
10
|
-
import { q as campaignSplitDigest } from "../campaign-evidence-
|
|
10
|
+
import { q as campaignSplitDigest } from "../campaign-evidence-Dt_rWPk2.js";
|
|
11
11
|
import { h as mapConcurrentRange } from "../journal-DvKCvjty.js";
|
|
12
|
-
import { C as inMemoryCampaignStorage, S as fsCampaignStorage } from "../external-optimizer-subprocess-
|
|
13
|
-
import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-
|
|
14
|
-
import { t as buildDefaultAnalystRegistry } from "../default-registry-
|
|
12
|
+
import { C as inMemoryCampaignStorage, S as fsCampaignStorage } from "../external-optimizer-subprocess-CkyUUV7h.js";
|
|
13
|
+
import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-CJN9dTQw.js";
|
|
14
|
+
import { t as buildDefaultAnalystRegistry } from "../default-registry-D4qXx2aG.js";
|
|
15
15
|
import { t as extractUsage } from "../extract-usage-DFATEuWW.js";
|
|
16
16
|
import { i as makeProposalFinding } from "../types-CW5b90JH.js";
|
|
17
|
-
import { t as createChatClient } from "../chat-client-
|
|
18
|
-
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-
|
|
19
|
-
import { c as heldOutGate, g as runReferenceEquivalenceJudge, h as createReferenceEquivalenceJudge, l as composeGate, m as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, p as REFERENCE_EQUIVALENCE_INPUT_LIMITS, t as skillOptOptimizationMethod, u as externalTextOptimizationMethod } from "../skillopt-optimization-method-
|
|
20
|
-
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-
|
|
21
|
-
import { t as fromOtelSpans } from "../otel-spans-
|
|
17
|
+
import { t as createChatClient } from "../chat-client-DUhWNQQS.js";
|
|
18
|
+
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-DUcMj-3D.js";
|
|
19
|
+
import { c as heldOutGate, g as runReferenceEquivalenceJudge, h as createReferenceEquivalenceJudge, l as composeGate, m as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, p as REFERENCE_EQUIVALENCE_INPUT_LIMITS, t as skillOptOptimizationMethod, u as externalTextOptimizationMethod } from "../skillopt-optimization-method-Bls9t-G_.js";
|
|
20
|
+
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-BI1RxzwD.js";
|
|
21
|
+
import { t as fromOtelSpans } from "../otel-spans-C8YIJWy1.js";
|
|
22
22
|
import { createHash } from "node:crypto";
|
|
23
23
|
import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
|
|
24
24
|
import { createReadStream } from "node:fs";
|
|
@@ -314,7 +314,8 @@ function evaluatePairedMeasurements(options) {
|
|
|
314
314
|
const regressionProven = new Set(dimensions.filter((name) => {
|
|
315
315
|
if (!guardedDimensions.has(name)) return false;
|
|
316
316
|
const baselineDim = measurements.map((m) => dimensionScore(m.baseline, name));
|
|
317
|
-
|
|
317
|
+
const candidateDim = measurements.map((m) => dimensionScore(m.candidate, name));
|
|
318
|
+
return decidePairedPromotion(candidateDim, baselineDim, {
|
|
318
319
|
confidence,
|
|
319
320
|
resamples,
|
|
320
321
|
statistic: "mean",
|
|
@@ -517,7 +518,7 @@ function measuredComparisonFromCandidateExperiment(options) {
|
|
|
517
518
|
/** Recompute every statistic and decision from the signed experiment receipts. */
|
|
518
519
|
function verifyCandidateExperimentComparison(input) {
|
|
519
520
|
const comparison = agentImprovementMeasuredComparisonSchema.parse(input);
|
|
520
|
-
|
|
521
|
+
const recomputed = measuredComparisonFromCandidateExperiment({
|
|
521
522
|
experiment: comparison.experiment,
|
|
522
523
|
measurements: comparison.measurements,
|
|
523
524
|
runId: comparison.provenance.runId,
|
|
@@ -529,7 +530,8 @@ function verifyCandidateExperimentComparison(input) {
|
|
|
529
530
|
cost: comparison.evaluation.measurement.cost
|
|
530
531
|
},
|
|
531
532
|
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
532
|
-
})
|
|
533
|
+
});
|
|
534
|
+
if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("candidate experiment comparison does not match its Runtime receipts");
|
|
533
535
|
return comparison;
|
|
534
536
|
}
|
|
535
537
|
function candidateMeasurementMaximumCostUsd(experiment) {
|
|
@@ -1018,7 +1020,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
|
|
|
1018
1020
|
/** Recompute a profile comparison from the exact sealed experiment and receipts. */
|
|
1019
1021
|
function verifyAgentProfileImprovementExperimentComparison(input) {
|
|
1020
1022
|
const comparison = agentProfileImprovementMeasuredComparisonSchema.parse(input);
|
|
1021
|
-
|
|
1023
|
+
const recomputed = measuredComparisonFromAgentProfileImprovementExperiment({
|
|
1022
1024
|
experiment: comparison.experiment,
|
|
1023
1025
|
measurements: comparison.measurements,
|
|
1024
1026
|
runId: comparison.provenance.runId,
|
|
@@ -1030,7 +1032,8 @@ function verifyAgentProfileImprovementExperimentComparison(input) {
|
|
|
1030
1032
|
cost: comparison.evaluation.measurement.cost
|
|
1031
1033
|
},
|
|
1032
1034
|
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
1033
|
-
})
|
|
1035
|
+
});
|
|
1036
|
+
if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("profile improvement comparison does not match its Runtime receipts");
|
|
1034
1037
|
return comparison;
|
|
1035
1038
|
}
|
|
1036
1039
|
function profileMeasurementMaximumCostUsd(experiment) {
|