@tangle-network/agent-eval 0.207.3 → 0.208.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-EpO0HURy.d.ts → agent-profile-CEhxR_I3.d.ts} +4 -4
- package/dist/{agent-profile-EpO0HURy.d.ts.map → agent-profile-CEhxR_I3.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +12 -12
- package/dist/analyst/index.js +5 -5
- package/dist/{analyze-runs-Q74o4Hfj.d.ts → analyze-runs-CTJyUplR.d.ts} +2 -2
- package/dist/{analyze-runs-Q74o4Hfj.d.ts.map → analyze-runs-CTJyUplR.d.ts.map} +1 -1
- package/dist/{benchmark-command-DtP94cOB.js → benchmark-command-BC265T1r.js} +6 -6
- package/dist/{benchmark-command-DtP94cOB.js.map → benchmark-command-BC265T1r.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +7 -7
- package/dist/{campaign-BL591dWw.js → campaign-C5pjR-Qn.js} +8 -8
- package/dist/{campaign-BL591dWw.js.map → campaign-C5pjR-Qn.js.map} +1 -1
- package/dist/{chat-client-5LLl4Ctl.js → chat-client-CydfmLdO.js} +3 -2
- package/dist/chat-client-CydfmLdO.js.map +1 -0
- package/dist/{chat-json-call-Digk-pfF.js → chat-json-call-BJJdJ2uj.js} +2 -2
- package/dist/{chat-json-call-Digk-pfF.js.map → chat-json-call-BJJdJ2uj.js.map} +1 -1
- package/dist/cli.js +6 -6
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +6 -6
- package/dist/{default-registry-BthquIoC.js → default-registry-DX5R7_n9.js} +2 -2
- package/dist/{default-registry-BthquIoC.js.map → default-registry-DX5R7_n9.js.map} +1 -1
- package/dist/{default-registry-BHEfZHQd.d.ts → default-registry-DZFWY2_E.d.ts} +6 -6
- package/dist/{default-registry-BHEfZHQd.d.ts.map → default-registry-DZFWY2_E.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CmItPfIq.js → define-agent-eval-BjKXcDQp.js} +3 -3
- package/dist/{define-agent-eval-CmItPfIq.js.map → define-agent-eval-BjKXcDQp.js.map} +1 -1
- package/dist/{define-agent-eval-B2YV-RDE.d.ts → define-agent-eval-C8zBSbsc.d.ts} +7 -7
- package/dist/{define-agent-eval-B2YV-RDE.d.ts.map → define-agent-eval-C8zBSbsc.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/{dspy-rlm-engine-DCQKNQMn.js → dspy-rlm-engine-sHwdSxK3.js} +2 -2
- package/dist/{dspy-rlm-engine-DCQKNQMn.js.map → dspy-rlm-engine-sHwdSxK3.js.map} +1 -1
- package/dist/{engine-F_PvwesR.d.ts → engine-DProXJ7A.d.ts} +4 -4
- package/dist/{engine-F_PvwesR.d.ts.map → engine-DProXJ7A.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-BBMMYk3g.d.ts → exact-types-Cq6xXNVR.d.ts} +2 -2
- package/dist/{exact-types-BBMMYk3g.d.ts.map → exact-types-Cq6xXNVR.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/{external-optimizer-process-CIV4EqiT.js → external-optimizer-process-D2Txd-7D.js} +2 -2
- package/dist/{external-optimizer-process-CIV4EqiT.js.map → external-optimizer-process-D2Txd-7D.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-BmKtH3T6.js → external-optimizer-subprocess-C0EsxEbI.js} +12 -4
- package/dist/{external-optimizer-subprocess-BmKtH3T6.js.map → external-optimizer-subprocess-C0EsxEbI.js.map} +1 -1
- package/dist/{feedback-trajectory-C6NJxXEa.d.ts → feedback-trajectory-C6Z8PRoF.d.ts} +3 -3
- package/dist/{feedback-trajectory-C6NJxXEa.d.ts.map → feedback-trajectory-C6Z8PRoF.d.ts.map} +1 -1
- package/dist/{index-CojsA_EE.d.ts → index-Bzto04CY.d.ts} +7 -7
- package/dist/{index-CojsA_EE.d.ts.map → index-Bzto04CY.d.ts.map} +1 -1
- package/dist/{index-DzYX3WBL.d.ts → index-DkucR0KJ.d.ts} +7 -7
- package/dist/{index-DzYX3WBL.d.ts.map → index-DkucR0KJ.d.ts.map} +1 -1
- package/dist/{index-DVDiJIhi.d.ts → index-nwNoS582.d.ts} +8 -8
- package/dist/{index-DVDiJIhi.d.ts.map → index-nwNoS582.d.ts.map} +1 -1
- package/dist/index.d.ts +19 -19
- package/dist/index.js +10 -10
- package/dist/{integrity-BWO6RPxv.d.ts → integrity-zloavsM_.d.ts} +2 -2
- package/dist/{integrity-BWO6RPxv.d.ts.map → integrity-zloavsM_.d.ts.map} +1 -1
- package/dist/{llm-client-DmCNzrc9.js → llm-client-v968tWw8.js} +76 -29
- package/dist/llm-client-v968tWw8.js.map +1 -0
- package/dist/{llm-judge-Fc3UIcPq.js → llm-judge-HLWASwUJ.js} +5 -5
- package/dist/{llm-judge-Fc3UIcPq.js.map → llm-judge-HLWASwUJ.js.map} +1 -1
- package/dist/{matrix-W1lbSx72.d.ts → matrix-Cqkk7eYI.d.ts} +2 -2
- package/dist/{matrix-W1lbSx72.d.ts.map → matrix-Cqkk7eYI.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{prime-protocol-IoN4Qb6c.d.ts → prime-protocol-Csr9kcOM.d.ts} +2 -2
- package/dist/{prime-protocol-IoN4Qb6c.d.ts.map → prime-protocol-Csr9kcOM.d.ts.map} +1 -1
- package/dist/{produced-state-CywjstcW.js → produced-state-Bzlh09oj.js} +3 -3
- package/dist/{produced-state-CywjstcW.js.map → produced-state-Bzlh09oj.js.map} +1 -1
- package/dist/{registry-CT-hMSkR.d.ts → registry-DMkmhBQ5.d.ts} +4 -4
- package/dist/{registry-CT-hMSkR.d.ts.map → registry-DMkmhBQ5.d.ts.map} +1 -1
- package/dist/{researcher-zTpxNJBy.d.ts → researcher-CZFZjj-u.d.ts} +3 -3
- package/dist/{researcher-zTpxNJBy.d.ts.map → researcher-CZFZjj-u.d.ts.map} +1 -1
- package/dist/rl.d.ts +4 -4
- package/dist/search/index.d.ts +2 -2
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-mzeNuzr8.js → search-kernel-CIie6xig.js} +44 -2
- package/dist/search-kernel-CIie6xig.js.map +1 -0
- package/dist/{search-kernel-CZ8KJQRw.d.ts → search-kernel-uKdPVRAe.d.ts} +18 -3
- package/dist/search-kernel-uKdPVRAe.d.ts.map +1 -0
- package/dist/{search-summary-CDURlgZR.js → search-summary-BV5uehLd.js} +2 -2
- package/dist/{search-summary-CDURlgZR.js.map → search-summary-BV5uehLd.js.map} +1 -1
- package/dist/{semantic-concept-judge-BbiYuv7U.js → semantic-concept-judge-D1wH-YKc.js} +4 -4
- package/dist/{semantic-concept-judge-BbiYuv7U.js.map → semantic-concept-judge-D1wH-YKc.js.map} +1 -1
- package/dist/{series-convergence-DLy0bQMV.d.ts → series-convergence-hEfKdoi-.d.ts} +2 -2
- package/dist/{series-convergence-DLy0bQMV.d.ts.map → series-convergence-hEfKdoi-.d.ts.map} +1 -1
- package/dist/{server-CoBZO1fj.js → server-BZOt7eNC.js} +2 -2
- package/dist/{server-CoBZO1fj.js.map → server-BZOt7eNC.js.map} +1 -1
- package/dist/{skillopt-optimization-method-BSgM_cbI.js → skillopt-optimization-method-Dcnh5Pvr.js} +5 -5
- package/dist/{skillopt-optimization-method-BSgM_cbI.js.map → skillopt-optimization-method-Dcnh5Pvr.js.map} +1 -1
- package/dist/{statistical-heldout-COfgpWah.d.ts → statistical-heldout-DFKtp-Qm.d.ts} +2 -2
- package/dist/{statistical-heldout-COfgpWah.d.ts.map → statistical-heldout-DFKtp-Qm.d.ts.map} +1 -1
- package/dist/{store-tool-spans-BYIarVrG.d.ts → store-tool-spans-9YJNMP-z.d.ts} +5 -5
- package/dist/{store-tool-spans-BYIarVrG.d.ts.map → store-tool-spans-9YJNMP-z.d.ts.map} +1 -1
- package/dist/{systemone-review-DGTKkLMG.d.ts → systemone-review-Bt4JcxMc.d.ts} +2 -2
- package/dist/{systemone-review-DGTKkLMG.d.ts.map → systemone-review-Bt4JcxMc.d.ts.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/{tool-groups-B7_jUPHp.d.ts → tool-groups-j19pmOfP.d.ts} +3 -3
- package/dist/tool-groups-j19pmOfP.d.ts.map +1 -0
- package/dist/{trace-questions-Dd3AMf-U.d.ts → trace-questions-dRvmsbTk.d.ts} +3 -3
- package/dist/{trace-questions-Dd3AMf-U.d.ts.map → trace-questions-dRvmsbTk.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +5 -5
- package/dist/{tree-CN8mlzUO.js → tree-CJOXbVcv.js} +2 -2
- package/dist/{tree-CN8mlzUO.js.map → tree-CJOXbVcv.js.map} +1 -1
- package/dist/{types-DZQpUJse.d.ts → types-BlSBUNRd.d.ts} +2 -2
- package/dist/{types-DZQpUJse.d.ts.map → types-BlSBUNRd.d.ts.map} +1 -1
- package/dist/{types-ZUy5HGY5.d.ts → types-CFCUZk0B.d.ts} +3 -3
- package/dist/{types-ZUy5HGY5.d.ts.map → types-CFCUZk0B.d.ts.map} +1 -1
- package/dist/{types-6VEehmEg.d.ts → types-DAPpAkd4.d.ts} +2 -2
- package/dist/{types-6VEehmEg.d.ts.map → types-DAPpAkd4.d.ts.map} +1 -1
- package/dist/{types-DLxe96rp.d.ts → types-Lub7o6bN.d.ts} +10 -3
- package/dist/{types-DLxe96rp.d.ts.map → types-Lub7o6bN.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +2 -2
- package/dist/wire/index.js +1 -1
- package/docs/changes/llm-call-pacing.md +56 -0
- package/docs/quickstart-external.md +268 -0
- package/docs/search-ledger.md +4 -1
- package/package.json +1 -1
- package/dist/chat-client-5LLl4Ctl.js.map +0 -1
- package/dist/llm-client-DmCNzrc9.js.map +0 -1
- package/dist/search-kernel-CZ8KJQRw.d.ts.map +0 -1
- package/dist/search-kernel-mzeNuzr8.js.map +0 -1
- package/dist/tool-groups-B7_jUPHp.d.ts.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { c as RunSplitTag } from "./run-record-DF-fuu31.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext } from "./types-
|
|
2
|
+
import { R as Scenario, d as DispatchContext } from "./types-CFCUZk0B.js";
|
|
3
3
|
//#region src/benchmarks/types.d.ts
|
|
4
4
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
5
5
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -99,4 +99,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
|
99
99
|
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
100
100
|
//#endregion
|
|
101
101
|
export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
|
|
102
|
-
//# sourceMappingURL=types-
|
|
102
|
+
//# sourceMappingURL=types-BlSBUNRd.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-BlSBUNRd.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { d as RunTokenUsage, s as RunSearchCoordinates } from "./run-record-DF-fuu31.js";
|
|
3
|
-
import { _ as ProposalFinding } from "./types-
|
|
4
|
-
import { M as LlmCallMetadata } from "./types-
|
|
3
|
+
import { _ as ProposalFinding } from "./types-DAPpAkd4.js";
|
|
4
|
+
import { M as LlmCallMetadata } from "./types-Lub7o6bN.js";
|
|
5
5
|
import { t as SeriesDistribution } from "./descriptive-B2iPaT9J.js";
|
|
6
6
|
import { A as SearchLedgerHash, G as SearchSplit, I as SearchNodeStatus, L as SearchOpenedEvent, N as SearchNodeDecision, O as SearchLedgerEntry, P as SearchNodeRef, Q as SearchTaskOutcome, R as SearchOperationKind, U as SearchReservation, Z as SearchTask, b as SearchClosedEvent, d as SearchCellCancelledEvent, i as SearchArtifactRef, l as SearchCandidateSurface, o as SearchAudit, p as SearchCellStage, t as NodeEstimate, w as SearchEdgeRecordedEvent, z as SearchOperationRecordedEvent } from "./search-ledger-types-p2xKfiDJ.js";
|
|
7
7
|
import { p as LedgerProjector } from "./index-Bj96W0jZ.js";
|
|
@@ -1015,4 +1015,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
1015
1015
|
}
|
|
1016
1016
|
//#endregion
|
|
1017
1017
|
export { SearchNode as $, LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, renderSearchSummary as J, SearchProposerView as K, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, SearchDecisionRecord as Q, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, SearchCell as X, searchProposerView as Y, SearchCompletion as Z, GateDecision as _, CampaignResult as a, SearchUnitScore as at, GenerationRecord as b, CampaignTraceWriter as c, searchTaskSetDigest as ct, DispatchContext as d, SearchOperation as et, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, SearchStateView as it, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, searchUnitScores as lt, GateCheckStatus as m, CampaignArtifactWriter as n, SearchSpend as nt, CampaignScenarioIdentity as o, searchCellId as ot, Gate as p, SearchSummaryOptions as q, CampaignCellResult as r, SearchState as rt, CampaignTokenUsage as s, searchNodeId as st, CampaignAggregates as t, SearchScoredCell as tt, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
1018
|
-
//# sourceMappingURL=types-
|
|
1018
|
+
//# sourceMappingURL=types-CFCUZk0B.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-CFCUZk0B.d.ts","names":[],"sources":["../src/campaign/search-state.ts","../src/campaign/search-summary.ts","../src/campaign/types.ts"],"mappings":";;;;;;;;;;iBAwDgB,aAAa,kBAAkB;;iBAK/B,aACd,kBACA,gBACA,gBACA,OAAO,aACP;;iBAMc,oBAAoB,gBAAgB,eAAe;UAkBlD;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;;EAEA;EACA;;;;;;;;iBASc,iBAAiB,gBAAgB,qBAAqB;UAoBrD;EACf;;EAEA;EACA,gBAAgB;EAChB,UAAU;EACV,mBAAmB;;;EAGnB;;EAEA;;;;EAIA,kBAAkB;EAClB;EACA;EACA,QAAQ;EACR;;EAEA;EACA,oBAAoB;EACpB;EACA,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,aAAa;EACb;EACA;EACA,SAAS;;EAET;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,aAAa;EACb;;EAEA,SAAS;;;EAGT,WAAW;;;EAGX,oBAAoB;;EAEpB;;EAEA;;UAGe;EACf;EACA;;;;;;;cA6BW,uBAAuB,gBAAgB,mBAAmB;WAC5D;UACD;UACA;UACA;UACA;UACA;UACA;mBACS;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;UACT;UACA;;mBAES;;UAET;mBACS;EAEjB,YAAY;EAgCZ,MAAM,OAAO,mBAAmB;EA6BhC,YAAY;;EAKZ,KAAK;UAIG;UAgCA;UA6CA;UAsBA;UA8BA;UA8CA;UAsHA;UAmEA;UA4EA;UAoBA;UAmDA;UAaA;;;UA2EA;;EASR;;;UAmBQ;;;UAkBA;;;;;;;;;;;UAoBA;;;EAsBR;IAAgB;IAA4B;;;EAa5C,cAAc;;EAKd,cAAc;;EAKd;IAAc;IAAkB,WAAW;;;EAK3C;;EAKA,aAAa;;EAcb,SAAS,iBAAiB;;EAM1B;;EAKA,YAAY;;EAKZ,oBAAoB;;EAKpB,SAAS,iBAAiB;;EAK1B;;EAKA,SAAS,iBAAiB;;EAM1B,YAAY;;EAMZ,cAAc,sBAAsB;;EAMpC,gBAAgB,gBAAgB,OAAO,cAAc;;;;;;;cAiB1C;WACF;WACA,QAAQ;WACR,QAAQ;WACR;IAAQ;IAAkB,WAAW;;;WAErC;WACA,OAAO;WACP,YAAY;;;;;;;WAOZ;IAAU;IAA4B;;mBAC9B;mBACA;EAEjB,YAAY,OAAO,aAAa;;MAgB5B;EAIJ,KAAK,iBAAiB;EAItB,QAAQ;;EAKR;;EAKA,SAAS;EAKT,gBAAgB;EAIhB,KAAK,iBAAiB;;EAKtB,SAAS;EAKT,KAAK,iBAAiB;;EAKtB,MAAM;IAAU;MAAyB;EAKzC,UAAU,sBAAsB;;;EAMhC,YAAY,gBAAgB,OAAO,cAAc;;EAKjD,WAAW,gBAAgB,OAAO,cAAc;;EAKhD,QAAQ,aAAa,uBAAuB;UAmCpC;;;;;;UCluCO;WACN;WACA;;;EAGT,YAAY,0BAA0B;;EAEtC,WAAW,0BAA0B;;;iBAIvB,mBAAmB,OAAO,kBAAkB;UAW3C;;;;;;;;EAQf,OAAO;;EAEP;;;;;;;;;;;iBAYc,oBAAoB,OAAO,iBAAiB,SAAS;;;;;UCtCpD;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;;WAIjC,gBAAgB;;;;;WAKhB;;;;WAIA,UAAU;IAAgB;IAAgB,UAAU;;;;WAGpD,QAAQ;;;;WAIR;;;WAGA,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;;UAMX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;;EAEA;IAAY;IAAsB;;EAClC;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;;;EAIA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { c as CostLedgerHandle, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { a as RunRecord, d as RunTokenUsage } from "./run-record-DF-fuu31.js";
|
|
3
|
-
import { c as JudgeInput, x as ChatClient } from "./types-
|
|
3
|
+
import { c as JudgeInput, x as ChatClient } from "./types-Lub7o6bN.js";
|
|
4
4
|
import { SpanKind } from "@tangle-network/agent-trace-contract";
|
|
5
5
|
import { RE2JS } from "re2js";
|
|
6
6
|
//#region src/trace-analyst/types.d.ts
|
|
@@ -630,4 +630,4 @@ type AnalystRunEvent = {
|
|
|
630
630
|
};
|
|
631
631
|
//#endregion
|
|
632
632
|
export { DEFAULT_TRACE_ANALYST_BUDGETS as A, TraceAnalystSpan as B, BoundedTraceAnalysisStoreOptions as C, TRACE_ANALYSIS_LIMITS as D, SpanSourceReader as E, SearchTraceResult as F, ViewTraceOversized as G, TraceAnalystSpanStatus as H, SpanMatchRecord as I, ViewTraceResult as K, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as L, ErrorCluster as M, QueryTracesPage as N, TraceAnalysisStore as O, SearchSpanResult as P, TraceAnalystByteBudgets as R, createBoundedTraceAnalysisStore as S, ReadSpanSourceResult as T, TraceAnalystTraceSummary as U, TraceAnalystSpanKind as V, ViewSpansResult as W, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, DatasetOverview as j, TraceAnalysisStoreContext as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, ReadSpanSourceInput as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystFilters as z };
|
|
633
|
-
//# sourceMappingURL=types-
|
|
633
|
+
//# sourceMappingURL=types-DAPpAkd4.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-DAPpAkd4.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/trace-analyst/store-boundary.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;KAsBY,uBAAuB;KAEvB;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC3MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;UAIM;EACf;EACA;EACA;EACA;EACA;EACA;;KAGU;EAEN;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA;;;;KAKI,oBACV,OAAO,qBACP,UAAU,8BACP,QAAQ;;;;;;;;UASI;;EAEf,iBAAiB;EAEjB,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;iBC1GJ,gCACd,QAAQ,oBACR,UAAS,mCACR;;;;;;;UCPc;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;;;cAKA;KAED,0BAA0B;;KAG1B;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
|
@@ -365,7 +365,7 @@ interface LlmCallRequest {
|
|
|
365
365
|
* capped CostLedger to reject the call before execution. Pass
|
|
366
366
|
* `customTokenPricing` when package pricing does not cover the model or endpoint. */
|
|
367
367
|
interface LlmChargeBounds {
|
|
368
|
-
/** Total physical requests, including retries and schema/temperature fallback. Default
|
|
368
|
+
/** Total physical requests, including retries and schema/temperature fallback. Default 6. */
|
|
369
369
|
maximumAttempts?: number;
|
|
370
370
|
/** The transport sends JSON mode instead of a response schema. */
|
|
371
371
|
jsonSchemaTransport?: 'native' | 'json-object';
|
|
@@ -496,6 +496,13 @@ interface LlmClientOptions extends LlmChargeBounds {
|
|
|
496
496
|
idempotencyKey?: string;
|
|
497
497
|
/** Default timeout in ms. Per-call can override. */
|
|
498
498
|
defaultTimeoutMs?: number;
|
|
499
|
+
/**
|
|
500
|
+
* Minimum milliseconds between physical request starts, including retries.
|
|
501
|
+
* Default 0 (disabled). Share one LlmClient per key, or reuse the same options
|
|
502
|
+
* object with callLlm/callLlmJson, to pace concurrent and sequential calls.
|
|
503
|
+
* Independent options objects and processes do not share a quota.
|
|
504
|
+
*/
|
|
505
|
+
minIntervalMs?: number;
|
|
499
506
|
/**
|
|
500
507
|
* Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
|
|
501
508
|
* each attempt's per-attempt timeout controller, so aborting it cancels
|
|
@@ -636,7 +643,7 @@ interface CustomTransportOpts extends BaseTransportOpts {
|
|
|
636
643
|
* `authHeader`. There is no environment fallback: agent-eval never goes
|
|
637
644
|
* looking for a key.
|
|
638
645
|
*/
|
|
639
|
-
interface OpenAiCompatibleTransportOpts extends BaseTransportOpts, Pick<LlmClientOptions, 'assertServedModel' | 'customTokenPricing' | 'deadlineMs' | 'defaultTimeoutMs' | 'fetch' | 'jsonPayloadMode' | 'jsonSchemaTransport' | 'provider' | 'rawSink' | 'signal' | 'thinking'> {
|
|
646
|
+
interface OpenAiCompatibleTransportOpts extends BaseTransportOpts, Pick<LlmClientOptions, 'assertServedModel' | 'customTokenPricing' | 'deadlineMs' | 'defaultTimeoutMs' | 'fetch' | 'jsonPayloadMode' | 'jsonSchemaTransport' | 'minIntervalMs' | 'provider' | 'rawSink' | 'signal' | 'thinking'> {
|
|
640
647
|
transport: 'openai-compatible';
|
|
641
648
|
/** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */
|
|
642
649
|
baseUrl: string;
|
|
@@ -862,4 +869,4 @@ interface CheckResult {
|
|
|
862
869
|
}
|
|
863
870
|
//#endregion
|
|
864
871
|
export { NoopRawProviderSink as $, createChatClient as A, LlmToolCall as B, ChatResponse as C, judgeFamily as Ct, MockTransportOpts as D, CustomTransportOpts as E, LlmChargeBounds as F, costReceiptFromLlmError as G, LlmToolDefinition as H, LlmMessage as I, stripFencedJson as J, isTransientLlmError as K, LlmResponseError as L, LlmCallMetadata as M, LlmCallRequest as N, OpenAiCompatibleTransportOpts as O, LlmCallResult as P, InMemoryRawProviderSinkOptions as Q, LlmThinkingMode as R, ChatRequest as S, assertCrossFamily as St, CreateChatClientOpts as T, LlmUsage as U, LlmToolChoice as V, costReceiptFromLlm as W, FileSystemRawProviderSinkOptions as X, FileSystemRawProviderSink as Y, InMemoryRawProviderSink as Z, ScenarioResult as _, checkServedModel as _t, CompletionCriterion as a, defaultProviderRedactor as at, ChatCallOpts as b, CrossFamilyError as bt, JudgeInput as c, AssertServedModelOptions as ct, PersonaConfig as d, ServedModelCheck as dt, ProviderRedactor as et, ProductClientConfig as f, ServedModelPolicy as ft, ScenarioFile as g, assertServedModels as gt, Scenario as h, assertServedModel as ht, CollectedArtifacts as i, RawProviderSinkFilter as it, LlmCallError as j, SandboxSdkTransportOpts as k, JudgeRubric as l, ModelSubstitutionError as lt, RubricDimension as m, assertCrossFamilyServed as mt, ArtifactResult as n, RawProviderEvent as nt, DriverState as o, providerFromBaseUrl as ot, RouteMap as p, ServedModelVerdict as pt, maximumChargeForLlmRequest as q, CheckResult as r, RawProviderSink as rt, JudgeFn as s, AssertCrossFamilyServedOptions as st, ArtifactCheck as t, RawProviderDirection as tt, JudgeScore as u, ServedCrossFamilyError as ut, Turn as v, servedModelAcceptable as vt, ChatTransport as w, ChatClient as x, JudgeFamily as xt, TurnResult as y, AssertCrossFamilyOptions as yt, LlmTokenLogprob as z };
|
|
865
|
-
//# sourceMappingURL=types-
|
|
872
|
+
//# sourceMappingURL=types-Lub7o6bN.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-Lub7o6bN.d.ts","names":[],"sources":["../src/judge-families.ts","../src/integrity/served-model.ts","../src/trace/raw-provider-sink.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;KAWY;;;;;;;iBAsEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCvFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;iBA4Cc,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;;;;;;;;KAc9B;UAWK;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KCjRS;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;iBAM5C,wBAAwB,OAAO,mBAAmB;UAqBjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;UCnLnB;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;EAGvD,YAAY;;EAEZ;;KAGU;;UAGK;EACf;EACA;IACE;IACA;IACA,YAAY;;;;KAKJ;EAIN;EAAkB;IAAY;;;;UAGnB;EACf;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;;EAErC,QAAQ;;EAER,aAAa;EACb;EACA;;;;;EAKA;IAAa;;;EAEb,WAAW;;EAEX;;;;;;UAOe;;EAEf;;EAEA;;EAEA,WAAW;;EAEX,qBAAqB;;iBAGP,2BACd,SAAS,KACP,0GAGF,UAAS,kBACR;UA+Bc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;;;UAKe;EACf;EACA;EACA,KAAK;IAAgB;IAAe;;;UAGrB;;EAEf;;EAEA,YAAY;EACZ,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;;;;;;;EAQA,WAAW,cAAc;;EAEzB,KAAK;;KAGK,kBAAkB,KAAK;;;;;;iBAOnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC,yBAAyB;;EAExC;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;;;;;EAMA;;EAEA,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBA6FhB,oBAAoB;;;;;;iBA+PpB,gBAAgB;;;;;;UCzsBf;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAMK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,0BACA,sBACA,gCACA;UAEM;EACR;;EAEA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;;;;;;;;;;;;;;;UAqB1C,sCACP,mBACN,KACE;EAcJ;;EAEA;;EAEA;;EAEA;;EAEA;IAAe;IAAc;;;;;;;UAOd,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UChK7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,OAAO;EACP,iBAAiB;EACjB,aAAa;EACb;EACA;EACA;EACA,WAAW;;EAEX,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IACE;MACE;MACA;MACA;;;EAGJ,OAAO;EACP,gBAAgB;;UAKD;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;;;;;;;;;KAWT;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { b as CustomTokenPricing, c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { x as ChatClient } from "../types-
|
|
2
|
+
import { x as ChatClient } from "../types-Lub7o6bN.js";
|
|
3
3
|
import { s as TraceStore } from "../store-C4avFjIi.js";
|
|
4
|
-
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-
|
|
4
|
+
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-C6Z8PRoF.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { ServerType } from "@hono/node-server";
|
|
7
7
|
import { Hono } from "hono";
|
package/dist/wire/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { A as WIRE_VERSION, C as RUBRIC_VERSION_SCHEME, D as TracesIngestRequestSchema, E as TraceEventSchema, O as TracesIngestResponseSchema, S as ListRubricsResponseSchema, T as RubricSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, j as hashRubric, k as VersionResponseSchema, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricDimensionSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-
|
|
1
|
+
import { A as WIRE_VERSION, C as RUBRIC_VERSION_SCHEME, D as TracesIngestRequestSchema, E as TraceEventSchema, O as TracesIngestResponseSchema, S as ListRubricsResponseSchema, T as RubricSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, j as hashRubric, k as VersionResponseSchema, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricDimensionSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-BZOt7eNC.js";
|
|
2
2
|
export { ErrorResponseSchema, FeedbackIngestResponseSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RUBRIC_VERSION_SCHEME, RubricDimensionSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# LLM retry delay and request pacing
|
|
2
|
+
|
|
3
|
+
Fleet item it-fd64863571. Source: tangle-network/ops-board#882.
|
|
4
|
+
|
|
5
|
+
The OpenAI-compatible transport accepts `minIntervalMs`, defaulting to `0`
|
|
6
|
+
(disabled). It spaces **physical request starts**, including transient retries,
|
|
7
|
+
schema negotiation and temperature fallback. Share one client for a key across
|
|
8
|
+
a campaign; the internal `callLlm` and `callLlmJson` functions instead share state
|
|
9
|
+
when passed the same options object. Independent options objects and processes
|
|
10
|
+
do not coordinate quotas. Credentials are not stored in a global key registry.
|
|
11
|
+
|
|
12
|
+
```ts
|
|
13
|
+
const chat = createChatClient({
|
|
14
|
+
transport: 'openai-compatible',
|
|
15
|
+
baseUrl,
|
|
16
|
+
apiKey,
|
|
17
|
+
minIntervalMs: Math.ceil(60_000 / requestsPerMinute),
|
|
18
|
+
})
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Concurrent callers recheck the interval after waking, so overdue waiters do not
|
|
22
|
+
create a catch-up burst. Responses may overlap. Per-call overrides on an internal
|
|
23
|
+
`LlmClient` retain its shared pacing state. Pacing is outside the HTTP-attempt
|
|
24
|
+
timeout, but inside the operation deadline, and cancellation interrupts waits.
|
|
25
|
+
A cancelled waiter does not reserve a future request slot. The raw request
|
|
26
|
+
event is written after the wait, at the claimed start, so a call cancelled or
|
|
27
|
+
expired while queued leaves no raw request or error event.
|
|
28
|
+
|
|
29
|
+
`Retry-After` already took precedence over the capped exponential backoff. The
|
|
30
|
+
regression suite now checks a full 30-second delay followed by a paced call
|
|
31
|
+
through all five raw/JSON/client/public transport paths, plus HTTP-date headers.
|
|
32
|
+
Long delays are split into timer-safe chunks instead of overflowing the timer's
|
|
33
|
+
signed 32-bit limit. Invalid numeric delays fall back to exponential backoff.
|
|
34
|
+
When an explicit operation deadline is shorter than `Retry-After`, the operation
|
|
35
|
+
ends without issuing an early retry.
|
|
36
|
+
|
|
37
|
+
The default `maximumAttempts` is now **6 total physical requests** (was 3), not
|
|
38
|
+
six retries. Explicit `maximumAttempts` and `TANGLE_LLM_MAXIMUM_ATTEMPTS` still
|
|
39
|
+
win. `maximumChargeForLlmRequest` uses the same default, so increased retry
|
|
40
|
+
headroom also increases the conservative cost reservation. No `maxRetries`
|
|
41
|
+
alias or second retry loop is introduced.
|
|
42
|
+
|
|
43
|
+
## Validation
|
|
44
|
+
|
|
45
|
+
Run the repository's existing Vitest suite:
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
pnpm test -- tests/llm-pacing.test.ts tests/llm-physical-attempts.test.ts src/llm-client.test.ts src/analyst/chat-client-openai-compatible.test.ts
|
|
49
|
+
pnpm typecheck
|
|
50
|
+
pnpm exec biome check src/llm-client.ts src/analyst/chat-client.ts tests/llm-pacing.test.ts
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The new tests use a mocked fetch transport and fake timers; no provider calls,
|
|
54
|
+
credentials or real 30-second sleeps are required. They also cover concurrent
|
|
55
|
+
starts, delayed raw sinks, fallback pacing, cancellation, deadlines, independent
|
|
56
|
+
options, disabled/invalid intervals, and the shared retry/cost allowance.
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
# External agent quickstart
|
|
2
|
+
|
|
3
|
+
Run an evaluation and an improvement loop in your own Node.js process.
|
|
4
|
+
No Tangle account, sandbox, runtime, or framework migration is required.
|
|
5
|
+
The first run is an offline wiring exercise; `--openai` explicitly opts into paid model calls.
|
|
6
|
+
|
|
7
|
+
Use `defineAgentEval()` from the [main README](../README.md#quickstart) for the higher-level API.
|
|
8
|
+
This guide exposes `dispatch`, `runCampaign()`, and `runImprovementLoop()` when you want to own the wiring.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
With Node.js 20.19 or newer, start in an empty directory:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
npm init -y
|
|
16
|
+
npm install @tangle-network/agent-eval
|
|
17
|
+
npm install --save-dev tsx
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Run a campaign and an improvement loop
|
|
21
|
+
|
|
22
|
+
Save this entire block as `eval.mts`. It includes a deterministic agent, a rubric,
|
|
23
|
+
a candidate proposer, and a reference adapter for the raw OpenAI-compatible chat API.
|
|
24
|
+
The adapter reuses the package's maintained transport and receipt helpers rather than introducing another SDK or runtime.
|
|
25
|
+
|
|
26
|
+
<!-- external-quickstart:program -->
|
|
27
|
+
```ts
|
|
28
|
+
import { randomUUID } from 'node:crypto'
|
|
29
|
+
import { join, resolve } from 'node:path'
|
|
30
|
+
import {
|
|
31
|
+
costReceiptFromLlm,
|
|
32
|
+
costReceiptFromLlmError,
|
|
33
|
+
maximumChargeForLlmRequest,
|
|
34
|
+
} from '@tangle-network/agent-eval'
|
|
35
|
+
import {
|
|
36
|
+
createChatClient,
|
|
37
|
+
defaultProductionGate,
|
|
38
|
+
type DispatchContext,
|
|
39
|
+
inMemoryCampaignStorage,
|
|
40
|
+
type JudgeConfig,
|
|
41
|
+
type MutableSurface,
|
|
42
|
+
type Scenario,
|
|
43
|
+
type SurfaceProposer,
|
|
44
|
+
} from '@tangle-network/agent-eval/contract'
|
|
45
|
+
import { runCampaign, runImprovementLoop } from '@tangle-network/agent-eval/campaign'
|
|
46
|
+
import { type DeploymentOutcome, InMemoryOutcomeStore } from '@tangle-network/agent-eval/rl'
|
|
47
|
+
|
|
48
|
+
interface SupportCase extends Scenario { ticket: string }
|
|
49
|
+
interface SupportAnswer { text: string }
|
|
50
|
+
|
|
51
|
+
// Replace this function with the SAME agent call your application uses.
|
|
52
|
+
// Its input/output need not know anything about agent-eval.
|
|
53
|
+
async function existingAgent(input: {
|
|
54
|
+
instructions: string
|
|
55
|
+
ticket: string
|
|
56
|
+
signal: AbortSignal
|
|
57
|
+
}) {
|
|
58
|
+
input.signal.throwIfAborted()
|
|
59
|
+
return {
|
|
60
|
+
message: input.instructions.includes('cite the ticket id')
|
|
61
|
+
? `Ticket ${input.ticket}: on it.`
|
|
62
|
+
: 'On it.',
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function requiredEnv(name: string): string {
|
|
67
|
+
const value = process.env[name]?.trim()
|
|
68
|
+
if (!value) throw new Error(`Set ${name} before using --openai`)
|
|
69
|
+
return value
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function tokenRate(name: string): number {
|
|
73
|
+
const value = Number(requiredEnv(name))
|
|
74
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a nonnegative rate`)
|
|
75
|
+
return value
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Reference adapter: raw OpenAI-compatible API, no Tangle execution service.
|
|
79
|
+
function openAiDispatch() {
|
|
80
|
+
const model = requiredEnv('LLM_MODEL')
|
|
81
|
+
const pricing = {
|
|
82
|
+
inputUsdPerMillion: tokenRate('LLM_INPUT_USD_PER_MILLION'),
|
|
83
|
+
outputUsdPerMillion: tokenRate('LLM_OUTPUT_USD_PER_MILLION'),
|
|
84
|
+
}
|
|
85
|
+
const chat = createChatClient({
|
|
86
|
+
transport: 'openai-compatible',
|
|
87
|
+
baseUrl: requiredEnv('LLM_BASE_URL'),
|
|
88
|
+
apiKey: requiredEnv('LLM_API_KEY'),
|
|
89
|
+
defaultModel: model,
|
|
90
|
+
})
|
|
91
|
+
return async (
|
|
92
|
+
surface: MutableSurface,
|
|
93
|
+
scenario: SupportCase,
|
|
94
|
+
ctx: DispatchContext,
|
|
95
|
+
): Promise<SupportAnswer> => {
|
|
96
|
+
const request = {
|
|
97
|
+
model,
|
|
98
|
+
messages: [
|
|
99
|
+
{ role: 'system' as const, content: String(surface) },
|
|
100
|
+
{ role: 'user' as const, content: `Please acknowledge ticket ${scenario.ticket}.` },
|
|
101
|
+
],
|
|
102
|
+
maxTokens: 256,
|
|
103
|
+
}
|
|
104
|
+
const paid = await ctx.cost.runPaidCall({
|
|
105
|
+
actor: 'external-support-agent',
|
|
106
|
+
model,
|
|
107
|
+
maximumCharge: maximumChargeForLlmRequest(request, {
|
|
108
|
+
customTokenPricing: pricing,
|
|
109
|
+
maximumAttempts: chat.maximumAttempts,
|
|
110
|
+
}),
|
|
111
|
+
execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }),
|
|
112
|
+
receipt: (result) => costReceiptFromLlm(result, pricing),
|
|
113
|
+
receiptFromError: (error) => costReceiptFromLlmError(error, pricing),
|
|
114
|
+
})
|
|
115
|
+
if (!paid.succeeded) throw paid.error
|
|
116
|
+
if (!paid.value.content.trim()) throw new Error('The agent returned an empty answer')
|
|
117
|
+
return { text: paid.value.content }
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const useOpenAi = process.argv.includes('--openai')
|
|
122
|
+
const dispatchWithSurface = useOpenAi
|
|
123
|
+
? openAiDispatch()
|
|
124
|
+
: async (surface: MutableSurface, scenario: SupportCase, ctx: DispatchContext) => {
|
|
125
|
+
const result = await existingAgent({
|
|
126
|
+
instructions: String(surface),
|
|
127
|
+
ticket: scenario.ticket,
|
|
128
|
+
signal: ctx.signal,
|
|
129
|
+
})
|
|
130
|
+
return { text: result.message }
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// A deterministic rubric is a judge. Use llmJudge() for semantic criteria.
|
|
134
|
+
const judge: JudgeConfig<SupportAnswer, SupportCase> = {
|
|
135
|
+
name: 'ticket-id',
|
|
136
|
+
judgeVersion: 'ticket-id-v1',
|
|
137
|
+
dimensions: [{ key: 'present', description: 'The answer cites the supplied ticket id' }],
|
|
138
|
+
score: ({ artifact, scenario }) => {
|
|
139
|
+
const present = artifact.text.includes(scenario.ticket) ? 1 : 0
|
|
140
|
+
return { dimensions: { present }, composite: present, notes: '' }
|
|
141
|
+
},
|
|
142
|
+
}
|
|
143
|
+
const scenarios: SupportCase[] = ['refund', 'shipping'].map((ticket) => ({
|
|
144
|
+
id: `train-${ticket}`, kind: 'support', ticket,
|
|
145
|
+
}))
|
|
146
|
+
const holdoutScenarios: SupportCase[] = ['cancel', 'billing'].map((ticket) => ({
|
|
147
|
+
id: `holdout-${ticket}`, kind: 'support', ticket,
|
|
148
|
+
}))
|
|
149
|
+
const baselineSurface = 'Answer politely.'
|
|
150
|
+
const proposer: SurfaceProposer = {
|
|
151
|
+
kind: 'cite-ticket',
|
|
152
|
+
async propose({ currentSurface }) {
|
|
153
|
+
return [{
|
|
154
|
+
surface: `${String(currentSurface)} Always cite the ticket id.`,
|
|
155
|
+
label: 'cite-ticket',
|
|
156
|
+
rationale: 'The rubric requires the supplied ticket id in the answer.',
|
|
157
|
+
}]
|
|
158
|
+
},
|
|
159
|
+
}
|
|
160
|
+
const runRoot = resolve('eval-quickstart', randomUUID())
|
|
161
|
+
const common = {
|
|
162
|
+
judges: [judge],
|
|
163
|
+
storage: inMemoryCampaignStorage(),
|
|
164
|
+
seed: 7,
|
|
165
|
+
reps: 1,
|
|
166
|
+
maxConcurrency: 2,
|
|
167
|
+
dispatchTimeoutMs: 30_000,
|
|
168
|
+
abortOnCellError: true,
|
|
169
|
+
expectUsage: useOpenAi ? 'assert' as const : 'off' as const,
|
|
170
|
+
// Each top-level call has its own $1 ceiling; the loop shares its ceiling
|
|
171
|
+
// across search and holdout. The script has $2 of total ledger budget at the declared rates.
|
|
172
|
+
costCeiling: useOpenAi ? 1 : undefined,
|
|
173
|
+
}
|
|
174
|
+
const baseline = await runCampaign({
|
|
175
|
+
...common,
|
|
176
|
+
scenarios,
|
|
177
|
+
dispatch: async function dispatch(scenario, ctx) {
|
|
178
|
+
return dispatchWithSurface(baselineSurface, scenario, ctx)
|
|
179
|
+
},
|
|
180
|
+
dispatchRef: useOpenAi ? 'external-openai-baseline-v1' : 'external-fixture-baseline-v1',
|
|
181
|
+
runDir: join(runRoot, 'baseline'),
|
|
182
|
+
})
|
|
183
|
+
const improvement = await runImprovementLoop({
|
|
184
|
+
...common,
|
|
185
|
+
scenarios,
|
|
186
|
+
holdoutScenarios,
|
|
187
|
+
baselineSurface,
|
|
188
|
+
dispatchWithSurface,
|
|
189
|
+
dispatchRef: useOpenAi ? 'external-openai-v1' : 'external-fixture-v1',
|
|
190
|
+
proposer,
|
|
191
|
+
populationSize: 1,
|
|
192
|
+
maxGenerations: 1,
|
|
193
|
+
gate: defaultProductionGate<SupportAnswer, SupportCase>({ holdoutScenarios, deltaThreshold: 0 }),
|
|
194
|
+
autoOnPromote: 'none',
|
|
195
|
+
runDir: join(runRoot, 'improvement'),
|
|
196
|
+
})
|
|
197
|
+
|
|
198
|
+
// Synthetic storage example, NOT a measured production outcome or judge score.
|
|
199
|
+
const outcomes = new InMemoryOutcomeStore()
|
|
200
|
+
const observation: DeploymentOutcome = {
|
|
201
|
+
runId: 'synthetic-deployment-run',
|
|
202
|
+
capturedAt: Date.now(),
|
|
203
|
+
metrics: { resolved: 1 },
|
|
204
|
+
source: 'quickstart-fixture',
|
|
205
|
+
}
|
|
206
|
+
await outcomes.append(observation)
|
|
207
|
+
console.log('baseline:', baseline.aggregates.byJudge['ticket-id']?.mean)
|
|
208
|
+
console.log('candidate on holdout:', improvement.winnerOnHoldout.aggregates.byJudge['ticket-id']?.mean)
|
|
209
|
+
console.log('decision:', improvement.gateResult.decision)
|
|
210
|
+
console.log('outcome rows:', (await outcomes.forRun(observation.runId)).length)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
```sh
|
|
214
|
+
npx tsx eval.mts
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
The offline fixture is constructed to score zero for the baseline and one for the candidate.
|
|
218
|
+
It uses only two holdout cases, so a higher score is not sufficient evidence to ship.
|
|
219
|
+
Inspect `improvement.gateResult`, not just the candidate's mean.
|
|
220
|
+
The candidate generator makes one known edit; this demonstrates the loop, not learned optimization or general agent quality.
|
|
221
|
+
`autoOnPromote: 'none'` reports the decision without changing your agent or opening a PR.
|
|
222
|
+
|
|
223
|
+
## Connect a real agent
|
|
224
|
+
|
|
225
|
+
Replace `existingAgent()` with your production SDK call, graph invocation, service, or local model.
|
|
226
|
+
Keep the adaptation in `dispatchWithSurface`: translate the surface and scenario into the agent's input,
|
|
227
|
+
forward cancellation, and return the artifact your judge scores.
|
|
228
|
+
Do not pass hidden answer keys or grading instructions into the agent request.
|
|
229
|
+
Keep execution failures as thrown errors rather than successful empty strings or task-score zeroes.
|
|
230
|
+
|
|
231
|
+
For the included OpenAI-compatible adapter, explicitly configure your endpoint, credentials,
|
|
232
|
+
a currently deployed model id, and its current per-million-token rates. For OpenAI directly,
|
|
233
|
+
the base URL is `https://api.openai.com/v1`. Then opt in:
|
|
234
|
+
|
|
235
|
+
```sh
|
|
236
|
+
# Set LLM_BASE_URL, LLM_API_KEY, LLM_MODEL,
|
|
237
|
+
# LLM_INPUT_USD_PER_MILLION and LLM_OUTPUT_USD_PER_MILLION in your shell.
|
|
238
|
+
npx tsx eval.mts --openai
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
This mode makes paid calls. It does not fall back to the fixture when a provider fails.
|
|
242
|
+
The maintained transport reports model identity, usage, retries, and failure receipts;
|
|
243
|
+
`ctx.cost.runPaidCall()` reserves for bounded output and the declared maximum attempts before dispatch.
|
|
244
|
+
Unknown usage is not invented, and `expectUsage: 'assert'` rejects missing dispatch evidence.
|
|
245
|
+
Set rates for your actual endpoint; local estimates are not provider invoices. Add cache-read/write
|
|
246
|
+
rates to `pricing` when applicable. Increase `maxTokens` and the explicit budget together when your model needs more output headroom.
|
|
247
|
+
|
|
248
|
+
For an existing multi-call agent, meter every paid call at the point where its receipt is available.
|
|
249
|
+
Do not set `expectUsage: 'off'` merely to make an uninstrumented paid agent appear to work.
|
|
250
|
+
For semantic scoring, use `llmJudge()` from `/contract` with a separate `ChatClient`;
|
|
251
|
+
see [judge verdicts](./verdicts.md). Judge failures must remain failures, not low task scores.
|
|
252
|
+
|
|
253
|
+
## Keep the useful boundaries
|
|
254
|
+
|
|
255
|
+
The same production execution function should serve your application and evaluation.
|
|
256
|
+
Candidate generation may use training cases, but final comparisons need separate cases;
|
|
257
|
+
real deployments also need representative data, calibrated judges, sufficient independent observations,
|
|
258
|
+
and an explicit approval policy. The tiny fixture is not release evidence.
|
|
259
|
+
|
|
260
|
+
`inMemoryCampaignStorage()` keeps campaign data in this process, and `InMemoryOutcomeStore` does the same
|
|
261
|
+
for downstream observations. Use `fsCampaignStorage()` and `FileSystemOutcomeStore` from the public barrels
|
|
262
|
+
when evidence must survive a restart. With durable campaign storage, include model, prompt, tools, and
|
|
263
|
+
configuration revisions in `dispatchRef` so different execution behavior cannot reuse a stale cache.
|
|
264
|
+
|
|
265
|
+
For real deployment feedback, use the actual production `RunRecord.runId` in `DeploymentOutcome.runId`
|
|
266
|
+
and record independently observed metrics. Do not relabel an evaluation score as a deployment outcome.
|
|
267
|
+
See [outcome validity](./outcome-validity.md), [campaign controls](./eval-surface-map.md), and
|
|
268
|
+
[evaluation integrity](./evaluation-integrity.md) for the next stage.
|
package/docs/search-ledger.md
CHANGED
|
@@ -477,6 +477,9 @@ The finalists are the nodes decided `finalist`, in decision order; whether the b
|
|
|
477
477
|
It returns `verified`, `mismatch` with the differences, or `unknown` for a claim another rule revision made.
|
|
478
478
|
The projector checks the claim's structure (every finalist named, the Bonferroni confidence of each test, a ship's held-out units and pinned judge) but not its numbers, so a store keeps what `verifySearchClaim` derives, never the producer's claim as written.
|
|
479
479
|
`runSearch` runs it on every close and on every rerun of a closed ledger, throws on `mismatch`, and returns it as `SearchRunResult.claimVerification`.
|
|
480
|
+
`searchClaimDecision(state, nodeId)` returns the `PairedPromotionDecision` the claim made for one tested finalist, made again from the closed ledger: the root as control, the improvement as `delta`, at the finalist's Bonferroni confidence, with the claim's estimator, resamples and seed.
|
|
481
|
+
It throws unless the result reproduces the claim's recorded test for that finalist, and returns null for a node the claim did not test.
|
|
482
|
+
A consumer that promotes the selected finalist cites this decision with the claim, rather than deciding again from the same cells at another confidence or seed.
|
|
480
483
|
A judge change is a changed `search-opened` header, which `SearchRecorder.open` refuses; it starts a derived search instead of mixing verdicts.
|
|
481
484
|
|
|
482
485
|
`compareOptimizationMethods` keeps its own held-out comparison for black-box methods such as GEPA, which return one winner and never see the test split.
|
|
@@ -555,7 +558,7 @@ Those claims need the sealed test split, the claim's power check, and held-out e
|
|
|
555
558
|
- `src/search/lenses/edit-credit.ts`: `editCredit` and `editCreditText`.
|
|
556
559
|
- `src/campaign/search-ledger-recording.ts`: `SearchRecorder` and the surface helpers.
|
|
557
560
|
- `src/campaign/search-kernel.ts`: `runSearch`, the executor, proposer and codec ports, `searchPolicyView` and `searchDivergence`.
|
|
558
|
-
- `src/campaign/search-claim.ts`: `planSearchClaim`, `decideSearchClaim`, `verifySearchClaim` and `searchClaimReserveUsd`.
|
|
561
|
+
- `src/campaign/search-claim.ts`: `planSearchClaim`, `decideSearchClaim`, `verifySearchClaim`, `searchClaimDecision` and `searchClaimReserveUsd`.
|
|
559
562
|
- `src/search/lenses/landscape.ts`, `plateau.ts` and `skill-manifold.ts`: the geometry lenses, the plateau score and the unit extension.
|
|
560
563
|
- `src/campaign/search-policy.ts`: `SearchPolicy`, `incumbent`, `incumbentWithOperatorBandit`, `crowdedFrontierParent`, `aide` and `beam`.
|
|
561
564
|
- `src/campaign/allocation.ts`: `SearchAllocator`, `uniform` and `asha`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.208.2",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chat-client-5LLl4Ctl.js","names":[],"sources":["../src/analyst/chat-client.ts"],"sourcesContent":["/**\n * Provider-neutral chat contract for every model call made by agent-eval.\n *\n * The caller owns model execution. Evaluation code receives canonical requests\n * and results without importing a provider SDK, and agent-eval reads no\n * environment variable to find a provider credential: every transport is bound\n * at the call site.\n *\n * Two ways to bind one. `custom` and `sandbox-sdk` take a `chat` function the\n * caller wrote. `openai-compatible` takes an endpoint and a credential the\n * caller passes as values, and agent-eval issues the HTTP request itself.\n *\n * WHY THE THIRD ONE EXISTS. Every consumer that wanted a plain OpenAI-style\n * endpoint had to hand-write the same fetch loop — retry on 429 and 5xx,\n * `json_schema` degrade, fenced-JSON stripping, and, load-bearing here, the\n * `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed`\n * read. A hand-written transport that omits `servedModel` makes both identity\n * checks read `unreported`, so the two-vendor judge rule they enforce silently\n * measures nothing. Measured motive: agent-eval 0.172.1 exported no such\n * transport, so a two-model judge panel in discovery-lab reached\n * `createChatClient({ transport: 'custom', baseUrl, apiKey })` — fields that\n * exist on no transport option — and every call threw\n * `TypeError: opts.chat is not a function`, which surfaced as\n * `aggregateJudgeVerdicts: all 2 judges failed`.\n *\n * The credential still never comes from agent-eval's environment. `baseUrl`\n * and the bearer are required arguments with no default and no fallback, so a\n * misconfigured caller gets a refusal at construction, never a call to an\n * endpoint it did not name.\n */\n\nimport {\n type LlmCallRequest,\n type LlmCallResult,\n LlmClient,\n type LlmClientOptions,\n} from '../llm-client'\n\n/**\n * Unified chat interface using the package's canonical LLM request and result.\n */\nexport interface ChatClient {\n /** Display name of the bound transport, included in telemetry. */\n readonly transport: ChatTransport\n /** Default model when the caller omits one. */\n readonly defaultModel?: string\n /** Total provider attempts this transport can make for one chat call. */\n readonly maximumAttempts?: number\n\n /** Implementations must enforce `req.maxTokens` when it is present. */\n chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>\n}\n\nexport type ChatTransport =\n | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK\n | 'custom' // caller-adapted SDK or transport\n | 'openai-compatible' // caller-named /v1 endpoint, driven by this package\n | 'mock' // test-time injection\n\nexport interface ChatRequest extends Omit<LlmCallRequest, 'model'> {\n /** Optional — falls back to ChatClient.defaultModel. */\n model?: string\n}\n\nexport type ChatResponse = LlmCallResult\n\nexport interface ChatCallOpts {\n /** Cancel the in-flight request. */\n signal?: AbortSignal\n /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */\n maxCostUsd?: number\n /** Correlation tag carried into request headers when the transport allows. */\n correlationId?: string\n /** Stable provider idempotency key for retries/redrives of one paid call. */\n idempotencyKey?: string\n}\n\n// ── Factory ─────────────────────────────────────────────────────────\n\nexport type CreateChatClientOpts =\n | SandboxSdkTransportOpts\n | CustomTransportOpts\n | OpenAiCompatibleTransportOpts\n | MockTransportOpts\n\ninterface BaseTransportOpts {\n defaultModel?: string\n /** Total provider attempts. Required for opaque transports used in capped runs. */\n maximumAttempts?: number\n}\n\n/**\n * Sandbox-SDK transport. The caller supplies a canonical chat function for an\n * already-configured Sandbox handle, so agent-eval does not import the SDK.\n */\nexport interface SandboxSdkTransportOpts extends BaseTransportOpts {\n transport: 'sandbox-sdk'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */\nexport interface CustomTransportOpts extends BaseTransportOpts {\n transport: 'custom'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and\n * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.\n *\n * Use this instead of hand-rolling a `custom` transport around `fetch`. The\n * result carries `servedModel` — the model id the provider echoed, verbatim —\n * which is the only field `assertServedModel` and `assertCrossFamilyServed`\n * can read to witness a gateway answering from a different model than the one\n * requested.\n *\n * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this\n * package's to append. A `baseUrl` that already carries the path is a\n * construction error, not a request to a doubled URL.\n *\n * Exactly one credential form is required — `apiKey`, `bearer`, or\n * `authHeader`. There is no environment fallback: agent-eval never goes\n * looking for a key.\n */\nexport interface OpenAiCompatibleTransportOpts\n extends BaseTransportOpts,\n Pick<\n LlmClientOptions,\n | 'assertServedModel'\n | 'customTokenPricing'\n | 'deadlineMs'\n | 'defaultTimeoutMs'\n | 'fetch'\n | 'jsonPayloadMode'\n | 'jsonSchemaTransport'\n | 'provider'\n | 'rawSink'\n | 'signal'\n | 'thinking'\n > {\n transport: 'openai-compatible'\n /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */\n baseUrl: string\n /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */\n apiKey?: string\n /** Bearer credential, alternate spelling. */\n bearer?: string\n /** Non-bearer authorization header, for endpoints that want their own scheme. */\n authHeader?: { name: string; value: string }\n}\n\n/**\n * Mock transport for tests. The handler receives the request and returns\n * whatever the test wants. No retries, no JSON-schema degrade.\n */\nexport interface MockTransportOpts extends BaseTransportOpts {\n transport: 'mock'\n handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * Build a ChatClient bound to a specific transport. The returned client\n * is safe to share across analysts in a single registry run.\n */\nexport function createChatClient(opts: CreateChatClientOpts): ChatClient {\n switch (opts.transport) {\n case 'sandbox-sdk':\n return {\n transport: 'sandbox-sdk',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'custom':\n return {\n transport: 'custom',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'openai-compatible':\n return openAiCompatibleClient(opts)\n case 'mock':\n return {\n transport: 'mock',\n defaultModel: opts.defaultModel,\n maximumAttempts: 1,\n chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts),\n }\n }\n}\n\nfunction resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {\n if (req.model) return req\n if (!defaultModel) {\n throw new Error(\n 'ChatClient.chat: no model on request and no defaultModel on the client. ' +\n 'Either pass req.model or bind defaultModel at createChatClient().',\n )\n }\n return { ...req, model: defaultModel }\n}\n\n/**\n * Build the OpenAI-compatible client, refusing a half-configured one first.\n *\n * Both refusals are construction-time on purpose. A missing credential or a\n * doubled path is a caller mistake that a request would turn into a 401 or a\n * 404 from an endpoint the caller did not intend to reach — the failure then\n * reads as a provider problem instead of a configuration one.\n */\nfunction openAiCompatibleClient(opts: OpenAiCompatibleTransportOpts): ChatClient {\n const baseUrl = opts.baseUrl?.trim().replace(/\\/+$/, '') ?? ''\n if (!baseUrl) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.\",\n )\n }\n if (/\\/chat\\/completions$/.test(baseUrl)) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; \" +\n `this package appends /chat/completions. Got ${baseUrl}`,\n )\n }\n if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader \" +\n 'is required. agent-eval reads no environment variable for a credential.',\n )\n }\n\n const clientOptions: LlmClientOptions = {\n baseUrl,\n ...(opts.apiKey ? { apiKey: opts.apiKey } : {}),\n ...(opts.bearer ? { bearer: opts.bearer } : {}),\n ...(opts.authHeader ? { authHeader: opts.authHeader } : {}),\n ...(opts.maximumAttempts === undefined ? {} : { maximumAttempts: opts.maximumAttempts }),\n ...(opts.assertServedModel === undefined ? {} : { assertServedModel: opts.assertServedModel }),\n ...(opts.customTokenPricing === undefined\n ? {}\n : { customTokenPricing: opts.customTokenPricing }),\n ...(opts.deadlineMs === undefined ? {} : { deadlineMs: opts.deadlineMs }),\n ...(opts.defaultTimeoutMs === undefined ? {} : { defaultTimeoutMs: opts.defaultTimeoutMs }),\n ...(opts.fetch === undefined ? {} : { fetch: opts.fetch }),\n ...(opts.jsonPayloadMode === undefined ? {} : { jsonPayloadMode: opts.jsonPayloadMode }),\n ...(opts.jsonSchemaTransport === undefined\n ? {}\n : { jsonSchemaTransport: opts.jsonSchemaTransport }),\n ...(opts.provider === undefined ? {} : { provider: opts.provider }),\n ...(opts.rawSink === undefined ? {} : { rawSink: opts.rawSink }),\n ...(opts.signal === undefined ? {} : { signal: opts.signal }),\n ...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),\n }\n const client = new LlmClient(clientOptions)\n\n return {\n transport: 'openai-compatible',\n ...(opts.defaultModel ? { defaultModel: opts.defaultModel } : {}),\n maximumAttempts: client.maximumAttempts,\n chat: async (req, callOpts) => {\n const resolved = resolveModel(req, opts.defaultModel)\n return client.call(resolved as LlmCallRequest, {\n ...(callOpts?.signal ? { signal: callOpts.signal } : {}),\n ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}),\n })\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoKA,SAAgB,iBAAiB,MAAwC;CACvE,QAAQ,KAAK,WAAb;EACE,KAAK,eACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,UACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,qBACH,OAAO,uBAAuB,IAAI;EACpC,KAAK,QACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB;GACjB,MAAM,OAAO,KAAK,aAAa,KAAK,QAAQ,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EAC5F;CACJ;AACF;AAEA,SAAS,aAAa,KAAkB,cAA+C;CACrF,IAAI,IAAI,OAAO,OAAO;CACtB,IAAI,CAAC,cACH,MAAM,IAAI,MACR,2IAEF;CAEF,OAAO;EAAE,GAAG;EAAK,OAAO;CAAa;AACvC;;;;;;;;;AAUA,SAAS,uBAAuB,MAAiD;CAC/E,MAAM,UAAU,KAAK,SAAS,KAAK,CAAC,CAAC,QAAQ,QAAQ,EAAE,KAAK;CAC5D,IAAI,CAAC,SACH,MAAM,IAAI,MACR,+FACF;CAEF,IAAI,uBAAuB,KAAK,OAAO,GACrC,MAAM,IAAI,MACR,qIACiD,SACnD;CAEF,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,YACxD,MAAM,IAAI,MACR,oKAEF;CAGF,MAAM,gBAAkC;EACtC;EACA,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,aAAa,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;EACzD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,sBAAsB,KAAA,IAAY,CAAC,IAAI,EAAE,mBAAmB,KAAK,kBAAkB;EAC5F,GAAI,KAAK,uBAAuB,KAAA,IAC5B,CAAC,IACD,EAAE,oBAAoB,KAAK,mBAAmB;EAClD,GAAI,KAAK,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,YAAY,KAAK,WAAW;EACvE,GAAI,KAAK,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,kBAAkB,KAAK,iBAAiB;EACzF,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,wBAAwB,KAAA,IAC7B,CAAC,IACD,EAAE,qBAAqB,KAAK,oBAAoB;EACpD,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;EAC3D,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;CACnE;CACA,MAAM,SAAS,IAAI,UAAU,aAAa;CAE1C,OAAO;EACL,WAAW;EACX,GAAI,KAAK,eAAe,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;EAC/D,iBAAiB,OAAO;EACxB,MAAM,OAAO,KAAK,aAAa;GAC7B,MAAM,WAAW,aAAa,KAAK,KAAK,YAAY;GACpD,OAAO,OAAO,KAAK,UAA4B;IAC7C,GAAI,UAAU,SAAS,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;IACtD,GAAI,UAAU,iBAAiB,EAAE,gBAAgB,SAAS,eAAe,IAAI,CAAC;GAChF,CAAC;EACH;CACF;AACF"}
|