@tangle-network/agent-eval 0.207.3 → 0.208.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/{agent-profile-EpO0HURy.d.ts → agent-profile-CEhxR_I3.d.ts} +4 -4
  4. package/dist/{agent-profile-EpO0HURy.d.ts.map → agent-profile-CEhxR_I3.d.ts.map} +1 -1
  5. package/dist/analyst/index.d.ts +12 -12
  6. package/dist/analyst/index.js +5 -5
  7. package/dist/{analyze-runs-Q74o4Hfj.d.ts → analyze-runs-CTJyUplR.d.ts} +2 -2
  8. package/dist/{analyze-runs-Q74o4Hfj.d.ts.map → analyze-runs-CTJyUplR.d.ts.map} +1 -1
  9. package/dist/{benchmark-command-DtP94cOB.js → benchmark-command-BC265T1r.js} +6 -6
  10. package/dist/{benchmark-command-DtP94cOB.js.map → benchmark-command-BC265T1r.js.map} +1 -1
  11. package/dist/benchmarks/index.d.ts +4 -4
  12. package/dist/benchmarks/index.js +3 -3
  13. package/dist/campaign/index.d.ts +7 -7
  14. package/dist/campaign/index.js +7 -7
  15. package/dist/{campaign-BL591dWw.js → campaign-C5pjR-Qn.js} +8 -8
  16. package/dist/{campaign-BL591dWw.js.map → campaign-C5pjR-Qn.js.map} +1 -1
  17. package/dist/{chat-client-5LLl4Ctl.js → chat-client-CydfmLdO.js} +3 -2
  18. package/dist/chat-client-CydfmLdO.js.map +1 -0
  19. package/dist/{chat-json-call-Digk-pfF.js → chat-json-call-BJJdJ2uj.js} +2 -2
  20. package/dist/{chat-json-call-Digk-pfF.js.map → chat-json-call-BJJdJ2uj.js.map} +1 -1
  21. package/dist/cli.js +6 -6
  22. package/dist/contract/index.d.ts +10 -10
  23. package/dist/contract/index.js +6 -6
  24. package/dist/{default-registry-BthquIoC.js → default-registry-DX5R7_n9.js} +2 -2
  25. package/dist/{default-registry-BthquIoC.js.map → default-registry-DX5R7_n9.js.map} +1 -1
  26. package/dist/{default-registry-BHEfZHQd.d.ts → default-registry-DZFWY2_E.d.ts} +6 -6
  27. package/dist/{default-registry-BHEfZHQd.d.ts.map → default-registry-DZFWY2_E.d.ts.map} +1 -1
  28. package/dist/{define-agent-eval-CmItPfIq.js → define-agent-eval-BjKXcDQp.js} +3 -3
  29. package/dist/{define-agent-eval-CmItPfIq.js.map → define-agent-eval-BjKXcDQp.js.map} +1 -1
  30. package/dist/{define-agent-eval-B2YV-RDE.d.ts → define-agent-eval-C8zBSbsc.d.ts} +7 -7
  31. package/dist/{define-agent-eval-B2YV-RDE.d.ts.map → define-agent-eval-C8zBSbsc.d.ts.map} +1 -1
  32. package/dist/diagnosis/index.d.ts +2 -2
  33. package/dist/{dspy-rlm-engine-DCQKNQMn.js → dspy-rlm-engine-sHwdSxK3.js} +2 -2
  34. package/dist/{dspy-rlm-engine-DCQKNQMn.js.map → dspy-rlm-engine-sHwdSxK3.js.map} +1 -1
  35. package/dist/{engine-F_PvwesR.d.ts → engine-DProXJ7A.d.ts} +4 -4
  36. package/dist/{engine-F_PvwesR.d.ts.map → engine-DProXJ7A.d.ts.map} +1 -1
  37. package/dist/evaluation.d.ts +2 -2
  38. package/dist/{exact-types-BBMMYk3g.d.ts → exact-types-Cq6xXNVR.d.ts} +2 -2
  39. package/dist/{exact-types-BBMMYk3g.d.ts.map → exact-types-Cq6xXNVR.d.ts.map} +1 -1
  40. package/dist/experiment/index.d.ts +1 -1
  41. package/dist/{external-optimizer-process-CIV4EqiT.js → external-optimizer-process-D2Txd-7D.js} +2 -2
  42. package/dist/{external-optimizer-process-CIV4EqiT.js.map → external-optimizer-process-D2Txd-7D.js.map} +1 -1
  43. package/dist/{external-optimizer-subprocess-BmKtH3T6.js → external-optimizer-subprocess-C0EsxEbI.js} +12 -4
  44. package/dist/{external-optimizer-subprocess-BmKtH3T6.js.map → external-optimizer-subprocess-C0EsxEbI.js.map} +1 -1
  45. package/dist/{feedback-trajectory-C6NJxXEa.d.ts → feedback-trajectory-C6Z8PRoF.d.ts} +3 -3
  46. package/dist/{feedback-trajectory-C6NJxXEa.d.ts.map → feedback-trajectory-C6Z8PRoF.d.ts.map} +1 -1
  47. package/dist/{index-CojsA_EE.d.ts → index-Bzto04CY.d.ts} +7 -7
  48. package/dist/{index-CojsA_EE.d.ts.map → index-Bzto04CY.d.ts.map} +1 -1
  49. package/dist/{index-DzYX3WBL.d.ts → index-DkucR0KJ.d.ts} +7 -7
  50. package/dist/{index-DzYX3WBL.d.ts.map → index-DkucR0KJ.d.ts.map} +1 -1
  51. package/dist/{index-DVDiJIhi.d.ts → index-nwNoS582.d.ts} +8 -8
  52. package/dist/{index-DVDiJIhi.d.ts.map → index-nwNoS582.d.ts.map} +1 -1
  53. package/dist/index.d.ts +19 -19
  54. package/dist/index.js +10 -10
  55. package/dist/{integrity-BWO6RPxv.d.ts → integrity-zloavsM_.d.ts} +2 -2
  56. package/dist/{integrity-BWO6RPxv.d.ts.map → integrity-zloavsM_.d.ts.map} +1 -1
  57. package/dist/{llm-client-DmCNzrc9.js → llm-client-v968tWw8.js} +76 -29
  58. package/dist/llm-client-v968tWw8.js.map +1 -0
  59. package/dist/{llm-judge-Fc3UIcPq.js → llm-judge-HLWASwUJ.js} +5 -5
  60. package/dist/{llm-judge-Fc3UIcPq.js.map → llm-judge-HLWASwUJ.js.map} +1 -1
  61. package/dist/{matrix-W1lbSx72.d.ts → matrix-Cqkk7eYI.d.ts} +2 -2
  62. package/dist/{matrix-W1lbSx72.d.ts.map → matrix-Cqkk7eYI.d.ts.map} +1 -1
  63. package/dist/meta-eval/index.d.ts +1 -1
  64. package/dist/multishot/golden/index.d.ts +1 -1
  65. package/dist/multishot/index.d.ts +2 -2
  66. package/dist/openapi.json +1 -1
  67. package/dist/{prime-protocol-IoN4Qb6c.d.ts → prime-protocol-Csr9kcOM.d.ts} +2 -2
  68. package/dist/{prime-protocol-IoN4Qb6c.d.ts.map → prime-protocol-Csr9kcOM.d.ts.map} +1 -1
  69. package/dist/{produced-state-CywjstcW.js → produced-state-Bzlh09oj.js} +3 -3
  70. package/dist/{produced-state-CywjstcW.js.map → produced-state-Bzlh09oj.js.map} +1 -1
  71. package/dist/{registry-CT-hMSkR.d.ts → registry-DMkmhBQ5.d.ts} +4 -4
  72. package/dist/{registry-CT-hMSkR.d.ts.map → registry-DMkmhBQ5.d.ts.map} +1 -1
  73. package/dist/{researcher-zTpxNJBy.d.ts → researcher-CZFZjj-u.d.ts} +3 -3
  74. package/dist/{researcher-zTpxNJBy.d.ts.map → researcher-CZFZjj-u.d.ts.map} +1 -1
  75. package/dist/rl.d.ts +4 -4
  76. package/dist/search/index.d.ts +2 -2
  77. package/dist/search/index.js +2 -2
  78. package/dist/{search-kernel-mzeNuzr8.js → search-kernel-CIie6xig.js} +44 -2
  79. package/dist/search-kernel-CIie6xig.js.map +1 -0
  80. package/dist/{search-kernel-CZ8KJQRw.d.ts → search-kernel-uKdPVRAe.d.ts} +18 -3
  81. package/dist/search-kernel-uKdPVRAe.d.ts.map +1 -0
  82. package/dist/{search-summary-CDURlgZR.js → search-summary-BV5uehLd.js} +2 -2
  83. package/dist/{search-summary-CDURlgZR.js.map → search-summary-BV5uehLd.js.map} +1 -1
  84. package/dist/{semantic-concept-judge-BbiYuv7U.js → semantic-concept-judge-D1wH-YKc.js} +4 -4
  85. package/dist/{semantic-concept-judge-BbiYuv7U.js.map → semantic-concept-judge-D1wH-YKc.js.map} +1 -1
  86. package/dist/{series-convergence-DLy0bQMV.d.ts → series-convergence-hEfKdoi-.d.ts} +2 -2
  87. package/dist/{series-convergence-DLy0bQMV.d.ts.map → series-convergence-hEfKdoi-.d.ts.map} +1 -1
  88. package/dist/{server-CoBZO1fj.js → server-BZOt7eNC.js} +2 -2
  89. package/dist/{server-CoBZO1fj.js.map → server-BZOt7eNC.js.map} +1 -1
  90. package/dist/{skillopt-optimization-method-BSgM_cbI.js → skillopt-optimization-method-Dcnh5Pvr.js} +5 -5
  91. package/dist/{skillopt-optimization-method-BSgM_cbI.js.map → skillopt-optimization-method-Dcnh5Pvr.js.map} +1 -1
  92. package/dist/{statistical-heldout-COfgpWah.d.ts → statistical-heldout-DFKtp-Qm.d.ts} +2 -2
  93. package/dist/{statistical-heldout-COfgpWah.d.ts.map → statistical-heldout-DFKtp-Qm.d.ts.map} +1 -1
  94. package/dist/{store-tool-spans-BYIarVrG.d.ts → store-tool-spans-9YJNMP-z.d.ts} +5 -5
  95. package/dist/{store-tool-spans-BYIarVrG.d.ts.map → store-tool-spans-9YJNMP-z.d.ts.map} +1 -1
  96. package/dist/{systemone-review-DGTKkLMG.d.ts → systemone-review-Bt4JcxMc.d.ts} +2 -2
  97. package/dist/{systemone-review-DGTKkLMG.d.ts.map → systemone-review-Bt4JcxMc.d.ts.map} +1 -1
  98. package/dist/systemone.d.ts +3 -3
  99. package/dist/{tool-groups-B7_jUPHp.d.ts → tool-groups-j19pmOfP.d.ts} +3 -3
  100. package/dist/tool-groups-j19pmOfP.d.ts.map +1 -0
  101. package/dist/{trace-questions-Dd3AMf-U.d.ts → trace-questions-dRvmsbTk.d.ts} +3 -3
  102. package/dist/{trace-questions-Dd3AMf-U.d.ts.map → trace-questions-dRvmsbTk.d.ts.map} +1 -1
  103. package/dist/trace-repair/index.d.ts +1 -1
  104. package/dist/traces.d.ts +5 -5
  105. package/dist/{tree-CN8mlzUO.js → tree-CJOXbVcv.js} +2 -2
  106. package/dist/{tree-CN8mlzUO.js.map → tree-CJOXbVcv.js.map} +1 -1
  107. package/dist/{types-DZQpUJse.d.ts → types-BlSBUNRd.d.ts} +2 -2
  108. package/dist/{types-DZQpUJse.d.ts.map → types-BlSBUNRd.d.ts.map} +1 -1
  109. package/dist/{types-ZUy5HGY5.d.ts → types-CFCUZk0B.d.ts} +3 -3
  110. package/dist/{types-ZUy5HGY5.d.ts.map → types-CFCUZk0B.d.ts.map} +1 -1
  111. package/dist/{types-6VEehmEg.d.ts → types-DAPpAkd4.d.ts} +2 -2
  112. package/dist/{types-6VEehmEg.d.ts.map → types-DAPpAkd4.d.ts.map} +1 -1
  113. package/dist/{types-DLxe96rp.d.ts → types-Lub7o6bN.d.ts} +10 -3
  114. package/dist/{types-DLxe96rp.d.ts.map → types-Lub7o6bN.d.ts.map} +1 -1
  115. package/dist/wire/index.d.ts +2 -2
  116. package/dist/wire/index.js +1 -1
  117. package/docs/changes/llm-call-pacing.md +56 -0
  118. package/docs/quickstart-external.md +268 -0
  119. package/docs/search-ledger.md +4 -1
  120. package/package.json +1 -1
  121. package/dist/chat-client-5LLl4Ctl.js.map +0 -1
  122. package/dist/llm-client-DmCNzrc9.js.map +0 -1
  123. package/dist/search-kernel-CZ8KJQRw.d.ts.map +0 -1
  124. package/dist/search-kernel-mzeNuzr8.js.map +0 -1
  125. package/dist/tool-groups-B7_jUPHp.d.ts.map +0 -1
@@ -1,5 +1,5 @@
1
1
  import { c as RunSplitTag } from "./run-record-DF-fuu31.js";
2
- import { R as Scenario, d as DispatchContext } from "./types-ZUy5HGY5.js";
2
+ import { R as Scenario, d as DispatchContext } from "./types-CFCUZk0B.js";
3
3
  //#region src/benchmarks/types.d.ts
4
4
  type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
5
5
  type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
@@ -99,4 +99,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
99
99
  declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
100
100
  //#endregion
101
101
  export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
102
- //# sourceMappingURL=types-DZQpUJse.d.ts.map
102
+ //# sourceMappingURL=types-BlSBUNRd.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-DZQpUJse.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
1
+ {"version":3,"file":"types-BlSBUNRd.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
@@ -1,7 +1,7 @@
1
1
  import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
2
2
  import { d as RunTokenUsage, s as RunSearchCoordinates } from "./run-record-DF-fuu31.js";
3
- import { _ as ProposalFinding } from "./types-6VEehmEg.js";
4
- import { M as LlmCallMetadata } from "./types-DLxe96rp.js";
3
+ import { _ as ProposalFinding } from "./types-DAPpAkd4.js";
4
+ import { M as LlmCallMetadata } from "./types-Lub7o6bN.js";
5
5
  import { t as SeriesDistribution } from "./descriptive-B2iPaT9J.js";
6
6
  import { A as SearchLedgerHash, G as SearchSplit, I as SearchNodeStatus, L as SearchOpenedEvent, N as SearchNodeDecision, O as SearchLedgerEntry, P as SearchNodeRef, Q as SearchTaskOutcome, R as SearchOperationKind, U as SearchReservation, Z as SearchTask, b as SearchClosedEvent, d as SearchCellCancelledEvent, i as SearchArtifactRef, l as SearchCandidateSurface, o as SearchAudit, p as SearchCellStage, t as NodeEstimate, w as SearchEdgeRecordedEvent, z as SearchOperationRecordedEvent } from "./search-ledger-types-p2xKfiDJ.js";
7
7
  import { p as LedgerProjector } from "./index-Bj96W0jZ.js";
@@ -1015,4 +1015,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
1015
1015
  }
1016
1016
  //#endregion
1017
1017
  export { SearchNode as $, LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, renderSearchSummary as J, SearchProposerView as K, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, SearchDecisionRecord as Q, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, SearchCell as X, searchProposerView as Y, SearchCompletion as Z, GateDecision as _, CampaignResult as a, SearchUnitScore as at, GenerationRecord as b, CampaignTraceWriter as c, searchTaskSetDigest as ct, DispatchContext as d, SearchOperation as et, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, SearchStateView as it, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, searchUnitScores as lt, GateCheckStatus as m, CampaignArtifactWriter as n, SearchSpend as nt, CampaignScenarioIdentity as o, searchCellId as ot, Gate as p, SearchSummaryOptions as q, CampaignCellResult as r, SearchState as rt, CampaignTokenUsage as s, searchNodeId as st, CampaignAggregates as t, SearchScoredCell as tt, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
1018
- //# sourceMappingURL=types-ZUy5HGY5.d.ts.map
1018
+ //# sourceMappingURL=types-CFCUZk0B.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-ZUy5HGY5.d.ts","names":[],"sources":["../src/campaign/search-state.ts","../src/campaign/search-summary.ts","../src/campaign/types.ts"],"mappings":";;;;;;;;;;iBAwDgB,aAAa,kBAAkB;;iBAK/B,aACd,kBACA,gBACA,gBACA,OAAO,aACP;;iBAMc,oBAAoB,gBAAgB,eAAe;UAkBlD;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;;EAEA;EACA;;;;;;;;iBASc,iBAAiB,gBAAgB,qBAAqB;UAoBrD;EACf;;EAEA;EACA,gBAAgB;EAChB,UAAU;EACV,mBAAmB;;;EAGnB;;EAEA;;;;EAIA,kBAAkB;EAClB;EACA;EACA,QAAQ;EACR;;EAEA;EACA,oBAAoB;EACpB;EACA,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,aAAa;EACb;EACA;EACA,SAAS;;EAET;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,aAAa;EACb;;EAEA,SAAS;;;EAGT,WAAW;;;EAGX,oBAAoB;;EAEpB;;EAEA;;UAGe;EACf;EACA;;;;;;;cA6BW,uBAAuB,gBAAgB,mBAAmB;WAC5D;UACD;UACA;UACA;UACA;UACA;UACA;mBACS;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;UACT;UACA;;mBAES;;UAET;mBACS;EAEjB,YAAY;EAgCZ,MAAM,OAAO,mBAAmB;EA6BhC,YAAY;;EAKZ,KAAK;UAIG;UAgCA;UA6CA;UAsBA;UA8BA;UA8CA;UAsHA;UAmEA;UA4EA;UAoBA;UAmDA;UAaA;;;UA2EA;;EASR;;;UAmBQ;;;UAkBA;;;;;;;;;;;UAoBA;;;EAsBR;IAAgB;IAA4B;;;EAa5C,cAAc;;EAKd,cAAc;;EAKd;IAAc;IAAkB,WAAW;;;EAK3C;;EAKA,aAAa;;EAcb,SAAS,iBAAiB;;EAM1B;;EAKA,YAAY;;EAKZ,oBAAoB;;EAKpB,SAAS,iBAAiB;;EAK1B;;EAKA,SAAS,iBAAiB;;EAM1B,YAAY;;EAMZ,cAAc,sBAAsB;;EAMpC,gBAAgB,gBAAgB,OAAO,cAAc;;;;;;;cAiB1C;WACF;WACA,QAAQ;WACR,QAAQ;WACR;IAAQ;IAAkB,WAAW;;;WAErC;WACA,OAAO;WACP,YAAY;;;;;;;WAOZ;IAAU;IAA4B;;mBAC9B;mBACA;EAEjB,YAAY,OAAO,aAAa;;MAgB5B;EAIJ,KAAK,iBAAiB;EAItB,QAAQ;;EAKR;;EAKA,SAAS;EAKT,gBAAgB;EAIhB,KAAK,iBAAiB;;EAKtB,SAAS;EAKT,KAAK,iBAAiB;;EAKtB,MAAM;IAAU;MAAyB;EAKzC,UAAU,sBAAsB;;;EAMhC,YAAY,gBAAgB,OAAO,cAAc;;EAKjD,WAAW,gBAAgB,OAAO,cAAc;;EAKhD,QAAQ,aAAa,uBAAuB;UAmCpC;;;;;;UCluCO;WACN;WACA;;;EAGT,YAAY,0BAA0B;;EAEtC,WAAW,0BAA0B;;;iBAIvB,mBAAmB,OAAO,kBAAkB;UAW3C;;;;;;;;EAQf,OAAO;;EAEP;;;;;;;;;;;iBAYc,oBAAoB,OAAO,iBAAiB,SAAS;;;;;UCtCpD;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;;WAIjC,gBAAgB;;;;;WAKhB;;;;WAIA,UAAU;IAAgB;IAAgB,UAAU;;;;WAGpD,QAAQ;;;;WAIR;;;WAGA,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;;UAMX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;;EAEA;IAAY;IAAsB;;EAClC;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;;;EAIA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
1
+ {"version":3,"file":"types-CFCUZk0B.d.ts","names":[],"sources":["../src/campaign/search-state.ts","../src/campaign/search-summary.ts","../src/campaign/types.ts"],"mappings":";;;;;;;;;;iBAwDgB,aAAa,kBAAkB;;iBAK/B,aACd,kBACA,gBACA,gBACA,OAAO,aACP;;iBAMc,oBAAoB,gBAAgB,eAAe;UAkBlD;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;;EAEA;EACA;;;;;;;;iBASc,iBAAiB,gBAAgB,qBAAqB;UAoBrD;EACf;;EAEA;EACA,gBAAgB;EAChB,UAAU;EACV,mBAAmB;;;EAGnB;;EAEA;;;;EAIA,kBAAkB;EAClB;EACA;EACA,QAAQ;EACR;;EAEA;EACA,oBAAoB;EACpB;EACA,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,aAAa;EACb;EACA;EACA,SAAS;;EAET;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,aAAa;EACb;;EAEA,SAAS;;;EAGT,WAAW;;;EAGX,oBAAoB;;EAEpB;;EAEA;;UAGe;EACf;EACA;;;;;;;cA6BW,uBAAuB,gBAAgB,mBAAmB;WAC5D;UACD;UACA;UACA;UACA;UACA;UACA;mBACS;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;UACT;UACA;;mBAES;;UAET;mBACS;EAEjB,YAAY;EAgCZ,MAAM,OAAO,mBAAmB;EA6BhC,YAAY;;EAKZ,KAAK;UAIG;UAgCA;UA6CA;UAsBA;UA8BA;UA8CA;UAsHA;UAmEA;UA4EA;UAoBA;UAmDA;UAaA;;;UA2EA;;EASR;;;UAmBQ;;;UAkBA;;;;;;;;;;;UAoBA;;;EAsBR;IAAgB;IAA4B;;;EAa5C,cAAc;;EAKd,cAAc;;EAKd;IAAc;IAAkB,WAAW;;;EAK3C;;EAKA,aAAa;;EAcb,SAAS,iBAAiB;;EAM1B;;EAKA,YAAY;;EAKZ,oBAAoB;;EAKpB,SAAS,iBAAiB;;EAK1B;;EAKA,SAAS,iBAAiB;;EAM1B,YAAY;;EAMZ,cAAc,sBAAsB;;EAMpC,gBAAgB,gBAAgB,OAAO,cAAc;;;;;;;cAiB1C;WACF;WACA,QAAQ;WACR,QAAQ;WACR;IAAQ;IAAkB,WAAW;;;WAErC;WACA,OAAO;WACP,YAAY;;;;;;;WAOZ;IAAU;IAA4B;;mBAC9B;mBACA;EAEjB,YAAY,OAAO,aAAa;;MAgB5B;EAIJ,KAAK,iBAAiB;EAItB,QAAQ;;EAKR;;EAKA,SAAS;EAKT,gBAAgB;EAIhB,KAAK,iBAAiB;;EAKtB,SAAS;EAKT,KAAK,iBAAiB;;EAKtB,MAAM;IAAU;MAAyB;EAKzC,UAAU,sBAAsB;;;EAMhC,YAAY,gBAAgB,OAAO,cAAc;;EAKjD,WAAW,gBAAgB,OAAO,cAAc;;EAKhD,QAAQ,aAAa,uBAAuB;UAmCpC;;;;;;UCluCO;WACN;WACA;;;EAGT,YAAY,0BAA0B;;EAEtC,WAAW,0BAA0B;;;iBAIvB,mBAAmB,OAAO,kBAAkB;UAW3C;;;;;;;;EAQf,OAAO;;EAEP;;;;;;;;;;;iBAYc,oBAAoB,OAAO,iBAAiB,SAAS;;;;;UCtCpD;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;;WAIjC,gBAAgB;;;;;WAKhB;;;;WAIA,UAAU;IAAgB;IAAgB,UAAU;;;;WAGpD,QAAQ;;;;WAIR;;;WAGA,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;;UAMX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;;EAEA;IAAY;IAAsB;;EAClC;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;;;EAIA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
@@ -1,6 +1,6 @@
1
1
  import { c as CostLedgerHandle, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
2
2
  import { a as RunRecord, d as RunTokenUsage } from "./run-record-DF-fuu31.js";
3
- import { c as JudgeInput, x as ChatClient } from "./types-DLxe96rp.js";
3
+ import { c as JudgeInput, x as ChatClient } from "./types-Lub7o6bN.js";
4
4
  import { SpanKind } from "@tangle-network/agent-trace-contract";
5
5
  import { RE2JS } from "re2js";
6
6
  //#region src/trace-analyst/types.d.ts
@@ -630,4 +630,4 @@ type AnalystRunEvent = {
630
630
  };
631
631
  //#endregion
632
632
  export { DEFAULT_TRACE_ANALYST_BUDGETS as A, TraceAnalystSpan as B, BoundedTraceAnalysisStoreOptions as C, TRACE_ANALYSIS_LIMITS as D, SpanSourceReader as E, SearchTraceResult as F, ViewTraceOversized as G, TraceAnalystSpanStatus as H, SpanMatchRecord as I, ViewTraceResult as K, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as L, ErrorCluster as M, QueryTracesPage as N, TraceAnalysisStore as O, SearchSpanResult as P, TraceAnalystByteBudgets as R, createBoundedTraceAnalysisStore as S, ReadSpanSourceResult as T, TraceAnalystTraceSummary as U, TraceAnalystSpanKind as V, ViewSpansResult as W, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, DatasetOverview as j, TraceAnalysisStoreContext as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, ReadSpanSourceInput as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystFilters as z };
633
- //# sourceMappingURL=types-6VEehmEg.d.ts.map
633
+ //# sourceMappingURL=types-DAPpAkd4.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-6VEehmEg.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/trace-analyst/store-boundary.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;KAsBY,uBAAuB;KAEvB;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC3MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;UAIM;EACf;EACA;EACA;EACA;EACA;EACA;;KAGU;EAEN;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA;;;;KAKI,oBACV,OAAO,qBACP,UAAU,8BACP,QAAQ;;;;;;;;UASI;;EAEf,iBAAiB;EAEjB,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;iBC1GJ,gCACd,QAAQ,oBACR,UAAS,mCACR;;;;;;;UCPc;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;;;cAKA;KAED,0BAA0B;;KAG1B;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
1
+ {"version":3,"file":"types-DAPpAkd4.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/trace-analyst/store-boundary.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;KAsBY,uBAAuB;KAEvB;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC3MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;UAIM;EACf;EACA;EACA;EACA;EACA;EACA;;KAGU;EAEN;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA;;;;KAKI,oBACV,OAAO,qBACP,UAAU,8BACP,QAAQ;;;;;;;;UASI;;EAEf,iBAAiB;EAEjB,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;iBC1GJ,gCACd,QAAQ,oBACR,UAAS,mCACR;;;;;;;UCPc;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;;;cAKA;KAED,0BAA0B;;KAG1B;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
@@ -365,7 +365,7 @@ interface LlmCallRequest {
365
365
  * capped CostLedger to reject the call before execution. Pass
366
366
  * `customTokenPricing` when package pricing does not cover the model or endpoint. */
367
367
  interface LlmChargeBounds {
368
- /** Total physical requests, including retries and schema/temperature fallback. Default 3. */
368
+ /** Total physical requests, including retries and schema/temperature fallback. Default 6. */
369
369
  maximumAttempts?: number;
370
370
  /** The transport sends JSON mode instead of a response schema. */
371
371
  jsonSchemaTransport?: 'native' | 'json-object';
@@ -496,6 +496,13 @@ interface LlmClientOptions extends LlmChargeBounds {
496
496
  idempotencyKey?: string;
497
497
  /** Default timeout in ms. Per-call can override. */
498
498
  defaultTimeoutMs?: number;
499
+ /**
500
+ * Minimum milliseconds between physical request starts, including retries.
501
+ * Default 0 (disabled). Share one LlmClient per key, or reuse the same options
502
+ * object with callLlm/callLlmJson, to pace concurrent and sequential calls.
503
+ * Independent options objects and processes do not share a quota.
504
+ */
505
+ minIntervalMs?: number;
499
506
  /**
500
507
  * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
501
508
  * each attempt's per-attempt timeout controller, so aborting it cancels
@@ -636,7 +643,7 @@ interface CustomTransportOpts extends BaseTransportOpts {
636
643
  * `authHeader`. There is no environment fallback: agent-eval never goes
637
644
  * looking for a key.
638
645
  */
639
- interface OpenAiCompatibleTransportOpts extends BaseTransportOpts, Pick<LlmClientOptions, 'assertServedModel' | 'customTokenPricing' | 'deadlineMs' | 'defaultTimeoutMs' | 'fetch' | 'jsonPayloadMode' | 'jsonSchemaTransport' | 'provider' | 'rawSink' | 'signal' | 'thinking'> {
646
+ interface OpenAiCompatibleTransportOpts extends BaseTransportOpts, Pick<LlmClientOptions, 'assertServedModel' | 'customTokenPricing' | 'deadlineMs' | 'defaultTimeoutMs' | 'fetch' | 'jsonPayloadMode' | 'jsonSchemaTransport' | 'minIntervalMs' | 'provider' | 'rawSink' | 'signal' | 'thinking'> {
640
647
  transport: 'openai-compatible';
641
648
  /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */
642
649
  baseUrl: string;
@@ -862,4 +869,4 @@ interface CheckResult {
862
869
  }
863
870
  //#endregion
864
871
  export { NoopRawProviderSink as $, createChatClient as A, LlmToolCall as B, ChatResponse as C, judgeFamily as Ct, MockTransportOpts as D, CustomTransportOpts as E, LlmChargeBounds as F, costReceiptFromLlmError as G, LlmToolDefinition as H, LlmMessage as I, stripFencedJson as J, isTransientLlmError as K, LlmResponseError as L, LlmCallMetadata as M, LlmCallRequest as N, OpenAiCompatibleTransportOpts as O, LlmCallResult as P, InMemoryRawProviderSinkOptions as Q, LlmThinkingMode as R, ChatRequest as S, assertCrossFamily as St, CreateChatClientOpts as T, LlmUsage as U, LlmToolChoice as V, costReceiptFromLlm as W, FileSystemRawProviderSinkOptions as X, FileSystemRawProviderSink as Y, InMemoryRawProviderSink as Z, ScenarioResult as _, checkServedModel as _t, CompletionCriterion as a, defaultProviderRedactor as at, ChatCallOpts as b, CrossFamilyError as bt, JudgeInput as c, AssertServedModelOptions as ct, PersonaConfig as d, ServedModelCheck as dt, ProviderRedactor as et, ProductClientConfig as f, ServedModelPolicy as ft, ScenarioFile as g, assertServedModels as gt, Scenario as h, assertServedModel as ht, CollectedArtifacts as i, RawProviderSinkFilter as it, LlmCallError as j, SandboxSdkTransportOpts as k, JudgeRubric as l, ModelSubstitutionError as lt, RubricDimension as m, assertCrossFamilyServed as mt, ArtifactResult as n, RawProviderEvent as nt, DriverState as o, providerFromBaseUrl as ot, RouteMap as p, ServedModelVerdict as pt, maximumChargeForLlmRequest as q, CheckResult as r, RawProviderSink as rt, JudgeFn as s, AssertCrossFamilyServedOptions as st, ArtifactCheck as t, RawProviderDirection as tt, JudgeScore as u, ServedCrossFamilyError as ut, Turn as v, servedModelAcceptable as vt, ChatTransport as w, ChatClient as x, JudgeFamily as xt, TurnResult as y, AssertCrossFamilyOptions as yt, LlmTokenLogprob as z };
865
- //# sourceMappingURL=types-DLxe96rp.d.ts.map
872
+ //# sourceMappingURL=types-Lub7o6bN.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-DLxe96rp.d.ts","names":[],"sources":["../src/judge-families.ts","../src/integrity/served-model.ts","../src/trace/raw-provider-sink.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;KAWY;;;;;;;iBAsEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCvFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;iBA4Cc,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;;;;;;;;KAc9B;UAWK;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KCjRS;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;iBAM5C,wBAAwB,OAAO,mBAAmB;UAqBjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;UCnLnB;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;EAGvD,YAAY;;EAEZ;;KAGU;;UAGK;EACf;EACA;IACE;IACA;IACA,YAAY;;;;KAKJ;EAIN;EAAkB;IAAY;;;;UAGnB;EACf;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;;EAErC,QAAQ;;EAER,aAAa;EACb;EACA;;;;;EAKA;IAAa;;;EAEb,WAAW;;EAEX;;;;;;UAOe;;EAEf;;EAEA;;EAEA,WAAW;;EAEX,qBAAqB;;iBAGP,2BACd,SAAS,KACP,0GAGF,UAAS,kBACR;UA+Bc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;;;UAKe;EACf;EACA;EACA,KAAK;IAAgB;IAAe;;;UAGrB;;EAEf;;EAEA,YAAY;EACZ,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;;;;;;;EAQA,WAAW,cAAc;;EAEzB,KAAK;;KAGK,kBAAkB,KAAK;;;;;;iBAOnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC,yBAAyB;;EAExC;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;;;;;EAMA;;EAEA,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBAwEhB,oBAAoB;;;;;;iBAsPpB,gBAAgB;;;;;;UCpqBf;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAMK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,0BACA,sBACA,gCACA;UAEM;EACR;;EAEA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;;;;;;;;;;;;;;;UAqB1C,sCACP,mBACN,KACE;EAaJ;;EAEA;;EAEA;;EAEA;;EAEA;IAAe;IAAc;;;;;;;UAOd,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UC/J7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,OAAO;EACP,iBAAiB;EACjB,aAAa;EACb;EACA;EACA;EACA,WAAW;;EAEX,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IACE;MACE;MACA;MACA;;;EAGJ,OAAO;EACP,gBAAgB;;UAKD;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;;;;;;;;;KAWT;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
1
+ {"version":3,"file":"types-Lub7o6bN.d.ts","names":[],"sources":["../src/judge-families.ts","../src/integrity/served-model.ts","../src/trace/raw-provider-sink.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;KAWY;;;;;;;iBAsEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCvFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;iBA4Cc,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;;;;;;;;KAc9B;UAWK;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KCjRS;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;iBAM5C,wBAAwB,OAAO,mBAAmB;UAqBjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;UCnLnB;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;EAGvD,YAAY;;EAEZ;;KAGU;;UAGK;EACf;EACA;IACE;IACA;IACA,YAAY;;;;KAKJ;EAIN;EAAkB;IAAY;;;;UAGnB;EACf;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;;EAErC,QAAQ;;EAER,aAAa;EACb;EACA;;;;;EAKA;IAAa;;;EAEb,WAAW;;EAEX;;;;;;UAOe;;EAEf;;EAEA;;EAEA,WAAW;;EAEX,qBAAqB;;iBAGP,2BACd,SAAS,KACP,0GAGF,UAAS,kBACR;UA+Bc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;;;UAKe;EACf;EACA;EACA,KAAK;IAAgB;IAAe;;;UAGrB;;EAEf;;EAEA,YAAY;EACZ,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;;;;;;;EAQA,WAAW,cAAc;;EAEzB,KAAK;;KAGK,kBAAkB,KAAK;;;;;;iBAOnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC,yBAAyB;;EAExC;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;;;;;EAMA;;EAEA,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBA6FhB,oBAAoB;;;;;;iBA+PpB,gBAAgB;;;;;;UCzsBf;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAMK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,0BACA,sBACA,gCACA;UAEM;EACR;;EAEA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;;;;;;;;;;;;;;;UAqB1C,sCACP,mBACN,KACE;EAcJ;;EAEA;;EAEA;;EAEA;;EAEA;IAAe;IAAc;;;;;;;UAOd,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UChK7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,OAAO;EACP,iBAAiB;EACjB,aAAa;EACb;EACA;EACA;EACA,WAAW;;EAEX,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IACE;MACE;MACA;MACA;;;EAGJ,OAAO;EACP,gBAAgB;;UAKD;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;;;;;;;;;KAWT;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
@@ -1,7 +1,7 @@
1
1
  import { b as CustomTokenPricing, c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
2
- import { x as ChatClient } from "../types-DLxe96rp.js";
2
+ import { x as ChatClient } from "../types-Lub7o6bN.js";
3
3
  import { s as TraceStore } from "../store-C4avFjIi.js";
4
- import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-C6NJxXEa.js";
4
+ import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-C6Z8PRoF.js";
5
5
  import { z } from "zod";
6
6
  import { ServerType } from "@hono/node-server";
7
7
  import { Hono } from "hono";
@@ -1,2 +1,2 @@
1
- import { A as WIRE_VERSION, C as RUBRIC_VERSION_SCHEME, D as TracesIngestRequestSchema, E as TraceEventSchema, O as TracesIngestResponseSchema, S as ListRubricsResponseSchema, T as RubricSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, j as hashRubric, k as VersionResponseSchema, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricDimensionSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-CoBZO1fj.js";
1
+ import { A as WIRE_VERSION, C as RUBRIC_VERSION_SCHEME, D as TracesIngestRequestSchema, E as TraceEventSchema, O as TracesIngestResponseSchema, S as ListRubricsResponseSchema, T as RubricSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, j as hashRubric, k as VersionResponseSchema, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricDimensionSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-BZOt7eNC.js";
2
2
  export { ErrorResponseSchema, FeedbackIngestResponseSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RUBRIC_VERSION_SCHEME, RubricDimensionSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
@@ -0,0 +1,56 @@
1
+ # LLM retry delay and request pacing
2
+
3
+ Fleet item it-fd64863571. Source: tangle-network/ops-board#882.
4
+
5
+ The OpenAI-compatible transport accepts `minIntervalMs`, defaulting to `0`
6
+ (disabled). It spaces **physical request starts**, including transient retries,
7
+ schema negotiation and temperature fallback. Share one client for a key across
8
+ a campaign; the internal `callLlm` and `callLlmJson` functions instead share state
9
+ when passed the same options object. Independent options objects and processes
10
+ do not coordinate quotas. Credentials are not stored in a global key registry.
11
+
12
+ ```ts
13
+ const chat = createChatClient({
14
+ transport: 'openai-compatible',
15
+ baseUrl,
16
+ apiKey,
17
+ minIntervalMs: Math.ceil(60_000 / requestsPerMinute),
18
+ })
19
+ ```
20
+
21
+ Concurrent callers recheck the interval after waking, so overdue waiters do not
22
+ create a catch-up burst. Responses may overlap. Per-call overrides on an internal
23
+ `LlmClient` retain its shared pacing state. Pacing is outside the HTTP-attempt
24
+ timeout, but inside the operation deadline, and cancellation interrupts waits.
25
+ A cancelled waiter does not reserve a future request slot. The raw request
26
+ event is written after the wait, at the claimed start, so a call cancelled or
27
+ expired while queued leaves no raw request or error event.
28
+
29
+ `Retry-After` already took precedence over the capped exponential backoff. The
30
+ regression suite now checks a full 30-second delay followed by a paced call
31
+ through all five raw/JSON/client/public transport paths, plus HTTP-date headers.
32
+ Long delays are split into timer-safe chunks instead of overflowing the timer's
33
+ signed 32-bit limit. Invalid numeric delays fall back to exponential backoff.
34
+ When an explicit operation deadline is shorter than `Retry-After`, the operation
35
+ ends without issuing an early retry.
36
+
37
+ The default `maximumAttempts` is now **6 total physical requests** (was 3), not
38
+ six retries. Explicit `maximumAttempts` and `TANGLE_LLM_MAXIMUM_ATTEMPTS` still
39
+ win. `maximumChargeForLlmRequest` uses the same default, so increased retry
40
+ headroom also increases the conservative cost reservation. No `maxRetries`
41
+ alias or second retry loop is introduced.
42
+
43
+ ## Validation
44
+
45
+ Run the repository's existing Vitest suite:
46
+
47
+ ```sh
48
+ pnpm test -- tests/llm-pacing.test.ts tests/llm-physical-attempts.test.ts src/llm-client.test.ts src/analyst/chat-client-openai-compatible.test.ts
49
+ pnpm typecheck
50
+ pnpm exec biome check src/llm-client.ts src/analyst/chat-client.ts tests/llm-pacing.test.ts
51
+ ```
52
+
53
+ The new tests use a mocked fetch transport and fake timers; no provider calls,
54
+ credentials or real 30-second sleeps are required. They also cover concurrent
55
+ starts, delayed raw sinks, fallback pacing, cancellation, deadlines, independent
56
+ options, disabled/invalid intervals, and the shared retry/cost allowance.
@@ -0,0 +1,268 @@
1
+ # External agent quickstart
2
+
3
+ Run an evaluation and an improvement loop in your own Node.js process.
4
+ No Tangle account, sandbox, runtime, or framework migration is required.
5
+ The first run is an offline wiring exercise; `--openai` explicitly opts into paid model calls.
6
+
7
+ Use `defineAgentEval()` from the [main README](../README.md#quickstart) for the higher-level API.
8
+ This guide exposes `dispatch`, `runCampaign()`, and `runImprovementLoop()` when you want to own the wiring.
9
+
10
+ ## Install
11
+
12
+ With Node.js 20.19 or newer, start in an empty directory:
13
+
14
+ ```sh
15
+ npm init -y
16
+ npm install @tangle-network/agent-eval
17
+ npm install --save-dev tsx
18
+ ```
19
+
20
+ ## Run a campaign and an improvement loop
21
+
22
+ Save this entire block as `eval.mts`. It includes a deterministic agent, a rubric,
23
+ a candidate proposer, and a reference adapter for the raw OpenAI-compatible chat API.
24
+ The adapter reuses the package's maintained transport and receipt helpers rather than introducing another SDK or runtime.
25
+
26
+ <!-- external-quickstart:program -->
27
+ ```ts
28
+ import { randomUUID } from 'node:crypto'
29
+ import { join, resolve } from 'node:path'
30
+ import {
31
+ costReceiptFromLlm,
32
+ costReceiptFromLlmError,
33
+ maximumChargeForLlmRequest,
34
+ } from '@tangle-network/agent-eval'
35
+ import {
36
+ createChatClient,
37
+ defaultProductionGate,
38
+ type DispatchContext,
39
+ inMemoryCampaignStorage,
40
+ type JudgeConfig,
41
+ type MutableSurface,
42
+ type Scenario,
43
+ type SurfaceProposer,
44
+ } from '@tangle-network/agent-eval/contract'
45
+ import { runCampaign, runImprovementLoop } from '@tangle-network/agent-eval/campaign'
46
+ import { type DeploymentOutcome, InMemoryOutcomeStore } from '@tangle-network/agent-eval/rl'
47
+
48
+ interface SupportCase extends Scenario { ticket: string }
49
+ interface SupportAnswer { text: string }
50
+
51
+ // Replace this function with the SAME agent call your application uses.
52
+ // Its input/output need not know anything about agent-eval.
53
+ async function existingAgent(input: {
54
+ instructions: string
55
+ ticket: string
56
+ signal: AbortSignal
57
+ }) {
58
+ input.signal.throwIfAborted()
59
+ return {
60
+ message: input.instructions.includes('cite the ticket id')
61
+ ? `Ticket ${input.ticket}: on it.`
62
+ : 'On it.',
63
+ }
64
+ }
65
+
66
+ function requiredEnv(name: string): string {
67
+ const value = process.env[name]?.trim()
68
+ if (!value) throw new Error(`Set ${name} before using --openai`)
69
+ return value
70
+ }
71
+
72
+ function tokenRate(name: string): number {
73
+ const value = Number(requiredEnv(name))
74
+ if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a nonnegative rate`)
75
+ return value
76
+ }
77
+
78
+ // Reference adapter: raw OpenAI-compatible API, no Tangle execution service.
79
+ function openAiDispatch() {
80
+ const model = requiredEnv('LLM_MODEL')
81
+ const pricing = {
82
+ inputUsdPerMillion: tokenRate('LLM_INPUT_USD_PER_MILLION'),
83
+ outputUsdPerMillion: tokenRate('LLM_OUTPUT_USD_PER_MILLION'),
84
+ }
85
+ const chat = createChatClient({
86
+ transport: 'openai-compatible',
87
+ baseUrl: requiredEnv('LLM_BASE_URL'),
88
+ apiKey: requiredEnv('LLM_API_KEY'),
89
+ defaultModel: model,
90
+ })
91
+ return async (
92
+ surface: MutableSurface,
93
+ scenario: SupportCase,
94
+ ctx: DispatchContext,
95
+ ): Promise<SupportAnswer> => {
96
+ const request = {
97
+ model,
98
+ messages: [
99
+ { role: 'system' as const, content: String(surface) },
100
+ { role: 'user' as const, content: `Please acknowledge ticket ${scenario.ticket}.` },
101
+ ],
102
+ maxTokens: 256,
103
+ }
104
+ const paid = await ctx.cost.runPaidCall({
105
+ actor: 'external-support-agent',
106
+ model,
107
+ maximumCharge: maximumChargeForLlmRequest(request, {
108
+ customTokenPricing: pricing,
109
+ maximumAttempts: chat.maximumAttempts,
110
+ }),
111
+ execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }),
112
+ receipt: (result) => costReceiptFromLlm(result, pricing),
113
+ receiptFromError: (error) => costReceiptFromLlmError(error, pricing),
114
+ })
115
+ if (!paid.succeeded) throw paid.error
116
+ if (!paid.value.content.trim()) throw new Error('The agent returned an empty answer')
117
+ return { text: paid.value.content }
118
+ }
119
+ }
120
+
121
+ const useOpenAi = process.argv.includes('--openai')
122
+ const dispatchWithSurface = useOpenAi
123
+ ? openAiDispatch()
124
+ : async (surface: MutableSurface, scenario: SupportCase, ctx: DispatchContext) => {
125
+ const result = await existingAgent({
126
+ instructions: String(surface),
127
+ ticket: scenario.ticket,
128
+ signal: ctx.signal,
129
+ })
130
+ return { text: result.message }
131
+ }
132
+
133
+ // A deterministic rubric is a judge. Use llmJudge() for semantic criteria.
134
+ const judge: JudgeConfig<SupportAnswer, SupportCase> = {
135
+ name: 'ticket-id',
136
+ judgeVersion: 'ticket-id-v1',
137
+ dimensions: [{ key: 'present', description: 'The answer cites the supplied ticket id' }],
138
+ score: ({ artifact, scenario }) => {
139
+ const present = artifact.text.includes(scenario.ticket) ? 1 : 0
140
+ return { dimensions: { present }, composite: present, notes: '' }
141
+ },
142
+ }
143
+ const scenarios: SupportCase[] = ['refund', 'shipping'].map((ticket) => ({
144
+ id: `train-${ticket}`, kind: 'support', ticket,
145
+ }))
146
+ const holdoutScenarios: SupportCase[] = ['cancel', 'billing'].map((ticket) => ({
147
+ id: `holdout-${ticket}`, kind: 'support', ticket,
148
+ }))
149
+ const baselineSurface = 'Answer politely.'
150
+ const proposer: SurfaceProposer = {
151
+ kind: 'cite-ticket',
152
+ async propose({ currentSurface }) {
153
+ return [{
154
+ surface: `${String(currentSurface)} Always cite the ticket id.`,
155
+ label: 'cite-ticket',
156
+ rationale: 'The rubric requires the supplied ticket id in the answer.',
157
+ }]
158
+ },
159
+ }
160
+ const runRoot = resolve('eval-quickstart', randomUUID())
161
+ const common = {
162
+ judges: [judge],
163
+ storage: inMemoryCampaignStorage(),
164
+ seed: 7,
165
+ reps: 1,
166
+ maxConcurrency: 2,
167
+ dispatchTimeoutMs: 30_000,
168
+ abortOnCellError: true,
169
+ expectUsage: useOpenAi ? 'assert' as const : 'off' as const,
170
+ // Each top-level call has its own $1 ceiling; the loop shares its ceiling
171
+ // across search and holdout. The script has $2 of total ledger budget at the declared rates.
172
+ costCeiling: useOpenAi ? 1 : undefined,
173
+ }
174
+ const baseline = await runCampaign({
175
+ ...common,
176
+ scenarios,
177
+ dispatch: async function dispatch(scenario, ctx) {
178
+ return dispatchWithSurface(baselineSurface, scenario, ctx)
179
+ },
180
+ dispatchRef: useOpenAi ? 'external-openai-baseline-v1' : 'external-fixture-baseline-v1',
181
+ runDir: join(runRoot, 'baseline'),
182
+ })
183
+ const improvement = await runImprovementLoop({
184
+ ...common,
185
+ scenarios,
186
+ holdoutScenarios,
187
+ baselineSurface,
188
+ dispatchWithSurface,
189
+ dispatchRef: useOpenAi ? 'external-openai-v1' : 'external-fixture-v1',
190
+ proposer,
191
+ populationSize: 1,
192
+ maxGenerations: 1,
193
+ gate: defaultProductionGate<SupportAnswer, SupportCase>({ holdoutScenarios, deltaThreshold: 0 }),
194
+ autoOnPromote: 'none',
195
+ runDir: join(runRoot, 'improvement'),
196
+ })
197
+
198
+ // Synthetic storage example, NOT a measured production outcome or judge score.
199
+ const outcomes = new InMemoryOutcomeStore()
200
+ const observation: DeploymentOutcome = {
201
+ runId: 'synthetic-deployment-run',
202
+ capturedAt: Date.now(),
203
+ metrics: { resolved: 1 },
204
+ source: 'quickstart-fixture',
205
+ }
206
+ await outcomes.append(observation)
207
+ console.log('baseline:', baseline.aggregates.byJudge['ticket-id']?.mean)
208
+ console.log('candidate on holdout:', improvement.winnerOnHoldout.aggregates.byJudge['ticket-id']?.mean)
209
+ console.log('decision:', improvement.gateResult.decision)
210
+ console.log('outcome rows:', (await outcomes.forRun(observation.runId)).length)
211
+ ```
212
+
213
+ ```sh
214
+ npx tsx eval.mts
215
+ ```
216
+
217
+ The offline fixture is constructed to score zero for the baseline and one for the candidate.
218
+ It uses only two holdout cases, so a higher score is not sufficient evidence to ship.
219
+ Inspect `improvement.gateResult`, not just the candidate's mean.
220
+ The candidate generator makes one known edit; this demonstrates the loop, not learned optimization or general agent quality.
221
+ `autoOnPromote: 'none'` reports the decision without changing your agent or opening a PR.
222
+
223
+ ## Connect a real agent
224
+
225
+ Replace `existingAgent()` with your production SDK call, graph invocation, service, or local model.
226
+ Keep the adaptation in `dispatchWithSurface`: translate the surface and scenario into the agent's input,
227
+ forward cancellation, and return the artifact your judge scores.
228
+ Do not pass hidden answer keys or grading instructions into the agent request.
229
+ Keep execution failures as thrown errors rather than successful empty strings or task-score zeroes.
230
+
231
+ For the included OpenAI-compatible adapter, explicitly configure your endpoint, credentials,
232
+ a currently deployed model id, and its current per-million-token rates. For OpenAI directly,
233
+ the base URL is `https://api.openai.com/v1`. Then opt in:
234
+
235
+ ```sh
236
+ # Set LLM_BASE_URL, LLM_API_KEY, LLM_MODEL,
237
+ # LLM_INPUT_USD_PER_MILLION and LLM_OUTPUT_USD_PER_MILLION in your shell.
238
+ npx tsx eval.mts --openai
239
+ ```
240
+
241
+ This mode makes paid calls. It does not fall back to the fixture when a provider fails.
242
+ The maintained transport reports model identity, usage, retries, and failure receipts;
243
+ `ctx.cost.runPaidCall()` reserves for bounded output and the declared maximum attempts before dispatch.
244
+ Unknown usage is not invented, and `expectUsage: 'assert'` rejects missing dispatch evidence.
245
+ Set rates for your actual endpoint; local estimates are not provider invoices. Add cache-read/write
246
+ rates to `pricing` when applicable. Increase `maxTokens` and the explicit budget together when your model needs more output headroom.
247
+
248
+ For an existing multi-call agent, meter every paid call at the point where its receipt is available.
249
+ Do not set `expectUsage: 'off'` merely to make an uninstrumented paid agent appear to work.
250
+ For semantic scoring, use `llmJudge()` from `/contract` with a separate `ChatClient`;
251
+ see [judge verdicts](./verdicts.md). Judge failures must remain failures, not low task scores.
252
+
253
+ ## Keep the useful boundaries
254
+
255
+ The same production execution function should serve your application and evaluation.
256
+ Candidate generation may use training cases, but final comparisons need separate cases;
257
+ real deployments also need representative data, calibrated judges, sufficient independent observations,
258
+ and an explicit approval policy. The tiny fixture is not release evidence.
259
+
260
+ `inMemoryCampaignStorage()` keeps campaign data in this process, and `InMemoryOutcomeStore` does the same
261
+ for downstream observations. Use `fsCampaignStorage()` and `FileSystemOutcomeStore` from the public barrels
262
+ when evidence must survive a restart. With durable campaign storage, include model, prompt, tools, and
263
+ configuration revisions in `dispatchRef` so different execution behavior cannot reuse a stale cache.
264
+
265
+ For real deployment feedback, use the actual production `RunRecord.runId` in `DeploymentOutcome.runId`
266
+ and record independently observed metrics. Do not relabel an evaluation score as a deployment outcome.
267
+ See [outcome validity](./outcome-validity.md), [campaign controls](./eval-surface-map.md), and
268
+ [evaluation integrity](./evaluation-integrity.md) for the next stage.
@@ -477,6 +477,9 @@ The finalists are the nodes decided `finalist`, in decision order; whether the b
477
477
  It returns `verified`, `mismatch` with the differences, or `unknown` for a claim another rule revision made.
478
478
  The projector checks the claim's structure (every finalist named, the Bonferroni confidence of each test, a ship's held-out units and pinned judge) but not its numbers, so a store keeps what `verifySearchClaim` derives, never the producer's claim as written.
479
479
  `runSearch` runs it on every close and on every rerun of a closed ledger, throws on `mismatch`, and returns it as `SearchRunResult.claimVerification`.
480
+ `searchClaimDecision(state, nodeId)` returns the `PairedPromotionDecision` the claim made for one tested finalist, made again from the closed ledger: the root as control, the improvement as `delta`, at the finalist's Bonferroni confidence, with the claim's estimator, resamples and seed.
481
+ It throws unless the result reproduces the claim's recorded test for that finalist, and returns null for a node the claim did not test.
482
+ A consumer that promotes the selected finalist cites this decision with the claim, rather than deciding again from the same cells at another confidence or seed.
480
483
  A judge change is a changed `search-opened` header, which `SearchRecorder.open` refuses; it starts a derived search instead of mixing verdicts.
481
484
 
482
485
  `compareOptimizationMethods` keeps its own held-out comparison for black-box methods such as GEPA, which return one winner and never see the test split.
@@ -555,7 +558,7 @@ Those claims need the sealed test split, the claim's power check, and held-out e
555
558
  - `src/search/lenses/edit-credit.ts`: `editCredit` and `editCreditText`.
556
559
  - `src/campaign/search-ledger-recording.ts`: `SearchRecorder` and the surface helpers.
557
560
  - `src/campaign/search-kernel.ts`: `runSearch`, the executor, proposer and codec ports, `searchPolicyView` and `searchDivergence`.
558
- - `src/campaign/search-claim.ts`: `planSearchClaim`, `decideSearchClaim`, `verifySearchClaim` and `searchClaimReserveUsd`.
561
+ - `src/campaign/search-claim.ts`: `planSearchClaim`, `decideSearchClaim`, `verifySearchClaim`, `searchClaimDecision` and `searchClaimReserveUsd`.
559
562
  - `src/search/lenses/landscape.ts`, `plateau.ts` and `skill-manifold.ts`: the geometry lenses, the plateau score and the unit extension.
560
563
  - `src/campaign/search-policy.ts`: `SearchPolicy`, `incumbent`, `incumbentWithOperatorBandit`, `crowdedFrontierParent`, `aide` and `beam`.
561
564
  - `src/campaign/allocation.ts`: `SearchAllocator`, `uniform` and `asha`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.207.3",
3
+ "version": "0.208.2",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {
@@ -1 +0,0 @@
1
- {"version":3,"file":"chat-client-5LLl4Ctl.js","names":[],"sources":["../src/analyst/chat-client.ts"],"sourcesContent":["/**\n * Provider-neutral chat contract for every model call made by agent-eval.\n *\n * The caller owns model execution. Evaluation code receives canonical requests\n * and results without importing a provider SDK, and agent-eval reads no\n * environment variable to find a provider credential: every transport is bound\n * at the call site.\n *\n * Two ways to bind one. `custom` and `sandbox-sdk` take a `chat` function the\n * caller wrote. `openai-compatible` takes an endpoint and a credential the\n * caller passes as values, and agent-eval issues the HTTP request itself.\n *\n * WHY THE THIRD ONE EXISTS. Every consumer that wanted a plain OpenAI-style\n * endpoint had to hand-write the same fetch loop — retry on 429 and 5xx,\n * `json_schema` degrade, fenced-JSON stripping, and, load-bearing here, the\n * `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed`\n * read. A hand-written transport that omits `servedModel` makes both identity\n * checks read `unreported`, so the two-vendor judge rule they enforce silently\n * measures nothing. Measured motive: agent-eval 0.172.1 exported no such\n * transport, so a two-model judge panel in discovery-lab reached\n * `createChatClient({ transport: 'custom', baseUrl, apiKey })` — fields that\n * exist on no transport option — and every call threw\n * `TypeError: opts.chat is not a function`, which surfaced as\n * `aggregateJudgeVerdicts: all 2 judges failed`.\n *\n * The credential still never comes from agent-eval's environment. `baseUrl`\n * and the bearer are required arguments with no default and no fallback, so a\n * misconfigured caller gets a refusal at construction, never a call to an\n * endpoint it did not name.\n */\n\nimport {\n type LlmCallRequest,\n type LlmCallResult,\n LlmClient,\n type LlmClientOptions,\n} from '../llm-client'\n\n/**\n * Unified chat interface using the package's canonical LLM request and result.\n */\nexport interface ChatClient {\n /** Display name of the bound transport, included in telemetry. */\n readonly transport: ChatTransport\n /** Default model when the caller omits one. */\n readonly defaultModel?: string\n /** Total provider attempts this transport can make for one chat call. */\n readonly maximumAttempts?: number\n\n /** Implementations must enforce `req.maxTokens` when it is present. */\n chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>\n}\n\nexport type ChatTransport =\n | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK\n | 'custom' // caller-adapted SDK or transport\n | 'openai-compatible' // caller-named /v1 endpoint, driven by this package\n | 'mock' // test-time injection\n\nexport interface ChatRequest extends Omit<LlmCallRequest, 'model'> {\n /** Optional — falls back to ChatClient.defaultModel. */\n model?: string\n}\n\nexport type ChatResponse = LlmCallResult\n\nexport interface ChatCallOpts {\n /** Cancel the in-flight request. */\n signal?: AbortSignal\n /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */\n maxCostUsd?: number\n /** Correlation tag carried into request headers when the transport allows. */\n correlationId?: string\n /** Stable provider idempotency key for retries/redrives of one paid call. */\n idempotencyKey?: string\n}\n\n// ── Factory ─────────────────────────────────────────────────────────\n\nexport type CreateChatClientOpts =\n | SandboxSdkTransportOpts\n | CustomTransportOpts\n | OpenAiCompatibleTransportOpts\n | MockTransportOpts\n\ninterface BaseTransportOpts {\n defaultModel?: string\n /** Total provider attempts. Required for opaque transports used in capped runs. */\n maximumAttempts?: number\n}\n\n/**\n * Sandbox-SDK transport. The caller supplies a canonical chat function for an\n * already-configured Sandbox handle, so agent-eval does not import the SDK.\n */\nexport interface SandboxSdkTransportOpts extends BaseTransportOpts {\n transport: 'sandbox-sdk'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */\nexport interface CustomTransportOpts extends BaseTransportOpts {\n transport: 'custom'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and\n * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.\n *\n * Use this instead of hand-rolling a `custom` transport around `fetch`. The\n * result carries `servedModel` — the model id the provider echoed, verbatim —\n * which is the only field `assertServedModel` and `assertCrossFamilyServed`\n * can read to witness a gateway answering from a different model than the one\n * requested.\n *\n * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this\n * package's to append. A `baseUrl` that already carries the path is a\n * construction error, not a request to a doubled URL.\n *\n * Exactly one credential form is required — `apiKey`, `bearer`, or\n * `authHeader`. There is no environment fallback: agent-eval never goes\n * looking for a key.\n */\nexport interface OpenAiCompatibleTransportOpts\n extends BaseTransportOpts,\n Pick<\n LlmClientOptions,\n | 'assertServedModel'\n | 'customTokenPricing'\n | 'deadlineMs'\n | 'defaultTimeoutMs'\n | 'fetch'\n | 'jsonPayloadMode'\n | 'jsonSchemaTransport'\n | 'provider'\n | 'rawSink'\n | 'signal'\n | 'thinking'\n > {\n transport: 'openai-compatible'\n /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */\n baseUrl: string\n /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */\n apiKey?: string\n /** Bearer credential, alternate spelling. */\n bearer?: string\n /** Non-bearer authorization header, for endpoints that want their own scheme. */\n authHeader?: { name: string; value: string }\n}\n\n/**\n * Mock transport for tests. The handler receives the request and returns\n * whatever the test wants. No retries, no JSON-schema degrade.\n */\nexport interface MockTransportOpts extends BaseTransportOpts {\n transport: 'mock'\n handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * Build a ChatClient bound to a specific transport. The returned client\n * is safe to share across analysts in a single registry run.\n */\nexport function createChatClient(opts: CreateChatClientOpts): ChatClient {\n switch (opts.transport) {\n case 'sandbox-sdk':\n return {\n transport: 'sandbox-sdk',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'custom':\n return {\n transport: 'custom',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'openai-compatible':\n return openAiCompatibleClient(opts)\n case 'mock':\n return {\n transport: 'mock',\n defaultModel: opts.defaultModel,\n maximumAttempts: 1,\n chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts),\n }\n }\n}\n\nfunction resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {\n if (req.model) return req\n if (!defaultModel) {\n throw new Error(\n 'ChatClient.chat: no model on request and no defaultModel on the client. ' +\n 'Either pass req.model or bind defaultModel at createChatClient().',\n )\n }\n return { ...req, model: defaultModel }\n}\n\n/**\n * Build the OpenAI-compatible client, refusing a half-configured one first.\n *\n * Both refusals are construction-time on purpose. A missing credential or a\n * doubled path is a caller mistake that a request would turn into a 401 or a\n * 404 from an endpoint the caller did not intend to reach — the failure then\n * reads as a provider problem instead of a configuration one.\n */\nfunction openAiCompatibleClient(opts: OpenAiCompatibleTransportOpts): ChatClient {\n const baseUrl = opts.baseUrl?.trim().replace(/\\/+$/, '') ?? ''\n if (!baseUrl) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.\",\n )\n }\n if (/\\/chat\\/completions$/.test(baseUrl)) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; \" +\n `this package appends /chat/completions. Got ${baseUrl}`,\n )\n }\n if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader \" +\n 'is required. agent-eval reads no environment variable for a credential.',\n )\n }\n\n const clientOptions: LlmClientOptions = {\n baseUrl,\n ...(opts.apiKey ? { apiKey: opts.apiKey } : {}),\n ...(opts.bearer ? { bearer: opts.bearer } : {}),\n ...(opts.authHeader ? { authHeader: opts.authHeader } : {}),\n ...(opts.maximumAttempts === undefined ? {} : { maximumAttempts: opts.maximumAttempts }),\n ...(opts.assertServedModel === undefined ? {} : { assertServedModel: opts.assertServedModel }),\n ...(opts.customTokenPricing === undefined\n ? {}\n : { customTokenPricing: opts.customTokenPricing }),\n ...(opts.deadlineMs === undefined ? {} : { deadlineMs: opts.deadlineMs }),\n ...(opts.defaultTimeoutMs === undefined ? {} : { defaultTimeoutMs: opts.defaultTimeoutMs }),\n ...(opts.fetch === undefined ? {} : { fetch: opts.fetch }),\n ...(opts.jsonPayloadMode === undefined ? {} : { jsonPayloadMode: opts.jsonPayloadMode }),\n ...(opts.jsonSchemaTransport === undefined\n ? {}\n : { jsonSchemaTransport: opts.jsonSchemaTransport }),\n ...(opts.provider === undefined ? {} : { provider: opts.provider }),\n ...(opts.rawSink === undefined ? {} : { rawSink: opts.rawSink }),\n ...(opts.signal === undefined ? {} : { signal: opts.signal }),\n ...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),\n }\n const client = new LlmClient(clientOptions)\n\n return {\n transport: 'openai-compatible',\n ...(opts.defaultModel ? { defaultModel: opts.defaultModel } : {}),\n maximumAttempts: client.maximumAttempts,\n chat: async (req, callOpts) => {\n const resolved = resolveModel(req, opts.defaultModel)\n return client.call(resolved as LlmCallRequest, {\n ...(callOpts?.signal ? { signal: callOpts.signal } : {}),\n ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}),\n })\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoKA,SAAgB,iBAAiB,MAAwC;CACvE,QAAQ,KAAK,WAAb;EACE,KAAK,eACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,UACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,qBACH,OAAO,uBAAuB,IAAI;EACpC,KAAK,QACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB;GACjB,MAAM,OAAO,KAAK,aAAa,KAAK,QAAQ,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EAC5F;CACJ;AACF;AAEA,SAAS,aAAa,KAAkB,cAA+C;CACrF,IAAI,IAAI,OAAO,OAAO;CACtB,IAAI,CAAC,cACH,MAAM,IAAI,MACR,2IAEF;CAEF,OAAO;EAAE,GAAG;EAAK,OAAO;CAAa;AACvC;;;;;;;;;AAUA,SAAS,uBAAuB,MAAiD;CAC/E,MAAM,UAAU,KAAK,SAAS,KAAK,CAAC,CAAC,QAAQ,QAAQ,EAAE,KAAK;CAC5D,IAAI,CAAC,SACH,MAAM,IAAI,MACR,+FACF;CAEF,IAAI,uBAAuB,KAAK,OAAO,GACrC,MAAM,IAAI,MACR,qIACiD,SACnD;CAEF,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,YACxD,MAAM,IAAI,MACR,oKAEF;CAGF,MAAM,gBAAkC;EACtC;EACA,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,aAAa,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;EACzD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,sBAAsB,KAAA,IAAY,CAAC,IAAI,EAAE,mBAAmB,KAAK,kBAAkB;EAC5F,GAAI,KAAK,uBAAuB,KAAA,IAC5B,CAAC,IACD,EAAE,oBAAoB,KAAK,mBAAmB;EAClD,GAAI,KAAK,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,YAAY,KAAK,WAAW;EACvE,GAAI,KAAK,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,kBAAkB,KAAK,iBAAiB;EACzF,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,wBAAwB,KAAA,IAC7B,CAAC,IACD,EAAE,qBAAqB,KAAK,oBAAoB;EACpD,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;EAC3D,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;CACnE;CACA,MAAM,SAAS,IAAI,UAAU,aAAa;CAE1C,OAAO;EACL,WAAW;EACX,GAAI,KAAK,eAAe,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;EAC/D,iBAAiB,OAAO;EACxB,MAAM,OAAO,KAAK,aAAa;GAC7B,MAAM,WAAW,aAAa,KAAK,KAAK,YAAY;GACpD,OAAO,OAAO,KAAK,UAA4B;IAC7C,GAAI,UAAU,SAAS,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;IACtD,GAAI,UAAU,iBAAiB,EAAE,gBAAgB,SAAS,eAAe,IAAI,CAAC;GAChF,CAAC;EACH;CACF;AACF"}