@tangle-network/agent-eval 0.145.21 → 0.146.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/dist/analyst/index.d.ts +13 -13
  3. package/dist/analyst/index.js +4 -4
  4. package/dist/{backend-integrity-BffcHGdm.d.ts → backend-integrity-HVmWyOaX.d.ts} +2 -2
  5. package/dist/{backend-integrity-BffcHGdm.d.ts.map → backend-integrity-HVmWyOaX.d.ts.map} +1 -1
  6. package/dist/{benchmark-DbiZbPdH.d.ts → benchmark-DTQ1RApl.d.ts} +3 -3
  7. package/dist/{benchmark-DbiZbPdH.d.ts.map → benchmark-DTQ1RApl.d.ts.map} +1 -1
  8. package/dist/{benchmark-command-BMwfxnb0.js → benchmark-command-CxLJljS1.js} +7 -7
  9. package/dist/{benchmark-command-BMwfxnb0.js.map → benchmark-command-CxLJljS1.js.map} +1 -1
  10. package/dist/benchmarks/index.d.ts +4 -4
  11. package/dist/benchmarks/index.js +3 -3
  12. package/dist/campaign/index.d.ts +8 -8
  13. package/dist/campaign/index.js +5 -5
  14. package/dist/{campaign-Ct6r9oNe.js → campaign-CbQXcARJ.js} +6 -6
  15. package/dist/{campaign-Ct6r9oNe.js.map → campaign-CbQXcARJ.js.map} +1 -1
  16. package/dist/{capture-fetch-hykwHfDI.d.ts → capture-fetch-BZiO2cEH.d.ts} +2 -2
  17. package/dist/{capture-fetch-hykwHfDI.d.ts.map → capture-fetch-BZiO2cEH.d.ts.map} +1 -1
  18. package/dist/{chat-client-Bp4Ebuuc.js → chat-client-BJmwfnjN.js} +2 -2
  19. package/dist/{chat-client-Bp4Ebuuc.js.map → chat-client-BJmwfnjN.js.map} +1 -1
  20. package/dist/cli.js +2 -2
  21. package/dist/{client-D_TIV9pJ.d.ts → client-YDkU5QST.d.ts} +2 -2
  22. package/dist/{client-D_TIV9pJ.d.ts.map → client-YDkU5QST.d.ts.map} +1 -1
  23. package/dist/contract/index.d.ts +10 -10
  24. package/dist/contract/index.js +5 -5
  25. package/dist/{default-registry-cpHtP2Og.d.ts → default-registry-DONui3mA.d.ts} +6 -6
  26. package/dist/{default-registry-cpHtP2Og.d.ts.map → default-registry-DONui3mA.d.ts.map} +1 -1
  27. package/dist/{define-agent-eval-BXusiVTQ.js → define-agent-eval-BGxmy4W6.js} +3 -3
  28. package/dist/{define-agent-eval-BXusiVTQ.js.map → define-agent-eval-BGxmy4W6.js.map} +1 -1
  29. package/dist/{define-agent-eval-BgG5DIOS.d.ts → define-agent-eval-DrpgTVPp.d.ts} +5 -5
  30. package/dist/{define-agent-eval-BgG5DIOS.d.ts.map → define-agent-eval-DrpgTVPp.d.ts.map} +1 -1
  31. package/dist/{dspy-rlm-engine-CqwhQQBw.js → dspy-rlm-engine-B2nst4NR.js} +2 -2
  32. package/dist/{dspy-rlm-engine-CqwhQQBw.js.map → dspy-rlm-engine-B2nst4NR.js.map} +1 -1
  33. package/dist/{engine-DJqRKbhs.d.ts → engine-Bc3seRPe.d.ts} +4 -4
  34. package/dist/{engine-DJqRKbhs.d.ts.map → engine-Bc3seRPe.d.ts.map} +1 -1
  35. package/dist/{eval-campaign-aNqpefCS.js → eval-campaign-DeGLwACc.js} +2 -2
  36. package/dist/{eval-campaign-aNqpefCS.js.map → eval-campaign-DeGLwACc.js.map} +1 -1
  37. package/dist/{exact-types-CzbhVDr2.d.ts → exact-types-CYFbvA5U.d.ts} +2 -2
  38. package/dist/{exact-types-CzbhVDr2.d.ts.map → exact-types-CYFbvA5U.d.ts.map} +1 -1
  39. package/dist/experiment/index.d.ts +2 -2
  40. package/dist/{external-optimizer-contracts-lixrOZdX.d.ts → external-optimizer-contracts-CKEY98bK.d.ts} +2 -2
  41. package/dist/{external-optimizer-contracts-lixrOZdX.d.ts.map → external-optimizer-contracts-CKEY98bK.d.ts.map} +1 -1
  42. package/dist/{external-optimizer-process-D-9iX64j.js → external-optimizer-process-DYNizz8W.js} +2 -2
  43. package/dist/{external-optimizer-process-D-9iX64j.js.map → external-optimizer-process-DYNizz8W.js.map} +1 -1
  44. package/dist/{external-optimizer-subprocess-x5AbhAYX.js → external-optimizer-subprocess-r6dP9DAj.js} +2 -2
  45. package/dist/{external-optimizer-subprocess-x5AbhAYX.js.map → external-optimizer-subprocess-r6dP9DAj.js.map} +1 -1
  46. package/dist/{feedback-trajectory-D9vYSob_.d.ts → feedback-trajectory-Cg9EDxKB.d.ts} +3 -3
  47. package/dist/{feedback-trajectory-D9vYSob_.d.ts.map → feedback-trajectory-Cg9EDxKB.d.ts.map} +1 -1
  48. package/dist/hosted/index.d.ts +2 -2
  49. package/dist/index-BgHYx1QG.d.ts +1 -0
  50. package/dist/{index-YrUFx3FU.d.ts → index-CevPFhDT.d.ts} +7 -7
  51. package/dist/{index-YrUFx3FU.d.ts.map → index-CevPFhDT.d.ts.map} +1 -1
  52. package/dist/{index-CM-e2LiV.d.ts → index-DGrcBMbC.d.ts} +9 -9
  53. package/dist/{index-CM-e2LiV.d.ts.map → index-DGrcBMbC.d.ts.map} +1 -1
  54. package/dist/index.d.ts +24 -25
  55. package/dist/index.d.ts.map +1 -1
  56. package/dist/index.js +18 -19
  57. package/dist/index.js.map +1 -1
  58. package/dist/{integrity-OrcI9Nau.d.ts → integrity-DDoeWibF.d.ts} +2 -2
  59. package/dist/{integrity-OrcI9Nau.d.ts.map → integrity-DDoeWibF.d.ts.map} +1 -1
  60. package/dist/{llm-client-d0-2TT1g.js → llm-client-_UE4fU7Z.js} +45 -2
  61. package/dist/{llm-client-d0-2TT1g.js.map → llm-client-_UE4fU7Z.js.map} +1 -1
  62. package/dist/{llm-judge-DqAkIQA1.js → llm-judge-B7ffRF8w.js} +3 -3
  63. package/dist/{llm-judge-DqAkIQA1.js.map → llm-judge-B7ffRF8w.js.map} +1 -1
  64. package/dist/matrix-su7mIfbB.d.ts +383 -0
  65. package/dist/matrix-su7mIfbB.d.ts.map +1 -0
  66. package/dist/meta-eval/index.d.ts +1 -1
  67. package/dist/metrics-Cl0L1KUy.js.map +1 -1
  68. package/dist/multishot/golden/index.d.ts +230 -0
  69. package/dist/multishot/golden/index.d.ts.map +1 -0
  70. package/dist/multishot/golden/index.js +7686 -0
  71. package/dist/multishot/golden/index.js.map +1 -0
  72. package/dist/multishot/index.d.ts +2 -380
  73. package/dist/multishot/index.d.ts.map +1 -1
  74. package/dist/multishot/index.js +1 -105
  75. package/dist/multishot/index.js.map +1 -1
  76. package/dist/openapi.json +1 -1
  77. package/dist/{produced-state-B9skdU8u.js → produced-state-D3k45S9a.js} +3 -3
  78. package/dist/{produced-state-B9skdU8u.js.map → produced-state-D3k45S9a.js.map} +1 -1
  79. package/dist/{promotion-policy-u3wj6w3U.d.ts → promotion-policy-BKi5OK5l.d.ts} +2 -2
  80. package/dist/{promotion-policy-u3wj6w3U.d.ts.map → promotion-policy-BKi5OK5l.d.ts.map} +1 -1
  81. package/dist/{provenance-AACfvbUR.d.ts → provenance-Bb7Zmyj2.d.ts} +6 -6
  82. package/dist/{provenance-AACfvbUR.d.ts.map → provenance-Bb7Zmyj2.d.ts.map} +1 -1
  83. package/dist/{registry-B_1Frl8a.d.ts → registry-CZgEmVXb.d.ts} +4 -4
  84. package/dist/{registry-B_1Frl8a.d.ts.map → registry-CZgEmVXb.d.ts.map} +1 -1
  85. package/dist/{researcher-Du-oniHp.d.ts → researcher-CC305Ed-.d.ts} +3 -3
  86. package/dist/{researcher-Du-oniHp.d.ts.map → researcher-CC305Ed-.d.ts.map} +1 -1
  87. package/dist/rl.d.ts +2 -2
  88. package/dist/rl.js +1 -1
  89. package/dist/{semantic-concept-judge-C6M-qOeb.js → semantic-concept-judge-BsY2Q0Oq.js} +2 -2
  90. package/dist/{semantic-concept-judge-C6M-qOeb.js.map → semantic-concept-judge-BsY2Q0Oq.js.map} +1 -1
  91. package/dist/{series-convergence-D1cL1f-4.d.ts → series-convergence-C9G-GNYK.d.ts} +2 -2
  92. package/dist/{series-convergence-D1cL1f-4.d.ts.map → series-convergence-C9G-GNYK.d.ts.map} +1 -1
  93. package/dist/{server-ulsOdrTI.js → server-cQXve8i4.js} +2 -2
  94. package/dist/{server-ulsOdrTI.js.map → server-cQXve8i4.js.map} +1 -1
  95. package/dist/{skillopt-optimization-method-DO6OFgcz.d.ts → skillopt-optimization-method-C9MrMgW9.d.ts} +5 -5
  96. package/dist/{skillopt-optimization-method-DO6OFgcz.d.ts.map → skillopt-optimization-method-C9MrMgW9.d.ts.map} +1 -1
  97. package/dist/{skillopt-optimization-method-CcWZBaSz.js → skillopt-optimization-method-Cv3K4DaS.js} +4 -4
  98. package/dist/{skillopt-optimization-method-CcWZBaSz.js.map → skillopt-optimization-method-Cv3K4DaS.js.map} +1 -1
  99. package/dist/{statistical-heldout-CBGPSZ5X.d.ts → statistical-heldout-fvxtHnKQ.d.ts} +2 -2
  100. package/dist/{statistical-heldout-CBGPSZ5X.d.ts.map → statistical-heldout-fvxtHnKQ.d.ts.map} +1 -1
  101. package/dist/{store-tool-spans-C9c4R7ca.d.ts → store-tool-spans-CV0hRsUp.d.ts} +4 -4
  102. package/dist/{store-tool-spans-C9c4R7ca.d.ts.map → store-tool-spans-CV0hRsUp.d.ts.map} +1 -1
  103. package/dist/{tool-groups-CZotz-e_.d.ts → tool-groups-CDDXNKhd.d.ts} +3 -3
  104. package/dist/tool-groups-CDDXNKhd.d.ts.map +1 -0
  105. package/dist/trace-repair/index.d.ts +1 -1
  106. package/dist/traces.d.ts +7 -7
  107. package/dist/types-B3jzCp0p.js +108 -0
  108. package/dist/types-B3jzCp0p.js.map +1 -0
  109. package/dist/{types-C4bSVIr7.d.ts → types-CWcPguwd.d.ts} +2 -2
  110. package/dist/{types-C4bSVIr7.d.ts.map → types-CWcPguwd.d.ts.map} +1 -1
  111. package/dist/{types-BjsNDR49.d.ts → types-DABZgDGV.d.ts} +3 -3
  112. package/dist/{types-BjsNDR49.d.ts.map → types-DABZgDGV.d.ts.map} +1 -1
  113. package/dist/{types-Cx3YUh2r.d.ts → types-XBRdmbj8.d.ts} +61 -2
  114. package/dist/types-XBRdmbj8.d.ts.map +1 -0
  115. package/dist/{types-vXyshMwx.d.ts → types-oAsa-fs4.d.ts} +2 -2
  116. package/dist/{types-vXyshMwx.d.ts.map → types-oAsa-fs4.d.ts.map} +1 -1
  117. package/dist/wire/index.d.ts +2 -2
  118. package/dist/wire/index.js +1 -1
  119. package/docs/multishot-golden-records.md +102 -0
  120. package/package.json +6 -1
  121. package/dist/index-BKShPTwZ.d.ts +0 -1
  122. package/dist/tool-groups-CZotz-e_.d.ts.map +0 -1
  123. package/dist/types-Cx3YUh2r.d.ts.map +0 -1
@@ -1,4 +1,4 @@
1
- import { R as Scenario, b as GenerationRecord, p as Gate, w as JudgeScore } from "./types-BjsNDR49.js";
1
+ import { R as Scenario, b as GenerationRecord, p as Gate, w as JudgeScore } from "./types-DABZgDGV.js";
2
2
  import { r as SignedManifest, w as EProcessState } from "./pre-registration-zFSLEiFU.js";
3
3
  import { a as PairedPromotionDecision, d as PairedBootstrapResult, i as PairedMcNemarEvidence, r as PairedDecisionStatistic, t as PairedDecisionMethod } from "./paired-promotion-decision-CGzg0cI_.js";
4
4
  //#region src/campaign/gates/sequential.d.ts
@@ -279,4 +279,4 @@ declare function dimensionRegressions(candidate: Map<string, Record<string, Judg
279
279
  }): DimensionRegression[];
280
280
  //#endregion
281
281
  export { detectScale as a, pairHoldout as c, SequentialDecision as d, SequentialObservation as f, sequentialPairedGate as g, sequentialDecide as h, PairedHoldout as i, SequentialDecideFn as l, SequentialPairedGateOptions as m, HeldoutSignificance as n, dimensionRegressions as o, SequentialPairedGate as p, HeldoutSignificanceOptions as r, heldoutSignificance as s, DimensionRegression as t, SequentialDecideOptions as u };
282
- //# sourceMappingURL=statistical-heldout-CBGPSZ5X.d.ts.map
282
+ //# sourceMappingURL=statistical-heldout-fvxtHnKQ.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"statistical-heldout-CBGPSZ5X.d.ts","names":[],"sources":["../src/campaign/gates/sequential.ts","../src/campaign/gates/statistical-heldout.ts"],"mappings":";;;;KAuCY;UAEK;EACf,UAAU;;EAEV;;EAEA;;;EAGA;;UAGe;;;EAGf;;;;EAIA;;;EAGA;;EAEA;;;;EAIA;;;EAGA;;;EAGA,kBAAkB;;EAElB;;UAGe,qBAAqB,qBAAqB,kBAAkB,WAAW,kBAC9E,KAAK,WAAW;;;;;;;;EAQxB,QAAQ,gBAAgB;;EAExB,SAAS;IAAkB,UAAU;;;;;;;;;;;;;;;;;iBA0MvB,qBAAqB,qBAAqB,kBAAkB,WAAW,UACrF,SAAS,8BACR,qBAAqB,WAAW;UAsFlB;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;GACd;IAAQ,SAAS;;IAAyB;IAAe;;;EAE1D,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;iBA0BK,iBAAiB,UAAS,0BAA+B;;;UCtXxD;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;;iBAcc,YACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,SAAS,GAAG,oCACX;UAsDc;EACf,QAAQ;;;;;;;;;;;;EAYR,WAAW;;;;EAIX,iBAAiB;;;;;;;EAOjB,UAAU;;EAEV,mBAAmB;;EAEnB,SAAS;;;;EAIT;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB;;;;EAIA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;EACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6Bc,oBACd,QAAQ,eACR,OAAM,6BACL;UAkEc;EACf;;;EAGA,WAAW;;;;EAIX;;EAEA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;EAET;;;;;EAKA;EACA;EACA;;;;;iBAMc,YAAY;;;;;;;;;;;;;;;;;;;iBAsBZ,qBACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,8BACA;EACE;EACA;EACA;EACA;;;EAGA;IAED"}
1
+ {"version":3,"file":"statistical-heldout-fvxtHnKQ.d.ts","names":[],"sources":["../src/campaign/gates/sequential.ts","../src/campaign/gates/statistical-heldout.ts"],"mappings":";;;;KAuCY;UAEK;EACf,UAAU;;EAEV;;EAEA;;;EAGA;;UAGe;;;EAGf;;;;EAIA;;;EAGA;;EAEA;;;;EAIA;;;EAGA;;;EAGA,kBAAkB;;EAElB;;UAGe,qBAAqB,qBAAqB,kBAAkB,WAAW,kBAC9E,KAAK,WAAW;;;;;;;;EAQxB,QAAQ,gBAAgB;;EAExB,SAAS;IAAkB,UAAU;;;;;;;;;;;;;;;;;iBA0MvB,qBAAqB,qBAAqB,kBAAkB,WAAW,UACrF,SAAS,8BACR,qBAAqB,WAAW;UAsFlB;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;GACd;IAAQ,SAAS;;IAAyB;IAAe;;;EAE1D,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;iBA0BK,iBAAiB,UAAS,0BAA+B;;;UCtXxD;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;;iBAcc,YACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,SAAS,GAAG,oCACX;UAsDc;EACf,QAAQ;;;;;;;;;;;;EAYR,WAAW;;;;EAIX,iBAAiB;;;;;;;EAOjB,UAAU;;EAEV,mBAAmB;;EAEnB,SAAS;;;;EAIT;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB;;;;EAIA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;EACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6Bc,oBACd,QAAQ,eACR,OAAM,6BACL;UAkEc;EACf;;;EAGA,WAAW;;;;EAIX;;EAEA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;EAET;;;;;EAKA;EACA;EACA;;;;;iBAMc,YAAY;;;;;;;;;;;;;;;;;;;iBAsBZ,qBACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,8BACA;EACE;EACA;EACA;EACA;;;EAGA;IAED"}
@@ -2,9 +2,9 @@ import { r as CaptureIntegrityError } from "./errors-DEE6u6ot.js";
2
2
  import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
3
3
  import { S as ToolSpan } from "./schema-BtVldJ3T.js";
4
4
  import { s as TraceStore } from "./store-CT9YIIve.js";
5
- import { A as SearchSpanResult, B as ViewSpansResult, D as DatasetOverview, F as TraceAnalystFilters, H as ViewTraceResult, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, f as AnalystUsageReceipt, i as AnalystFinding, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore } from "./types-vXyshMwx.js";
6
- import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-DJqRKbhs.js";
7
- import { t as TraceToolGroupName } from "./tool-groups-CZotz-e_.js";
5
+ import { A as SearchSpanResult, B as ViewSpansResult, D as DatasetOverview, F as TraceAnalystFilters, H as ViewTraceResult, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, f as AnalystUsageReceipt, i as AnalystFinding, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore } from "./types-oAsa-fs4.js";
6
+ import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-Bc3seRPe.js";
7
+ import { t as TraceToolGroupName } from "./tool-groups-CDDXNKhd.js";
8
8
  //#region src/trace/otel.d.ts
9
9
  declare const OTEL_AGENT_EVAL_SCOPE: {
10
10
  name: string;
@@ -339,4 +339,4 @@ declare class ToolTraceMissingError extends CaptureIntegrityError {
339
339
  declare function toolSpansToTraceAnalysisStore(spans: readonly ToolSpan[] | null | undefined, options?: ToolSpansToTraceAnalysisStoreOptions): TraceAnalysisStore;
340
340
  //#endregion
341
341
  export { createBoundedTraceAnalysisStore as A, OtlpSpan as B, scoreTraceInsightReadiness as C, AnalyzeTracesResult as D, AnalyzeTracesOptions as E, redactString as F, redactValue as I, OTEL_AGENT_EVAL_SCOPE as L, REDACTION_VERSION as M, RedactionReport as N, analyzeTraces as O, RedactionRule as P, OtlpExport as R, planTraceInsightQuestions as S, AnalyzeTracesInput as T, exportRunAsOtlp as V, buildTraceInsightPrompt as _, ToolSpansToTraceAnalysisStoreOptions as a, domainEvidencePattern as b, TraceInsightFinding as c, TraceInsightQualityGate as d, TraceInsightQuestion as f, buildTraceInsightContext as g, TraceInsightTask as h, OtlpFileTraceStoreOptions as i, DEFAULT_REDACTION_RULES as j, OtlpFlatLine as k, TraceInsightPanelRole as l, TraceInsightSuite as m, toolSpansToTraceAnalysisStore as n, otlpTextToTraceAnalysisStore as o, TraceInsightReadiness as p, OtlpFileTraceStore as r, TraceInsightContext as s, ToolTraceMissingError as t, TraceInsightPromptInput as u, defaultTraceInsightPanel as v, tokenizeDomainWords as w, inferDomainKeywords as x, describeTraceInsightScope as y, OtlpResourceSpans as z };
342
- //# sourceMappingURL=store-tool-spans-C9c4R7ca.d.ts.map
342
+ //# sourceMappingURL=store-tool-spans-CV0hRsUp.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"store-tool-spans-C9c4R7ca.d.ts","names":[],"sources":["../src/trace/otel.ts","../src/trace/redact.ts","../src/trace-analyst/store-boundary.ts","../src/trace/otlp-flat.ts","../src/trace-analyst/analyst.ts","../src/trace-analyst/insights.ts","../src/trace-analyst/store-otlp.ts","../src/trace-analyst/store-tool-spans.ts"],"mappings":";;;;;;;;cAqBa;EAA0B;EAAoC;;UAE1D;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;IACV;IACA;MAAS;MAAsB;MAAmB;MAAsB;;;EAE1E,SAAS;IAAQ;IAAsB;IAAc,aAAa;;EAClE;IAAW;IAAc;;;UAGV;EACf;IAAY,YAAY;;EACxB,YAAY;IAAQ,cAAc;IAAuB,OAAO;;;UAGjD;EACf,eAAe;;;iBAIK,gBACpB,OAAO,YACP,eACA,gBAAe,4CACd,QAAQ;;;;;;;;;;;;;;;;UCvCM;EACf;EACA,SAAS;;EAET;;UAGe;EACf;EACA,QAAQ;;;cAIG,yBAAyB;cAezB;;;;;iBAMG,aACd,eACA,QAAO;EACJ;EAAgB,QAAQ;;;;;;;iBAuBb,YACd,gBACA,QAAO,iBACP,SAAQ;EACL;EAAgB,QAAQ;;;;;iBCjDb,gCACd,QAAQ,oBACR,UAAS,mCACR;;;KC5BS;UAEK;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE,MAAM;IACN;;EAEF;IAAY,YAAY;;EACxB,YAAY;EACZ,SAAS;IAAQ;IAAc;IAAuB,aAAa;;;;;UCNpD;EACf;EACA;EACA;EACA;;KAGU,sBAAsB;UAEjB;;EAEf,iBAAiB;;EAEjB,QAAQ;;EAER;;EAEA,YAAY;EACZ,SAAS,QAAQ;EACjB;EACA;EACA,aAAa;EACb;EACA,yBAAyB;EACzB,4BAA4B;EAC5B,eAAe,SAAS;EACxB,OAAO;EACP,OAAO,iBAAiB,SAAS;EACjC,SAAS;;;;;;;;iBASW,cACpB,OAAO,oBACP,SAAS,uBACR,QAAQ;;;UCtDM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf,OAAO;EACP,WAAW;EACX,QAAQ;EACR,SAAS;EACT;;UAGe;EACf,OAAO;EACP;EACA;EACA,WAAW;EACX,OAAO;EACP,UAAU;EACV,OAAO;EACP,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;iBA0BO,oBAAoB;iBAMpB,oBAAoB,OAAO;iBAuB3B,sBAAsB,qBAAqB;iBAS3C,0BAA0B,OAAO;iBAsBjC,0BAA0B,OAAO,0BAA0B;iBAsD3D,yBAAyB,OAAO,0BAA0B;iBAa1D,2BAA2B,SAAS,sBAAsB;iBA2D1D,4BAA4B;iBA6B5B,wBAAwB,OAAO;;;UC1K9B;;EAEf;;EAEA;;EAEA;;EAEA;;KAGG,gCAAgC;UAEpB,kCAAkC;;EAEjD;;;;;;;;EAQA;;uBAMa,kCAAkC;mBAC9B;mBACA;mBACA;mBACA;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;EAyBZ,SAAS,kBAAkB,UAAU,4BAA4B;EAOjE,SACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT;EAeG,YACJ,UAAU,qBACV,UAAU,4BACT,QAAQ;EAoDL,YACJ;IAAQ,UAAU;IAAqB;IAAe;KACtD,UAAU,4BACT,QAAQ;EAmBL,YACJ,UAAU,qBACV,UAAU,4BACT;EAMG,UACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT,QAAQ;EAuCL,UACJ;IAAQ;IAAkB;IAA6B;KACvD,UAAU,4BACT,QAAQ;EAsFL,YACJ;IAAQ;IAAkB;IAAuB;KACjD,UAAU,4BACT,QAAQ;EA4CL,WACJ;IAAQ;IAAkB;IAAiB;IAAuB;KAClE,UAAU,4BACT,QAAQ;;;EA6CL,cAAc,UAAU,4BAA4B;UAI5C;qBAmBK,WAAW,SAAS,cAAc,QAAQ;qBAE1C;UAEL;UAmBA;UAsHA;UA+DN;UAkBA;UAwDA;UAmBM;;cA+CH,2BAA2B;mBACrB;mBACA;EAEjB,YAAY,MAAM;;;YAYF,WAAW,SAAS,cAAc,QAAQ;YAkBhD;;;iBAwBI,6BACd,cACA,OAAM,uCACL;;;;cC/7BU,8BAA8B;EACzC;;;;;;iBASc,8BACd,gBAAgB,+BAChB,UAAS,uCACR"}
1
+ {"version":3,"file":"store-tool-spans-CV0hRsUp.d.ts","names":[],"sources":["../src/trace/otel.ts","../src/trace/redact.ts","../src/trace-analyst/store-boundary.ts","../src/trace/otlp-flat.ts","../src/trace-analyst/analyst.ts","../src/trace-analyst/insights.ts","../src/trace-analyst/store-otlp.ts","../src/trace-analyst/store-tool-spans.ts"],"mappings":";;;;;;;;cAqBa;EAA0B;EAAoC;;UAE1D;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;IACV;IACA;MAAS;MAAsB;MAAmB;MAAsB;;;EAE1E,SAAS;IAAQ;IAAsB;IAAc,aAAa;;EAClE;IAAW;IAAc;;;UAGV;EACf;IAAY,YAAY;;EACxB,YAAY;IAAQ,cAAc;IAAuB,OAAO;;;UAGjD;EACf,eAAe;;;iBAIK,gBACpB,OAAO,YACP,eACA,gBAAe,4CACd,QAAQ;;;;;;;;;;;;;;;;UCvCM;EACf;EACA,SAAS;;EAET;;UAGe;EACf;EACA,QAAQ;;;cAIG,yBAAyB;cAezB;;;;;iBAMG,aACd,eACA,QAAO;EACJ;EAAgB,QAAQ;;;;;;;iBAuBb,YACd,gBACA,QAAO,iBACP,SAAQ;EACL;EAAgB,QAAQ;;;;;iBCjDb,gCACd,QAAQ,oBACR,UAAS,mCACR;;;KC5BS;UAEK;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE,MAAM;IACN;;EAEF;IAAY,YAAY;;EACxB,YAAY;EACZ,SAAS;IAAQ;IAAc;IAAuB,aAAa;;;;;UCNpD;EACf;EACA;EACA;EACA;;KAGU,sBAAsB;UAEjB;;EAEf,iBAAiB;;EAEjB,QAAQ;;EAER;;EAEA,YAAY;EACZ,SAAS,QAAQ;EACjB;EACA;EACA,aAAa;EACb;EACA,yBAAyB;EACzB,4BAA4B;EAC5B,eAAe,SAAS;EACxB,OAAO;EACP,OAAO,iBAAiB,SAAS;EACjC,SAAS;;;;;;;;iBASW,cACpB,OAAO,oBACP,SAAS,uBACR,QAAQ;;;UCtDM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf,OAAO;EACP,WAAW;EACX,QAAQ;EACR,SAAS;EACT;;UAGe;EACf,OAAO;EACP;EACA;EACA,WAAW;EACX,OAAO;EACP,UAAU;EACV,OAAO;EACP,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;iBA0BO,oBAAoB;iBAMpB,oBAAoB,OAAO;iBAuB3B,sBAAsB,qBAAqB;iBAS3C,0BAA0B,OAAO;iBAsBjC,0BAA0B,OAAO,0BAA0B;iBAsD3D,yBAAyB,OAAO,0BAA0B;iBAa1D,2BAA2B,SAAS,sBAAsB;iBA2D1D,4BAA4B;iBA6B5B,wBAAwB,OAAO;;;UC1K9B;;EAEf;;EAEA;;EAEA;;EAEA;;KAGG,gCAAgC;UAEpB,kCAAkC;;EAEjD;;;;;;;;EAQA;;uBAMa,kCAAkC;mBAC9B;mBACA;mBACA;mBACA;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;EAyBZ,SAAS,kBAAkB,UAAU,4BAA4B;EAOjE,SACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT;EAeG,YACJ,UAAU,qBACV,UAAU,4BACT,QAAQ;EAoDL,YACJ;IAAQ,UAAU;IAAqB;IAAe;KACtD,UAAU,4BACT,QAAQ;EAmBL,YACJ,UAAU,qBACV,UAAU,4BACT;EAMG,UACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT,QAAQ;EAuCL,UACJ;IAAQ;IAAkB;IAA6B;KACvD,UAAU,4BACT,QAAQ;EAsFL,YACJ;IAAQ;IAAkB;IAAuB;KACjD,UAAU,4BACT,QAAQ;EA4CL,WACJ;IAAQ;IAAkB;IAAiB;IAAuB;KAClE,UAAU,4BACT,QAAQ;;;EA6CL,cAAc,UAAU,4BAA4B;UAI5C;qBAmBK,WAAW,SAAS,cAAc,QAAQ;qBAE1C;UAEL;UAmBA;UAsHA;UA+DN;UAkBA;UAwDA;UAmBM;;cA+CH,2BAA2B;mBACrB;mBACA;EAEjB,YAAY,MAAM;;;YAYF,WAAW,SAAS,cAAc,QAAQ;YAkBhD;;;iBAwBI,6BACd,cACA,OAAM,uCACL;;;;cC/7BU,8BAA8B;EACzC;;;;;;iBASc,8BACd,gBAAgB,+BAChB,UAAS,uCACR"}
@@ -1,5 +1,5 @@
1
- import { w as TraceAnalysisStore } from "./types-vXyshMwx.js";
2
- import { _ as TraceAnalysisToolDescriptor } from "./engine-DJqRKbhs.js";
1
+ import { w as TraceAnalysisStore } from "./types-oAsa-fs4.js";
2
+ import { _ as TraceAnalysisToolDescriptor } from "./engine-Bc3seRPe.js";
3
3
  //#region src/analyst/tool-groups.d.ts
4
4
  /** Named tool sets. Kinds pass `tools: TRACE_TOOL_GROUPS.failureForensics` etc. */
5
5
  type TraceToolGroupName =
@@ -25,4 +25,4 @@ type TraceToolGroupName =
25
25
  declare function buildTraceToolsForGroup(group: TraceToolGroupName, store: TraceAnalysisStore): TraceAnalysisToolDescriptor[];
26
26
  //#endregion
27
27
  export { buildTraceToolsForGroup as n, TraceToolGroupName as t };
28
- //# sourceMappingURL=tool-groups-CZotz-e_.d.ts.map
28
+ //# sourceMappingURL=tool-groups-CDDXNKhd.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tool-groups-CDDXNKhd.d.ts","names":[],"sources":["../src/analyst/tool-groups.ts"],"mappings":";;;;KAmBY;;;;;;;;;;;;;;;;;;;;iBAgDI,wBACd,OAAO,oBACP,OAAO,qBACN"}
@@ -2,7 +2,7 @@ import { r as CaptureIntegrityError } from "../errors-DEE6u6ot.js";
2
2
  import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
3
3
  import { b as CustomTokenPricing, c as CostLedgerHandle, p as CostProvenance } from "../cost-ledger-DbQdN3nO.js";
4
4
  import { u as RunTokenUsage } from "../run-record-DVV82Gwh.js";
5
- import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-DJqRKbhs.js";
5
+ import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-Bc3seRPe.js";
6
6
  import { t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
7
7
  import { g as isRecordedTimeout, o as RecordedTrajectoryStep } from "../steps-AmkT-GIM.js";
8
8
  //#region src/trace-repair/mini-swe-scaffold.d.ts
package/dist/traces.d.ts CHANGED
@@ -1,17 +1,17 @@
1
1
  import { c as ValidationError, o as LimitExceededError, r as CaptureIntegrityError, s as NotFoundError, t as AgentEvalError } from "./errors-DEE6u6ot.js";
2
2
  import { C as TraceEvent, D as isSandboxSpan, E as isRetrievalSpan, O as isToolSpan, S as ToolSpan, T as isLlmSpan, _ as Span, a as FAILURE_CLASSES, b as SpanStatus, c as JudgeSpan, d as RetrievalSpan, f as Run, g as SandboxSpan, h as RunStatus, i as EventKind, l as LlmSpan, m as RunOutcome, n as BudgetLedgerEntry, o as FailureClass, p as RunLayer, r as BudgetSpec, s as GenericSpan, t as Artifact, u as Message, v as SpanBase, w as isJudgeSpan, x as TRACE_SCHEMA_VERSION, y as SpanKind } from "./schema-BtVldJ3T.js";
3
3
  import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
4
- import { a as ExtractedUsage, c as extractUsageFromResponse, i as ExtractUsageFromSseOptions, l as extractUsageFromSse, n as CaptureFetchOptions, o as SseUsageMode, r as captureFetchToRawSink, s as extractUsage, t as CaptureFetchContext } from "./capture-fetch-hykwHfDI.js";
5
- import { $ as InMemoryRawProviderSinkOptions, Q as InMemoryRawProviderSink, X as FileSystemRawProviderSink, Z as FileSystemRawProviderSinkOptions, at as RawProviderSinkFilter, et as NoopRawProviderSink, it as RawProviderSink, nt as RawProviderDirection, ot as defaultProviderRedactor, rt as RawProviderEvent, st as providerFromBaseUrl, tt as ProviderRedactor } from "./types-Cx3YUh2r.js";
4
+ import { a as ExtractedUsage, c as extractUsageFromResponse, i as ExtractUsageFromSseOptions, l as extractUsageFromSse, n as CaptureFetchOptions, o as SseUsageMode, r as captureFetchToRawSink, s as extractUsage, t as CaptureFetchContext } from "./capture-fetch-BZiO2cEH.js";
5
+ import { _t as providerFromBaseUrl, ct as InMemoryRawProviderSink, dt as ProviderRedactor, ft as RawProviderDirection, gt as defaultProviderRedactor, ht as RawProviderSinkFilter, lt as InMemoryRawProviderSinkOptions, mt as RawProviderSink, ot as FileSystemRawProviderSink, pt as RawProviderEvent, st as FileSystemRawProviderSinkOptions, ut as NoopRawProviderSink } from "./types-XBRdmbj8.js";
6
6
  import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-CT9YIIve.js";
7
7
  import { a as TraceEmitterOptions, i as TraceEmitter, n as RunCompleteHookContext, o as llmSpanFromProvider, r as SpanHandle, t as RunCompleteHook } from "./emitter-DGQGoLyj.js";
8
- import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-OrcI9Nau.js";
9
- import { A as createBoundedTraceAnalysisStore, B as OtlpSpan, C as scoreTraceInsightReadiness, D as AnalyzeTracesResult, E as AnalyzeTracesOptions, F as redactString, I as redactValue, L as OTEL_AGENT_EVAL_SCOPE, M as REDACTION_VERSION, N as RedactionReport, O as analyzeTraces, P as RedactionRule, R as OtlpExport, S as planTraceInsightQuestions, T as AnalyzeTracesInput, V as exportRunAsOtlp, _ as buildTraceInsightPrompt, a as ToolSpansToTraceAnalysisStoreOptions, b as domainEvidencePattern, c as TraceInsightFinding, d as TraceInsightQualityGate, f as TraceInsightQuestion, g as buildTraceInsightContext, h as TraceInsightTask, i as OtlpFileTraceStoreOptions, j as DEFAULT_REDACTION_RULES, k as OtlpFlatLine, l as TraceInsightPanelRole, m as TraceInsightSuite, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightReadiness, r as OtlpFileTraceStore, s as TraceInsightContext, t as ToolTraceMissingError, u as TraceInsightPromptInput, v as defaultTraceInsightPanel, w as tokenizeDomainWords, x as inferDomainKeywords, y as describeTraceInsightScope, z as OtlpResourceSpans } from "./store-tool-spans-C9c4R7ca.js";
8
+ import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-DDoeWibF.js";
9
+ import { A as createBoundedTraceAnalysisStore, B as OtlpSpan, C as scoreTraceInsightReadiness, D as AnalyzeTracesResult, E as AnalyzeTracesOptions, F as redactString, I as redactValue, L as OTEL_AGENT_EVAL_SCOPE, M as REDACTION_VERSION, N as RedactionReport, O as analyzeTraces, P as RedactionRule, R as OtlpExport, S as planTraceInsightQuestions, T as AnalyzeTracesInput, V as exportRunAsOtlp, _ as buildTraceInsightPrompt, a as ToolSpansToTraceAnalysisStoreOptions, b as domainEvidencePattern, c as TraceInsightFinding, d as TraceInsightQualityGate, f as TraceInsightQuestion, g as buildTraceInsightContext, h as TraceInsightTask, i as OtlpFileTraceStoreOptions, j as DEFAULT_REDACTION_RULES, k as OtlpFlatLine, l as TraceInsightPanelRole, m as TraceInsightSuite, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightReadiness, r as OtlpFileTraceStore, s as TraceInsightContext, t as ToolTraceMissingError, u as TraceInsightPromptInput, v as defaultTraceInsightPanel, w as tokenizeDomainWords, x as inferDomainKeywords, y as describeTraceInsightScope, z as OtlpResourceSpans } from "./store-tool-spans-CV0hRsUp.js";
10
10
  import { C as TOOL_LATENCY_MS, D as asNumber, E as applyLlmSpanOtlpAttributes, O as contextInputTokens, S as TOOL_ARGS_CAPTURED, T as TOOL_NAME_ATTR_KEYS, _ as LlmSpanOtlpInput, a as LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, b as RUN_COST_ATTR_KEYS, c as LLM_COST_USD, d as LLM_MODEL_ATTR_KEYS, f as LLM_MODEL_NAME, g as LLM_REASONING_TOKEN_ATTR_KEYS, h as LLM_REASONING_TOKENS, i as LLM_CACHE_WRITE_TOKENS, k as firstNumberAttr, l as LLM_INPUT_TOKENS, m as LLM_OUTPUT_TOKEN_ATTR_KEYS, n as LLM_CACHED_TOKENS, o as LLM_CONTEXT_TOKENS, p as LLM_OUTPUT_TOKENS, r as LLM_CACHED_TOKEN_ATTR_KEYS, s as LLM_COST_ATTR_KEYS, t as INPUT_VALUE, u as LLM_INPUT_TOKEN_ATTR_KEYS, v as OPENINFERENCE_SPAN_KIND, w as TOOL_NAME, x as SPAN_KIND_ATTR_KEYS, y as OUTPUT_VALUE } from "./attribute-vocabulary-DLJ6303h.js";
11
11
  import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-CJ_DX8vl.js";
12
- import "./index-BKShPTwZ.js";
13
- import { A as SearchSpanResult, B as ViewSpansResult, C as TRACE_ANALYSIS_LIMITS, D as DatasetOverview, E as DEFAULT_TRACE_ANALYST_BUDGETS, F as TraceAnalystFilters, H as ViewTraceResult, I as TraceAnalystSpan, L as TraceAnalystSpanKind, M as SpanMatchRecord, N as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, O as ErrorCluster, P as TraceAnalystByteBudgets, R as TraceAnalystSpanStatus, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, V as ViewTraceOversized, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore, z as TraceAnalystTraceSummary } from "./types-vXyshMwx.js";
14
- import { _ as TraceAnalysisToolDescriptor, g as TRACE_ANALYST_TOOL_NAMESPACE, h as BuildTraceAnalysisToolsOptions, v as buildTraceAnalysisToolDescriptors, y as traceAnalystFunctionGroup } from "./engine-DJqRKbhs.js";
12
+ import "./index-BgHYx1QG.js";
13
+ import { A as SearchSpanResult, B as ViewSpansResult, C as TRACE_ANALYSIS_LIMITS, D as DatasetOverview, E as DEFAULT_TRACE_ANALYST_BUDGETS, F as TraceAnalystFilters, H as ViewTraceResult, I as TraceAnalystSpan, L as TraceAnalystSpanKind, M as SpanMatchRecord, N as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, O as ErrorCluster, P as TraceAnalystByteBudgets, R as TraceAnalystSpanStatus, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, V as ViewTraceOversized, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore, z as TraceAnalystTraceSummary } from "./types-oAsa-fs4.js";
14
+ import { _ as TraceAnalysisToolDescriptor, g as TRACE_ANALYST_TOOL_NAMESPACE, h as BuildTraceAnalysisToolsOptions, v as buildTraceAnalysisToolDescriptors, y as traceAnalystFunctionGroup } from "./engine-Bc3seRPe.js";
15
15
  //#region src/trace/otlp-attributes.d.ts
16
16
  type ToolSpanOtlpInput = Pick<ToolSpan, 'toolName' | 'args' | 'argsCaptured' | 'result' | 'latencyMs'>;
17
17
  type OtlpSpanRole = 'AGENT' | 'CHAIN' | 'EVALUATOR' | 'GUARDRAIL' | 'LLM' | 'SPAN' | 'TOOL' | 'UNKNOWN';
@@ -0,0 +1,108 @@
1
+ //#region src/multishot/types.ts
2
+ var MultishotDriverEmptyError = class extends Error {
3
+ turn;
4
+ constructor(turn) {
5
+ super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`);
6
+ this.turn = turn;
7
+ this.name = "MultishotDriverEmptyError";
8
+ }
9
+ };
10
+ var MultishotFatalToolError = class extends Error {
11
+ constructor(message) {
12
+ super(message);
13
+ this.name = "MultishotFatalToolError";
14
+ }
15
+ };
16
+ var MultishotShotResultError = class extends Error {
17
+ constructor(reason) {
18
+ super(`multishot: shot returned an invalid MultishotResult — ${reason}`);
19
+ this.name = "MultishotShotResultError";
20
+ }
21
+ };
22
+ const MULTISHOT_ROLES = /* @__PURE__ */ new Set([
23
+ "user",
24
+ "assistant",
25
+ "tool"
26
+ ]);
27
+ /** Contract guard for the value a caller-supplied shot resolves with. The
28
+ * matrix writes per-cell artifacts, builds judge inputs, and meters cost from
29
+ * this value, so a malformed result must stop the cell instead of scoring a
30
+ * degraded one. Two silent degradations this closes: an artifact with no
31
+ * `type` matches neither the code nor the content artifact set, so the cell
32
+ * scores as though the artifact was never produced; a non-finite `costUsd`
33
+ * reaches `summary.totalCostUsd` and makes every cost number NaN.
34
+ *
35
+ * A rejected cell is still billed: the matrix cell reads the shot's own
36
+ * `costUsd` when it is a usable amount and declares that spend on the throw,
37
+ * so money the shot spent before returning a malformed result stays in the
38
+ * cumulative sum the cost ceiling reads. A result whose `costUsd` is itself
39
+ * malformed carries no usable amount, and the cell records as `uncaptured`.
40
+ *
41
+ * Every required field of `MultishotMessage` and `MultishotArtifact` is
42
+ * checked, including `toolCalls` elements and `invocation.args`. Optional
43
+ * fields are checked only when present. */
44
+ function assertMultishotShotResult(value) {
45
+ if (typeof value !== "object" || value === null) throw new MultishotShotResultError(`expected an object, received ${describeValue(value)}`);
46
+ const result = value;
47
+ if (!Array.isArray(result.transcript)) throw new MultishotShotResultError(`transcript must be an array, received ${describeValue(result.transcript)}`);
48
+ if (!Array.isArray(result.artifacts)) throw new MultishotShotResultError(`artifacts must be an array, received ${describeValue(result.artifacts)}`);
49
+ result.transcript.forEach(assertMessage);
50
+ result.artifacts.forEach(assertArtifact);
51
+ assertFiniteCount(result.toolCalls, "toolCalls");
52
+ assertFiniteCount(result.durationMs, "durationMs");
53
+ assertFiniteCount(result.costUsd, "costUsd");
54
+ if (result.costProvenance !== void 0) assertCostProvenance(result.costProvenance);
55
+ }
56
+ function assertCostProvenance(value) {
57
+ const row = requireRow(value, "costProvenance");
58
+ if (row.kind !== "observed" && row.kind !== "estimated" && row.kind !== "uncaptured") throw new MultishotShotResultError(`costProvenance.kind must be observed, estimated or uncaptured, received ${describeValue(row.kind)}`);
59
+ if (row.kind === "uncaptured") {
60
+ if (row.usd !== null) throw new MultishotShotResultError(`uncaptured costProvenance.usd must be null, received ${describeValue(row.usd)}`);
61
+ return;
62
+ }
63
+ assertFiniteCount(row.usd, "costProvenance.usd");
64
+ }
65
+ function assertMessage(value, index) {
66
+ const row = requireRow(value, `transcript[${index}]`);
67
+ if (typeof row.role !== "string" || !MULTISHOT_ROLES.has(row.role)) throw new MultishotShotResultError(`transcript[${index}].role must be user, assistant or tool, received ${describeValue(row.role)}`);
68
+ assertString(row.content, `transcript[${index}].content`);
69
+ if (row.toolCallId !== void 0) assertString(row.toolCallId, `transcript[${index}].toolCallId`);
70
+ if (row.toolCalls === void 0) return;
71
+ if (!Array.isArray(row.toolCalls)) throw new MultishotShotResultError(`transcript[${index}].toolCalls must be an array when present, received ${describeValue(row.toolCalls)}`);
72
+ row.toolCalls.forEach((call, callIndex) => {
73
+ const field = `transcript[${index}].toolCalls[${callIndex}]`;
74
+ const row = requireRow(call, field);
75
+ assertString(row.id, `${field}.id`);
76
+ assertString(row.name, `${field}.name`);
77
+ requireRow(row.args, `${field}.args`);
78
+ });
79
+ }
80
+ function assertArtifact(value, index) {
81
+ const row = requireRow(value, `artifacts[${index}]`);
82
+ assertString(row.type, `artifacts[${index}].type`);
83
+ assertString(row.content, `artifacts[${index}].content`);
84
+ assertFiniteCount(row.turn, `artifacts[${index}].turn`);
85
+ const invocation = requireRow(row.invocation, `artifacts[${index}].invocation`);
86
+ assertString(invocation.name, `artifacts[${index}].invocation.name`);
87
+ requireRow(invocation.args, `artifacts[${index}].invocation.args`);
88
+ }
89
+ function requireRow(value, field) {
90
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new MultishotShotResultError(`${field} must be an object, received ${describeValue(value)}`);
91
+ return value;
92
+ }
93
+ function assertString(value, field) {
94
+ if (typeof value !== "string") throw new MultishotShotResultError(`${field} must be a string, received ${describeValue(value)}`);
95
+ }
96
+ function assertFiniteCount(value, field) {
97
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new MultishotShotResultError(`${field} must be a finite number >= 0, received ${describeValue(value)}`);
98
+ }
99
+ function describeValue(value) {
100
+ if (value === null) return "null";
101
+ if (Array.isArray(value)) return "an array";
102
+ if (typeof value === "object") return "an object";
103
+ return `${typeof value} ${String(value)}`;
104
+ }
105
+ //#endregion
106
+ export { assertMultishotShotResult as i, MultishotFatalToolError as n, MultishotShotResultError as r, MultishotDriverEmptyError as t };
107
+
108
+ //# sourceMappingURL=types-B3jzCp0p.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types-B3jzCp0p.js","names":[],"sources":["../src/multishot/types.ts"],"sourcesContent":["// Public types for the multishot substrate.\n\nimport type { CostProvenance } from '../cost-ledger'\n\nexport interface MultishotMessage {\n role: 'user' | 'assistant' | 'tool'\n content: string\n toolCallId?: string\n toolCalls?: Array<{ id: string; name: string; args: Record<string, unknown> }>\n}\n\nexport interface MultishotArtifact {\n type: string\n turn: number\n invocation: { name: string; args: Record<string, unknown> }\n content: string\n}\n\nexport interface MultishotResult {\n transcript: MultishotMessage[]\n artifacts: MultishotArtifact[]\n toolCalls: number\n durationMs: number\n /** Known spend. A subtotal, not a total, when `costProvenance.kind` is\n * `uncaptured`. */\n costUsd: number\n /** Origin of `costUsd`. A shot that priced every call reports `estimated`\n * or `observed`; a shot with a call the router priced at nothing reports\n * `uncaptured`, and the matrix records the cell as under-counted instead of\n * presenting the subtotal as a complete estimate.\n *\n * Optional so an engine written before this field keeps working; the matrix\n * then judges the cell on judge receipts alone, as it did before. */\n costProvenance?: CostProvenance\n}\n\nexport interface MultishotToolDefinition {\n type: 'function'\n function: {\n name: string\n description: string\n parameters: Record<string, unknown>\n }\n}\n\n/** One chat-completion request the multishot loop issues for a single agent\n * (or driver) inference step. Mirrors the OpenAI-compat body the loop would\n * otherwise POST to the Tangle router. */\nexport interface MultishotTransportRequest {\n model: string\n messages: Array<Record<string, unknown>>\n tools?: MultishotToolDefinition[]\n temperature?: number\n maxTokens?: number\n signal?: AbortSignal\n}\n\nexport interface MultishotTransportToolCall {\n id: string\n type: 'function'\n function: { name: string; arguments: string }\n}\n\nexport interface MultishotTransportResponse {\n message: { content?: string | null; tool_calls?: MultishotTransportToolCall[] }\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n /** Actual spend for this call. When omitted, the loop meters cost from\n * `usage` via the per-model router estimator (estimateRouterCost). */\n costUsd?: number\n}\n\n/** Execution seam for one leg of the multishot loop. When provided, it\n * replaces the internal router HTTP call for that leg — the loop still owns\n * turn scheduling, tool dispatch, transcript capture, and cost metering.\n * agent-eval has no dependency on agent-runtime; adapt agent-runtime's\n * resolveAgentBackend (or any sandbox/cli-bridge/router client) into this\n * signature product-side. */\nexport type MultishotTransport = (\n req: MultishotTransportRequest,\n) => Promise<MultishotTransportResponse>\n\nexport type MultishotToolExecutor = (\n args: Record<string, unknown>,\n ctx: { apiKey: string; baseUrl: string; signal?: AbortSignal },\n) => Promise<{ content: string; costUsd: number }>\n\nexport interface MultishotPersona {\n /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */\n id: string\n /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */\n [k: string]: unknown\n}\n\n/**\n * Persona-shaping callbacks. Both are OPTIONAL: when omitted, the loop derives\n * them from the `AgentProfile` + persona payload (see `defaultShapeFromProfile`)\n * so a pure-profile call — `runMultishot({ profile, persona })` — works with no\n * role-builder functions. Provide callbacks only to override the derived shape.\n */\nexport interface MultishotShape<TPersona extends MultishotPersona> {\n /** Opening user message (turn 0) — the persona's first ask. */\n buildOpener?: (persona: TPersona) => string\n /** System prompt the driver LLM uses to roleplay the persona. Should set\n * voice, goals, constraints, time-pressure, and the \"never go silent\" rule. */\n buildDriverSystemPrompt?: (persona: TPersona) => string\n}\n\nexport class MultishotDriverEmptyError extends Error {\n constructor(public readonly turn: number) {\n super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`)\n this.name = 'MultishotDriverEmptyError'\n }\n}\n\nexport class MultishotFatalToolError extends Error {\n constructor(message: string) {\n super(message)\n this.name = 'MultishotFatalToolError'\n }\n}\n\nexport class MultishotShotResultError extends Error {\n constructor(reason: string) {\n super(`multishot: shot returned an invalid MultishotResult — ${reason}`)\n this.name = 'MultishotShotResultError'\n }\n}\n\nconst MULTISHOT_ROLES = new Set(['user', 'assistant', 'tool'])\n\n/** Contract guard for the value a caller-supplied shot resolves with. The\n * matrix writes per-cell artifacts, builds judge inputs, and meters cost from\n * this value, so a malformed result must stop the cell instead of scoring a\n * degraded one. Two silent degradations this closes: an artifact with no\n * `type` matches neither the code nor the content artifact set, so the cell\n * scores as though the artifact was never produced; a non-finite `costUsd`\n * reaches `summary.totalCostUsd` and makes every cost number NaN.\n *\n * A rejected cell is still billed: the matrix cell reads the shot's own\n * `costUsd` when it is a usable amount and declares that spend on the throw,\n * so money the shot spent before returning a malformed result stays in the\n * cumulative sum the cost ceiling reads. A result whose `costUsd` is itself\n * malformed carries no usable amount, and the cell records as `uncaptured`.\n *\n * Every required field of `MultishotMessage` and `MultishotArtifact` is\n * checked, including `toolCalls` elements and `invocation.args`. Optional\n * fields are checked only when present. */\nexport function assertMultishotShotResult(value: unknown): asserts value is MultishotResult {\n if (typeof value !== 'object' || value === null) {\n throw new MultishotShotResultError(`expected an object, received ${describeValue(value)}`)\n }\n const result = value as Record<string, unknown>\n if (!Array.isArray(result.transcript)) {\n throw new MultishotShotResultError(\n `transcript must be an array, received ${describeValue(result.transcript)}`,\n )\n }\n if (!Array.isArray(result.artifacts)) {\n throw new MultishotShotResultError(\n `artifacts must be an array, received ${describeValue(result.artifacts)}`,\n )\n }\n result.transcript.forEach(assertMessage)\n result.artifacts.forEach(assertArtifact)\n assertFiniteCount(result.toolCalls, 'toolCalls')\n assertFiniteCount(result.durationMs, 'durationMs')\n assertFiniteCount(result.costUsd, 'costUsd')\n if (result.costProvenance !== undefined) assertCostProvenance(result.costProvenance)\n}\n\nfunction assertCostProvenance(value: unknown): void {\n const row = requireRow(value, 'costProvenance')\n if (row.kind !== 'observed' && row.kind !== 'estimated' && row.kind !== 'uncaptured') {\n throw new MultishotShotResultError(\n `costProvenance.kind must be observed, estimated or uncaptured, received ${describeValue(row.kind)}`,\n )\n }\n // The matrix reads the amount from `costUsd`; `usd` only has to agree with\n // the kind, so an uncaptured provenance cannot smuggle in a total.\n if (row.kind === 'uncaptured') {\n if (row.usd !== null) {\n throw new MultishotShotResultError(\n `uncaptured costProvenance.usd must be null, received ${describeValue(row.usd)}`,\n )\n }\n return\n }\n assertFiniteCount(row.usd, 'costProvenance.usd')\n}\n\nfunction assertMessage(value: unknown, index: number): void {\n const row = requireRow(value, `transcript[${index}]`)\n if (typeof row.role !== 'string' || !MULTISHOT_ROLES.has(row.role)) {\n throw new MultishotShotResultError(\n `transcript[${index}].role must be user, assistant or tool, received ${describeValue(row.role)}`,\n )\n }\n assertString(row.content, `transcript[${index}].content`)\n if (row.toolCallId !== undefined) {\n assertString(row.toolCallId, `transcript[${index}].toolCallId`)\n }\n if (row.toolCalls === undefined) return\n if (!Array.isArray(row.toolCalls)) {\n throw new MultishotShotResultError(\n `transcript[${index}].toolCalls must be an array when present, received ${describeValue(row.toolCalls)}`,\n )\n }\n row.toolCalls.forEach((call, callIndex) => {\n const field = `transcript[${index}].toolCalls[${callIndex}]`\n const row = requireRow(call, field)\n assertString(row.id, `${field}.id`)\n assertString(row.name, `${field}.name`)\n requireRow(row.args, `${field}.args`)\n })\n}\n\nfunction assertArtifact(value: unknown, index: number): void {\n const row = requireRow(value, `artifacts[${index}]`)\n assertString(row.type, `artifacts[${index}].type`)\n assertString(row.content, `artifacts[${index}].content`)\n assertFiniteCount(row.turn, `artifacts[${index}].turn`)\n const invocation = requireRow(row.invocation, `artifacts[${index}].invocation`)\n assertString(invocation.name, `artifacts[${index}].invocation.name`)\n requireRow(invocation.args, `artifacts[${index}].invocation.args`)\n}\n\nfunction requireRow(value: unknown, field: string): Record<string, unknown> {\n if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n throw new MultishotShotResultError(\n `${field} must be an object, received ${describeValue(value)}`,\n )\n }\n return value as Record<string, unknown>\n}\n\nfunction assertString(value: unknown, field: string): void {\n if (typeof value !== 'string') {\n throw new MultishotShotResultError(\n `${field} must be a string, received ${describeValue(value)}`,\n )\n }\n}\n\nfunction assertFiniteCount(value: unknown, field: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {\n throw new MultishotShotResultError(\n `${field} must be a finite number >= 0, received ${describeValue(value)}`,\n )\n }\n}\n\nfunction describeValue(value: unknown): string {\n if (value === null) return 'null'\n if (Array.isArray(value)) return 'an array'\n if (typeof value === 'object') return 'an object'\n return `${typeof value} ${String(value)}`\n}\n"],"mappings":";AA2GA,IAAa,4BAAb,cAA+C,MAAM;CACvB;CAA5B,YAAY,MAA8B;EACxC,MAAM,0DAA0D,KAAK,gBAAgB;EAD3D,KAAA,OAAA;EAE1B,KAAK,OAAO;CACd;AACF;AAEA,IAAa,0BAAb,cAA6C,MAAM;CACjD,YAAY,SAAiB;EAC3B,MAAM,OAAO;EACb,KAAK,OAAO;CACd;AACF;AAEA,IAAa,2BAAb,cAA8C,MAAM;CAClD,YAAY,QAAgB;EAC1B,MAAM,yDAAyD,QAAQ;EACvE,KAAK,OAAO;CACd;AACF;AAEA,MAAM,kCAAkB,IAAI,IAAI;CAAC;CAAQ;CAAa;AAAM,CAAC;;;;;;;;;;;;;;;;;;AAmB7D,SAAgB,0BAA0B,OAAkD;CAC1F,IAAI,OAAO,UAAU,YAAY,UAAU,MACzC,MAAM,IAAI,yBAAyB,gCAAgC,cAAc,KAAK,GAAG;CAE3F,MAAM,SAAS;CACf,IAAI,CAAC,MAAM,QAAQ,OAAO,UAAU,GAClC,MAAM,IAAI,yBACR,yCAAyC,cAAc,OAAO,UAAU,GAC1E;CAEF,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GACjC,MAAM,IAAI,yBACR,wCAAwC,cAAc,OAAO,SAAS,GACxE;CAEF,OAAO,WAAW,QAAQ,aAAa;CACvC,OAAO,UAAU,QAAQ,cAAc;CACvC,kBAAkB,OAAO,WAAW,WAAW;CAC/C,kBAAkB,OAAO,YAAY,YAAY;CACjD,kBAAkB,OAAO,SAAS,SAAS;CAC3C,IAAI,OAAO,mBAAmB,KAAA,GAAW,qBAAqB,OAAO,cAAc;AACrF;AAEA,SAAS,qBAAqB,OAAsB;CAClD,MAAM,MAAM,WAAW,OAAO,gBAAgB;CAC9C,IAAI,IAAI,SAAS,cAAc,IAAI,SAAS,eAAe,IAAI,SAAS,cACtE,MAAM,IAAI,yBACR,2EAA2E,cAAc,IAAI,IAAI,GACnG;CAIF,IAAI,IAAI,SAAS,cAAc;EAC7B,IAAI,IAAI,QAAQ,MACd,MAAM,IAAI,yBACR,wDAAwD,cAAc,IAAI,GAAG,GAC/E;EAEF;CACF;CACA,kBAAkB,IAAI,KAAK,oBAAoB;AACjD;AAEA,SAAS,cAAc,OAAgB,OAAqB;CAC1D,MAAM,MAAM,WAAW,OAAO,cAAc,MAAM,EAAE;CACpD,IAAI,OAAO,IAAI,SAAS,YAAY,CAAC,gBAAgB,IAAI,IAAI,IAAI,GAC/D,MAAM,IAAI,yBACR,cAAc,MAAM,mDAAmD,cAAc,IAAI,IAAI,GAC/F;CAEF,aAAa,IAAI,SAAS,cAAc,MAAM,UAAU;CACxD,IAAI,IAAI,eAAe,KAAA,GACrB,aAAa,IAAI,YAAY,cAAc,MAAM,aAAa;CAEhE,IAAI,IAAI,cAAc,KAAA,GAAW;CACjC,IAAI,CAAC,MAAM,QAAQ,IAAI,SAAS,GAC9B,MAAM,IAAI,yBACR,cAAc,MAAM,sDAAsD,cAAc,IAAI,SAAS,GACvG;CAEF,IAAI,UAAU,SAAS,MAAM,cAAc;EACzC,MAAM,QAAQ,cAAc,MAAM,cAAc,UAAU;EAC1D,MAAM,MAAM,WAAW,MAAM,KAAK;EAClC,aAAa,IAAI,IAAI,GAAG,MAAM,IAAI;EAClC,aAAa,IAAI,MAAM,GAAG,MAAM,MAAM;EACtC,WAAW,IAAI,MAAM,GAAG,MAAM,MAAM;CACtC,CAAC;AACH;AAEA,SAAS,eAAe,OAAgB,OAAqB;CAC3D,MAAM,MAAM,WAAW,OAAO,aAAa,MAAM,EAAE;CACnD,aAAa,IAAI,MAAM,aAAa,MAAM,OAAO;CACjD,aAAa,IAAI,SAAS,aAAa,MAAM,UAAU;CACvD,kBAAkB,IAAI,MAAM,aAAa,MAAM,OAAO;CACtD,MAAM,aAAa,WAAW,IAAI,YAAY,aAAa,MAAM,aAAa;CAC9E,aAAa,WAAW,MAAM,aAAa,MAAM,kBAAkB;CACnE,WAAW,WAAW,MAAM,aAAa,MAAM,kBAAkB;AACnE;AAEA,SAAS,WAAW,OAAgB,OAAwC;CAC1E,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GACpE,MAAM,IAAI,yBACR,GAAG,MAAM,+BAA+B,cAAc,KAAK,GAC7D;CAEF,OAAO;AACT;AAEA,SAAS,aAAa,OAAgB,OAAqB;CACzD,IAAI,OAAO,UAAU,UACnB,MAAM,IAAI,yBACR,GAAG,MAAM,8BAA8B,cAAc,KAAK,GAC5D;AAEJ;AAEA,SAAS,kBAAkB,OAAgB,OAAqB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAClE,MAAM,IAAI,yBACR,GAAG,MAAM,0CAA0C,cAAc,KAAK,GACxE;AAEJ;AAEA,SAAS,cAAc,OAAwB;CAC7C,IAAI,UAAU,MAAM,OAAO;CAC3B,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO;CACjC,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,OAAO,GAAG,OAAO,MAAM,GAAG,OAAO,KAAK;AACxC"}
@@ -1,5 +1,5 @@
1
1
  import { s as RunSplitTag } from "./run-record-DVV82Gwh.js";
2
- import { R as Scenario, d as DispatchContext } from "./types-BjsNDR49.js";
2
+ import { R as Scenario, d as DispatchContext } from "./types-DABZgDGV.js";
3
3
  //#region src/benchmarks/types.d.ts
4
4
  type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
5
5
  type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
@@ -90,4 +90,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
90
90
  declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
91
91
  //#endregion
92
92
  export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
93
- //# sourceMappingURL=types-C4bSVIr7.d.ts.map
93
+ //# sourceMappingURL=types-CWcPguwd.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-C4bSVIr7.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
1
+ {"version":3,"file":"types-CWcPguwd.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
@@ -1,7 +1,7 @@
1
1
  import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
2
2
  import { u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
3
- import { E as LlmCallMetadata } from "./types-Cx3YUh2r.js";
4
- import { _ as ProposalFinding } from "./types-vXyshMwx.js";
3
+ import { E as LlmCallMetadata } from "./types-XBRdmbj8.js";
4
+ import { _ as ProposalFinding } from "./types-oAsa-fs4.js";
5
5
  //#region src/campaign/types.d.ts
6
6
  /** Stable identifier + kind tag for any scenario. Consumers
7
7
  * extend with their per-domain payload (persona, task, requirement, ...). */
@@ -626,4 +626,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
626
626
  }
627
627
  //#endregion
628
628
  export { LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, GateDecision as _, CampaignResult as a, GenerationRecord as b, CampaignTraceWriter as c, DispatchContext as d, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, GateCheckStatus as m, CampaignArtifactWriter as n, CampaignScenarioIdentity as o, Gate as p, CampaignCellResult as r, CampaignTokenUsage as s, CampaignAggregates as t, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
629
- //# sourceMappingURL=types-BjsNDR49.d.ts.map
629
+ //# sourceMappingURL=types-DABZgDGV.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-BjsNDR49.d.ts","names":[],"sources":["../src/campaign/types.ts"],"mappings":";;;;;;;UAgCiB;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;EAIV;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;iBAKc,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;WACjC,gBAAgB;WAChB,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;UAGe;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
1
+ {"version":3,"file":"types-DABZgDGV.d.ts","names":[],"sources":["../src/campaign/types.ts"],"mappings":";;;;;;;UAgCiB;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;EAIV;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;iBAKc,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;WACjC,gBAAgB;WAChB,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;UAGe;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
@@ -204,6 +204,21 @@ interface ServedModelCheck {
204
204
  /** True for every verdict except `exact` and `alias`. */
205
205
  substituted: boolean;
206
206
  }
207
+ /**
208
+ * Classify one requested/served pair. Pure — no I/O — so it is safe inside
209
+ * response handlers, reducers, and CI gates.
210
+ *
211
+ * `served` is the id echoed by the provider (OpenAI-compatible bodies put it
212
+ * at `model`). `null`/`undefined` means the body omitted it; that is
213
+ * `unreported`, NOT a pass — a provider that does not name what answered has
214
+ * not proven identity, and a transport that drops the field must not read as
215
+ * agreement.
216
+ */
217
+ declare function checkServedModel(requested: string, served: string | null | undefined): ServedModelCheck;
218
+ declare class ModelSubstitutionError extends AgentEvalError {
219
+ readonly checks: ReadonlyArray<ServedModelCheck>;
220
+ constructor(message: string, checks: ReadonlyArray<ServedModelCheck>);
221
+ }
207
222
  interface AssertServedModelOptions {
208
223
  /**
209
224
  * Accept a different model of the same provider family (e.g. requested
@@ -219,6 +234,50 @@ interface AssertServedModelOptions {
219
234
  /** Prefixed to the thrown message, e.g. the judge or campaign cell name. */
220
235
  context?: string;
221
236
  }
237
+ /**
238
+ * The one place the accept/reject policy lives, so a caller that reports
239
+ * substitution (a preflight table, a run record) and a caller that throws on it
240
+ * can never drift apart. A cross-family substitution is never acceptable.
241
+ */
242
+ declare function servedModelAcceptable(check: ServedModelCheck, opts?: AssertServedModelOptions): boolean;
243
+ /**
244
+ * Throw `ModelSubstitutionError` unless the served id is the requested model.
245
+ * Returns the check on success so callers can record the served id alongside
246
+ * the result.
247
+ */
248
+ declare function assertServedModel(requested: string, served: string | null | undefined, opts?: AssertServedModelOptions): ServedModelCheck;
249
+ /**
250
+ * Batch form: check every pair and throw naming EVERY substitution, so one
251
+ * failure does not hide the rest. Returns all checks on success.
252
+ */
253
+ declare function assertServedModels(pairs: ReadonlyArray<{
254
+ requested: string;
255
+ served: string | null | undefined;
256
+ }>, opts?: AssertServedModelOptions): ServedModelCheck[];
257
+ interface AssertCrossFamilyServedOptions extends AssertServedModelOptions {
258
+ /** Minimum distinct SERVED families required. Default 2. */
259
+ minFamilies?: number;
260
+ /** Count `unknown`-family served ids toward the total. Default false. */
261
+ allowUnknown?: boolean;
262
+ }
263
+ declare class ServedCrossFamilyError extends AgentEvalError {
264
+ readonly families: JudgeFamily[];
265
+ readonly checks: ReadonlyArray<ServedModelCheck>;
266
+ constructor(message: string, families: JudgeFamily[], checks: ReadonlyArray<ServedModelCheck>);
267
+ }
268
+ /**
269
+ * Family-diversity rule over the models that actually ANSWERED.
270
+ *
271
+ * `assertCrossFamily` (../judge-families) reads the requested ids and so
272
+ * cannot see a gateway that answers three "different" requests from one
273
+ * provider. This one asserts no substitution first, then counts families from
274
+ * the served ids — a panel that collapsed to one family under the hood fails
275
+ * here even though its request list looked diverse.
276
+ */
277
+ declare function assertCrossFamilyServed(pairs: ReadonlyArray<{
278
+ requested: string;
279
+ served: string | null | undefined;
280
+ }>, opts?: AssertCrossFamilyServedOptions): JudgeFamily[];
222
281
  //#endregion
223
282
  //#region src/llm-client.d.ts
224
283
  interface LlmMessage {
@@ -782,5 +841,5 @@ interface CheckResult {
782
841
  actual: string;
783
842
  }
784
843
  //#endregion
785
- export { InMemoryRawProviderSinkOptions as $, LlmClientOptions as A, isTransientLlmError as B, SandboxSdkTransportOpts as C, LlmCallRequest as D, LlmCallMetadata as E, assertLlmRoute as F, AssertCrossFamilyOptions as G, probeLlm as H, callLlm as I, assertCrossFamily as J, CrossFamilyError as K, callLlmJson as L, LlmResponseError as M, LlmRouteRequirements as N, LlmCallResult as O, LlmUsage as P, InMemoryRawProviderSink as Q, costReceiptFromLlm as R, RouterTransportOpts as S, LlmCallError as T, stripFencedJson as U, maximumChargeForLlmRequest as V, ServedModelCheck as W, FileSystemRawProviderSink as X, judgeFamily as Y, FileSystemRawProviderSinkOptions as Z, CliBridgeTransportOpts as _, JudgeInput as a, RawProviderSinkFilter as at, DirectProviderTransportOpts as b, PersonaConfig as c, Scenario as d, NoopRawProviderSink as et, ChatCallOpts as f, ChatTransport as g, ChatResponse as h, JudgeFn as i, RawProviderSink as it, LlmMessage as j, LlmClient as k, ProductClientConfig as l, ChatRequest as m, CompletionCriterion as n, RawProviderDirection as nt, JudgeRubric as o, defaultProviderRedactor as ot, ChatClient as p, JudgeFamily as q, DriverState as r, RawProviderEvent as rt, JudgeScore as s, providerFromBaseUrl as st, CheckResult as t, ProviderRedactor as tt, RouteMap as u, CreateChatClientOpts as v, createChatClient as w, MockTransportOpts as x, CustomTransportOpts as y, costReceiptFromLlmError as z };
786
- //# sourceMappingURL=types-Cx3YUh2r.d.ts.map
844
+ export { checkServedModel as $, LlmClientOptions as A, isTransientLlmError as B, SandboxSdkTransportOpts as C, LlmCallRequest as D, LlmCallMetadata as E, assertLlmRoute as F, AssertServedModelOptions as G, probeLlm as H, callLlm as I, ServedModelCheck as J, ModelSubstitutionError as K, callLlmJson as L, LlmResponseError as M, LlmRouteRequirements as N, LlmCallResult as O, LlmUsage as P, assertServedModels as Q, costReceiptFromLlm as R, RouterTransportOpts as S, LlmCallError as T, stripFencedJson as U, maximumChargeForLlmRequest as V, AssertCrossFamilyServedOptions as W, assertCrossFamilyServed as X, ServedModelVerdict as Y, assertServedModel as Z, CliBridgeTransportOpts as _, providerFromBaseUrl as _t, JudgeInput as a, judgeFamily as at, DirectProviderTransportOpts as b, PersonaConfig as c, InMemoryRawProviderSink as ct, Scenario as d, ProviderRedactor as dt, servedModelAcceptable as et, ChatCallOpts as f, RawProviderDirection as ft, ChatTransport as g, defaultProviderRedactor as gt, ChatResponse as h, RawProviderSinkFilter as ht, JudgeFn as i, assertCrossFamily as it, LlmMessage as j, LlmClient as k, ProductClientConfig as l, InMemoryRawProviderSinkOptions as lt, ChatRequest as m, RawProviderSink as mt, CompletionCriterion as n, CrossFamilyError as nt, JudgeRubric as o, FileSystemRawProviderSink as ot, ChatClient as p, RawProviderEvent as pt, ServedCrossFamilyError as q, DriverState as r, JudgeFamily as rt, JudgeScore as s, FileSystemRawProviderSinkOptions as st, CheckResult as t, AssertCrossFamilyOptions as tt, RouteMap as u, NoopRawProviderSink as ut, CreateChatClientOpts as v, createChatClient as w, MockTransportOpts as x, CustomTransportOpts as y, costReceiptFromLlmError as z };
845
+ //# sourceMappingURL=types-XBRdmbj8.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types-XBRdmbj8.d.ts","names":[],"sources":["../src/trace/raw-provider-sink.ts","../src/judge-families.ts","../src/integrity/served-model.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA8BY;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;;iBAmB5C,wBAAwB,OAAO,mBAAmB;UA8CjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;;;;;;;;;;;KC5QxB;;;;;;;iBAkEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCnFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;iBA4Cc,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;UAOzB;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;UCjPc;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;KAI7C;UAEK;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;EACrC;EACA;;EAEA,WAAW;;EAEX;;;;;;iBAOc,2BACd,SAAS,KAAK,iFACd,UAAS,mBACR;UAgCc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;EAEA,KAAK;;KAGK,kBAAkB,KAAK;;iBAGnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA2Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC;;EAEf;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;EAEA;;EAEA,qBAAqB;;;;;;;EAOrB;;;;;;EAMA;;EAEA,WAAW;;EAEX,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBAsEhB,oBAAoB;;;;;;iBA6IpB,gBAAgB;;;;;;iBA6EV,QACpB,KAAK,gBACL,OAAM,mBACL,QAAQ;;;;;;;iBAoVW,YAAY,aAChC,KAAK,gBACL,OAAM,mBACL;EAAU,OAAO;EAAG,QAAQ;;UA6Ed;;;;;;;EAOf;;;;;EAKA,kBAAkB,eAAe;;EAEjC,kBAAkB,eAAe;;EAEjC;;;;;EAKA;;;;;;;;;;;iBAYc,eAAe,MAAM,kBAAkB,MAAK;;;;;;;;;;;;;;;;;;iBA6EtC,SACpB,eACA,OAAM;EAAqB;IAC1B;EACD;EACA;EACA;;EAEA;;EAEA;;;;;;;cAoCW;WACF;mBACQ;EAEjB,YAAY,OAAM;EAKlB,KAAK,KAAK,gBAAgB,MAAM,mBAAmB,QAAQ;EAK3D,SAAS,aACP,KAAK,gBACL,MAAM,mBACL;IAAU,OAAO;IAAG,QAAQ;;;;;;;;UCjqChB;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAQK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,sBACA,yBACA,8BACA,0BACA,sBACA;UAEM;EACR;;EAEA;;UAGe,4BAA4B;EAC3C;EACA;EACA;;UAGe,+BAA+B;EAC9C;EACA;EACA;;UAGe,oCAAoC;EACnD;EACA;EACA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;UAO1C,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UCjH7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAmBe;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UASe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAyBe;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;UAGJ;EACf;EACA;;;;;;;;;;KAWU;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB,mBAAmB;EACnB;EACA;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
@@ -1,6 +1,6 @@
1
1
  import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
2
2
  import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
3
- import { a as JudgeInput, p as ChatClient } from "./types-Cx3YUh2r.js";
3
+ import { a as JudgeInput, p as ChatClient } from "./types-XBRdmbj8.js";
4
4
  import { RE2JS } from "re2js";
5
5
  //#region src/trace-analyst/types.d.ts
6
6
  /**
@@ -594,4 +594,4 @@ type AnalystRunEvent = {
594
594
  };
595
595
  //#endregion
596
596
  export { SearchSpanResult as A, ViewSpansResult as B, TRACE_ANALYSIS_LIMITS as C, DatasetOverview as D, DEFAULT_TRACE_ANALYST_BUDGETS as E, TraceAnalystFilters as F, ViewTraceResult as H, TraceAnalystSpan as I, TraceAnalystSpanKind as L, SpanMatchRecord as M, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as N, ErrorCluster as O, TraceAnalystByteBudgets as P, TraceAnalystSpanStatus as R, BoundedTraceAnalysisStoreOptions as S, TraceAnalysisStoreContext as T, ViewTraceOversized as V, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, SearchTraceResult as j, QueryTracesPage as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, TraceAnalysisStore as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystTraceSummary as z };
597
- //# sourceMappingURL=types-vXyshMwx.d.ts.map
597
+ //# sourceMappingURL=types-oAsa-fs4.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-vXyshMwx.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KAgBY;KAUA;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC7MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;;;;;;;UAUM;EACf,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;;;;UC/DH;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;KAGD;;KAGA;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
1
+ {"version":3,"file":"types-oAsa-fs4.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KAgBY;KAUA;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC7MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;;;;;;;UAUM;EACf,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;;;;UC/DH;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;KAGD;;KAGA;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
@@ -1,7 +1,7 @@
1
1
  import { c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
2
- import { A as LlmClientOptions, N as LlmRouteRequirements } from "../types-Cx3YUh2r.js";
2
+ import { A as LlmClientOptions, N as LlmRouteRequirements } from "../types-XBRdmbj8.js";
3
3
  import { s as TraceStore } from "../store-CT9YIIve.js";
4
- import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-D9vYSob_.js";
4
+ import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-Cg9EDxKB.js";
5
5
  import { z } from "zod";
6
6
  import { ServerType } from "@hono/node-server";
7
7
  import { Hono } from "hono";
@@ -1,2 +1,2 @@
1
- import { A as TraceEventSchema, C as HealthResponseSchema, D as RubricDimensionSchema, E as ListRubricsResponseSchema, F as hashRubric, M as TracesIngestResponseSchema, N as VersionResponseSchema, O as RubricInfoSchema, P as WIRE_VERSION, S as FeedbackTrajectorySchema, T as JudgeResultSchema, _ as ErrorResponseSchema, a as runRpcBatch, b as FeedbackIngestResponseSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as listBuiltinRubrics, h as getBuiltinRubric, i as dispatchRpc, j as TracesIngestRequestSchema, k as RubricSchema, l as handleFeedbackIngest, m as BUILTIN_RUBRICS, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FailureModeSchema, w as JudgeRequestSchema, x as FeedbackLabelSchema, y as FeedbackAttemptSchema } from "../server-ulsOdrTI.js";
1
+ import { A as TraceEventSchema, C as HealthResponseSchema, D as RubricDimensionSchema, E as ListRubricsResponseSchema, F as hashRubric, M as TracesIngestResponseSchema, N as VersionResponseSchema, O as RubricInfoSchema, P as WIRE_VERSION, S as FeedbackTrajectorySchema, T as JudgeResultSchema, _ as ErrorResponseSchema, a as runRpcBatch, b as FeedbackIngestResponseSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as listBuiltinRubrics, h as getBuiltinRubric, i as dispatchRpc, j as TracesIngestRequestSchema, k as RubricSchema, l as handleFeedbackIngest, m as BUILTIN_RUBRICS, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FailureModeSchema, w as JudgeRequestSchema, x as FeedbackLabelSchema, y as FeedbackAttemptSchema } from "../server-cQXve8i4.js";
2
2
  export { BUILTIN_RUBRICS, ErrorResponseSchema, FailureModeSchema, FeedbackAttemptSchema, FeedbackIngestResponseSchema, FeedbackLabelSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RubricDimensionSchema, RubricInfoSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
@@ -0,0 +1,102 @@
1
+ # Multishot golden records
2
+
3
+ `@tangle-network/agent-eval/multishot/golden` holds frozen recordings of what a multishot conversation engine produces on a closed set of deterministic scenarios.
4
+ Point your engine at them and any orchestration drift surfaces as a named field.
5
+
6
+ ## What a record holds
7
+
8
+ Each scenario records two things.
9
+
10
+ **The request ledger** — every transport call each leg received, in issue order: the model, the temperature, the token budget, the advertised tool definitions, and the full message log.
11
+ Tools are compared by value, not by array identity: an engine may rebuild the array, but a changed name, description or parameter schema changes what the agent is offered.
12
+ This is where two orchestrators diverge without their return value changing: a wrong follow-up token budget, a driver rotation that stops one model early, a point-of-view translation that drops a tool row from the driver's view.
13
+
14
+ **The outcome** — either the `MultishotResult` without its wall-clock `durationMs`, or the throw reduced to its constructor name, its message, and the cell spend it declares for the cost ceiling.
15
+
16
+ The matrix scenarios add the returned `MatrixResult`, the judge calls, and every file the run persisted under its run directory.
17
+
18
+ Wall-clock and run-identity keys (`durationMs`, `meanDurationMs`, `matrixId`, `runId`) are removed before comparison — no two runs agree on them.
19
+ The summary Markdown keeps its text with the rendered duration masked.
20
+
21
+ ## Use it
22
+
23
+ ```ts
24
+ import { describe, it } from 'vitest'
25
+ import {
26
+ assertMultishotGoldenScenario,
27
+ multishotGoldenScenarios,
28
+ } from '@tangle-network/agent-eval/multishot/golden'
29
+ import { runMyEngine } from './my-engine'
30
+
31
+ describe('my engine reproduces the multishot golden records', () => {
32
+ for (const scenario of multishotGoldenScenarios()) {
33
+ it(scenario.description, async () => {
34
+ await assertMultishotGoldenScenario({ engine: runMyEngine, scenario })
35
+ })
36
+ }
37
+ })
38
+ ```
39
+
40
+ `assertMultishotGoldenScenario` throws `MultishotGoldenMismatchError` listing every field that moved.
41
+ `checkMultishotGoldenScenario` returns the same report without throwing, for a caller that wants to inspect it.
42
+ `checkMultishotGolden` runs every SHOT scenario in one call; the matrix scenarios have their own pair below.
43
+ It refuses an `only` id the catalog does not hold, so a stale id after a rename stops the check instead of greening a run of zero scenarios.
44
+
45
+ The matrix pair is `assertMultishotMatrixGoldenScenario` / `checkMultishotMatrixGoldenScenario`; both take a `runDir` the engine may write into, and both install a deterministic judge wire on `globalThis.fetch` for the duration of the run.
46
+ That wire is process-wide, so run matrix checks serially within one process and keep other fetch traffic out of it.
47
+ Both rules are enforced, not just documented: a second concurrent install throws, and the wire fails loud on any request it does not recognise rather than answering it.
48
+
49
+ ## Determinism rules
50
+
51
+ Every scenario is a closed system: scripted transports, scripted tool executors, a fixed persona and profile, fixed token budgets.
52
+ No network, no random number, and no clock in a COMPARED field — the fixture envelope carries a `recordedAt` stamp as provenance, and nothing compares it.
53
+ Matrix cells run one at a time, so the request ledger is a property of the conversation engine rather than of how two engines interleave their microtasks.
54
+
55
+ Judge calls fan out across three slots through `Promise.all`, so their issue order is a detail of the cell body rather than observable behaviour.
56
+ Their content is behaviour, so they are compared as a set with a stable order.
57
+
58
+ ## Records are frozen
59
+
60
+ A version file is written once and never edited.
61
+ A golden record that can be regenerated over itself proves nothing: a regression would simply be re-recorded as the new truth.
62
+
63
+ `scripts/record-multishot-golden.ts` refuses to overwrite an existing version.
64
+ A deliberate behaviour change mints a NEW version file, registers it in `src/multishot/golden/records/index.ts` beside the old one, and moves `CURRENT_MULTISHOT_GOLDEN_VERSION`.
65
+ The diff between the two files is the reviewable evidence of what moved, and the previous contract stays runnable through `goldenRecords('v1')`.
66
+
67
+ ## Regenerate
68
+
69
+ The recorder never picks an engine for you: the reference has to be named.
70
+ Once the loop leaves this package there is no engine here to default to, and a script that silently picked one would record the wrong thing.
71
+
72
+ ```bash
73
+ pnpm tsx scripts/record-multishot-golden.ts \
74
+ --version v2 \
75
+ --engine <module>#<export> \
76
+ --matrix-engine <module>#<export>
77
+ ```
78
+
79
+ `<module>` resolves from the repository root and may point outside it, so a reference engine that lives in a consumer works:
80
+
81
+ ```bash
82
+ pnpm tsx scripts/record-multishot-golden.ts \
83
+ --version v2 \
84
+ --engine ../gtm-agent/eval/lib/multishot-graph.ts#runMultishotGraph \
85
+ --matrix-engine ../gtm-agent/eval/lib/multishot-matrix-graph.ts#runMultishotMatrixGraph
86
+ ```
87
+
88
+ Then register the new file in `src/multishot/golden/records/index.ts` and move `CURRENT_MULTISHOT_GOLDEN_VERSION` to it.
89
+ The recorder writes the file and nothing else; until it is registered, `goldenRecords()` still resolves the old version and the new file is inert.
90
+
91
+ Every scenario is captured twice and the two captures must agree.
92
+ A scenario that is not reproducible cannot detect a regression, so an unstable capture fails the run instead of freezing a coin flip.
93
+ Captures run with the network refused and with the same `durationMs` contract the check enforces, so the recorder cannot freeze a record its own reference engine would fail.
94
+
95
+ ## Adding a scenario
96
+
97
+ Add it to `multishotGoldenScenarios()` in `src/multishot/golden/scenarios.ts`, then mint a new version.
98
+ The record-set integrity test requires one record per catalog scenario in catalog order, so a scenario with no record fails the suite rather than passing silently.
99
+
100
+ A version may be dropped once no supported consumer checks against it and its behaviour is fully covered by a later version. Until then every version stays: an old record is how a consumer still on an older engine keeps a runnable contract.
101
+
102
+ `v1` was captured from the `./multishot` loop at agent-eval 0.145.21 — the engine the merged loop-to-graph parity proofs compared against.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.145.21",
3
+ "version": "0.146.0",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {
@@ -89,6 +89,11 @@
89
89
  "import": "./dist/multishot/index.js",
90
90
  "default": "./dist/multishot/index.js"
91
91
  },
92
+ "./multishot/golden": {
93
+ "types": "./dist/multishot/golden/index.d.ts",
94
+ "import": "./dist/multishot/golden/index.js",
95
+ "default": "./dist/multishot/golden/index.js"
96
+ },
92
97
  "./campaign": {
93
98
  "types": "./dist/campaign/index.d.ts",
94
99
  "import": "./dist/campaign/index.js",
@@ -1 +0,0 @@
1
- import "./capture-fetch-hykwHfDI.js";
@@ -1 +0,0 @@
1
- {"version":3,"file":"tool-groups-CZotz-e_.d.ts","names":[],"sources":["../src/analyst/tool-groups.ts"],"mappings":";;;;KAmBY;;;;;;;;;;;;;;;;;;;;iBAgDI,wBACd,OAAO,oBACP,OAAO,qBACN"}