@tangle-network/agent-eval 0.137.0 → 0.139.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/CHANGELOG.md +78 -0
  2. package/README.md +34 -0
  3. package/dist/analyst/index.d.ts +485 -104
  4. package/dist/analyst/index.d.ts.map +1 -1
  5. package/dist/analyst/index.js +10 -607
  6. package/dist/analyst/index.js.map +1 -1
  7. package/dist/{benchmark-YDrpumqB.js → benchmark-CYtcIF2V.js} +299 -159
  8. package/dist/benchmark-CYtcIF2V.js.map +1 -0
  9. package/dist/{benchmark-CHX4orG7.d.ts → benchmark-DDVdWcwA.d.ts} +67 -15
  10. package/dist/benchmark-DDVdWcwA.d.ts.map +1 -0
  11. package/dist/benchmark-command-BKfjOBJ5.js +4537 -0
  12. package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
  13. package/dist/benchmarks/index.d.ts +1 -1
  14. package/dist/benchmarks/index.js +1 -1
  15. package/dist/{benchmarks-DCLkQOmc.js → benchmarks-zxhy1QV3.js} +4 -3
  16. package/dist/{benchmarks-DCLkQOmc.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
  17. package/dist/campaign/index.d.ts +5 -5
  18. package/dist/campaign/index.js +4 -3
  19. package/dist/{campaign-lgObcHFC.js → campaign-DrS6_hLd.js} +19 -11
  20. package/dist/campaign-DrS6_hLd.js.map +1 -0
  21. package/dist/canonical-D011XM8r.js +86 -0
  22. package/dist/canonical-D011XM8r.js.map +1 -0
  23. package/dist/cli.js +10 -3
  24. package/dist/cli.js.map +1 -1
  25. package/dist/{client-C8L6h6Wf.d.ts → client-BohnDFBq.d.ts} +4 -4
  26. package/dist/{client-C8L6h6Wf.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
  27. package/dist/{completion-verifier-DSyRNVzU.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
  28. package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
  29. package/dist/contract/index.d.ts +10 -10
  30. package/dist/contract/index.js +9 -8
  31. package/dist/contract/index.js.map +1 -1
  32. package/dist/control.d.ts +2 -2
  33. package/dist/{cost-ledger-D-5_-dhi.js → cost-ledger-CZ9diLxY.js} +90 -45
  34. package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
  35. package/dist/{cost-ledger-D2o6JOrL.d.ts → cost-ledger-DKgyIWRj.d.ts} +6 -2
  36. package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
  37. package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
  38. package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
  39. package/dist/default-registry-BgJJItGr.js +2364 -0
  40. package/dist/default-registry-BgJJItGr.js.map +1 -0
  41. package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
  42. package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
  43. package/dist/{eval-campaign-CHqfLnff.js → eval-campaign-BmptJj50.js} +2 -2
  44. package/dist/{eval-campaign-CHqfLnff.js.map → eval-campaign-BmptJj50.js.map} +1 -1
  45. package/dist/exact-types-MaaFcllV.d.ts +234 -0
  46. package/dist/exact-types-MaaFcllV.d.ts.map +1 -0
  47. package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
  48. package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
  49. package/dist/{extract-usage-p-56bh8q.js → extract-usage-DZs601Va.js} +2 -2
  50. package/dist/{extract-usage-p-56bh8q.js.map → extract-usage-DZs601Va.js.map} +1 -1
  51. package/dist/{feedback-trajectory-N_F0PwHz.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -2
  52. package/dist/feedback-trajectory-BJUWOkJM.d.ts.map +1 -0
  53. package/dist/fuzz.d.ts +1 -1
  54. package/dist/fuzz.js +1 -1
  55. package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
  56. package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
  57. package/dist/hosted/index.d.ts +3 -3
  58. package/dist/{index-C-Pr4OWg.d.ts → index-BTm_P9aC.d.ts} +12 -11
  59. package/dist/index-BTm_P9aC.d.ts.map +1 -0
  60. package/dist/{index-U3RHOShi.d.ts → index-CWOPCJiw.d.ts} +2 -2
  61. package/dist/{index-U3RHOShi.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
  62. package/dist/{index-DRNl6g_N.d.ts → index-CtR1xh4V.d.ts} +3 -3
  63. package/dist/{index-DRNl6g_N.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
  64. package/dist/index-DEb46kc6.d.ts.map +1 -1
  65. package/dist/{index-BnP1QJUv.d.ts → index-_66rVpwN.d.ts} +5 -5
  66. package/dist/{index-BnP1QJUv.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
  67. package/dist/index.d.ts +35 -55
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +55 -514
  70. package/dist/index.js.map +1 -1
  71. package/dist/{insight-report-B9ooYH_g.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
  72. package/dist/{insight-report-B9ooYH_g.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
  73. package/dist/{integrity-CKxosZ5Z.d.ts → integrity-COTh3DTH.d.ts} +2 -2
  74. package/dist/{integrity-CKxosZ5Z.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
  75. package/dist/kind-factory-CFxA0JQX.js +2133 -0
  76. package/dist/kind-factory-CFxA0JQX.js.map +1 -0
  77. package/dist/ledger-core/index.js +2 -1
  78. package/dist/{ledger-core-t6sItivm.js → ledger-core-Dxz0Rkwa.js} +210 -99
  79. package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
  80. package/dist/{llm-client-DKB25jV8.js → llm-client-bkztEfIx.js} +5 -5
  81. package/dist/llm-client-bkztEfIx.js.map +1 -0
  82. package/dist/meta-eval/index.d.ts +2 -2
  83. package/dist/multishot/index.d.ts +2 -2
  84. package/dist/openapi.json +1 -1
  85. package/dist/{proposal-findings-DCawte-y.js → proposal-findings-2GIUo1et.js} +2 -68
  86. package/dist/proposal-findings-2GIUo1et.js.map +1 -0
  87. package/dist/{release-report-CofgVNZt.d.ts → release-report-fZarvIm-.d.ts} +3 -3
  88. package/dist/{release-report-CofgVNZt.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
  89. package/dist/{replay-K8FaC0CB.d.ts → replay-DjG4IG60.d.ts} +34 -143
  90. package/dist/replay-DjG4IG60.d.ts.map +1 -0
  91. package/dist/{replay-Bju0T8Ls.js → replay-SA4OB7O7.js} +48 -136
  92. package/dist/replay-SA4OB7O7.js.map +1 -0
  93. package/dist/reporting.d.ts +4 -4
  94. package/dist/{researcher-Da0Wj-bt.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
  95. package/dist/{researcher-Da0Wj-bt.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
  96. package/dist/{reward-hacking-CQ3hTCO3.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
  97. package/dist/{reward-hacking-CQ3hTCO3.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
  98. package/dist/rl.d.ts +5 -5
  99. package/dist/rl.js +1 -1
  100. package/dist/rollout/index.d.ts +1 -1
  101. package/dist/rollout/index.js +2 -2
  102. package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
  103. package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
  104. package/dist/{rubric-predictive-validity-C4sztLR3.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
  105. package/dist/{rubric-predictive-validity-C4sztLR3.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
  106. package/dist/{run-evidence-BDIircdA.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
  107. package/dist/{run-evidence-BDIircdA.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
  108. package/dist/{run-record-BPCa2rQ8.d.ts → run-record-CztDMXVF.d.ts} +2 -2
  109. package/dist/{run-record-BPCa2rQ8.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
  110. package/dist/{semantic-concept-judge-Bz64IckK.js → semantic-concept-judge-BuIJ9IfB.js} +49 -6
  111. package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
  112. package/dist/{server-KjXZZUDX.js → server-DaCpLfi0.js} +3 -3
  113. package/dist/{server-KjXZZUDX.js.map → server-DaCpLfi0.js.map} +1 -1
  114. package/dist/single-run-lock-BTTtPZ9N.js +989 -0
  115. package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
  116. package/dist/{skill-usage-CFDLLlhF.d.ts → skill-usage-B-BFS8M2.d.ts} +65 -40
  117. package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
  118. package/dist/{skillopt-optimization-method-f4o9sUT4.js → skillopt-optimization-method-BbGnCC53.js} +20 -979
  119. package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
  120. package/dist/{skillopt-optimization-method-BpbnlvAZ.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
  121. package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
  122. package/dist/{statistics-_7P642CN.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
  123. package/dist/{statistics-_7P642CN.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
  124. package/dist/store-otlp-DX4fGIcf.js +757 -0
  125. package/dist/store-otlp-DX4fGIcf.js.map +1 -0
  126. package/dist/{summary-report-DHipz9Kx.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
  127. package/dist/{summary-report-DHipz9Kx.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
  128. package/dist/tool-groups-CdYq22lX.d.ts +258 -0
  129. package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
  130. package/dist/traces.d.ts +7 -6
  131. package/dist/traces.js +5 -4
  132. package/dist/{types-CTGbIm57.d.ts → types-BBFNHxSK.d.ts} +5 -5
  133. package/dist/{types-CTGbIm57.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
  134. package/dist/{types-CKswbJGO.d.ts → types-DoEYskCd.d.ts} +5 -5
  135. package/dist/{types-CKswbJGO.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
  136. package/dist/{types-CTvKfr5F.d.ts → types-uPrS6mD-.d.ts} +2 -2
  137. package/dist/{types-CTvKfr5F.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
  138. package/dist/usage-receipt-CgxMEBZq.js +134 -0
  139. package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
  140. package/dist/wire/index.d.ts +3 -3
  141. package/dist/wire/index.js +1 -1
  142. package/docs/trace-analysis.md +191 -385
  143. package/package.json +5 -4
  144. package/dist/analyze-runs-PVtnfjvA.d.ts +0 -72
  145. package/dist/analyze-runs-PVtnfjvA.d.ts.map +0 -1
  146. package/dist/benchmark-CHX4orG7.d.ts.map +0 -1
  147. package/dist/benchmark-YDrpumqB.js.map +0 -1
  148. package/dist/campaign-lgObcHFC.js.map +0 -1
  149. package/dist/completion-verifier-DSyRNVzU.d.ts.map +0 -1
  150. package/dist/concurrency-MUjT7VjM.js +0 -109
  151. package/dist/concurrency-MUjT7VjM.js.map +0 -1
  152. package/dist/cost-ledger-D-5_-dhi.js.map +0 -1
  153. package/dist/cost-ledger-D2o6JOrL.d.ts.map +0 -1
  154. package/dist/default-registry-CLXbRt0f.js +0 -2594
  155. package/dist/default-registry-CLXbRt0f.js.map +0 -1
  156. package/dist/default-registry-Dc5D_Loc.d.ts +0 -202
  157. package/dist/default-registry-Dc5D_Loc.d.ts.map +0 -1
  158. package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +0 -1
  159. package/dist/index-C-Pr4OWg.d.ts.map +0 -1
  160. package/dist/ledger-core-t6sItivm.js.map +0 -1
  161. package/dist/llm-client-DKB25jV8.js.map +0 -1
  162. package/dist/proposal-findings-DCawte-y.js.map +0 -1
  163. package/dist/registry-BdM7SuTr.d.ts +0 -124
  164. package/dist/registry-BdM7SuTr.d.ts.map +0 -1
  165. package/dist/replay-Bju0T8Ls.js.map +0 -1
  166. package/dist/replay-K8FaC0CB.d.ts.map +0 -1
  167. package/dist/semantic-concept-judge-Bz64IckK.js.map +0 -1
  168. package/dist/skill-usage-CFDLLlhF.d.ts.map +0 -1
  169. package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +0 -1
  170. package/dist/skillopt-optimization-method-f4o9sUT4.js.map +0 -1
  171. package/dist/tools-DZk2Jn64.js +0 -1876
  172. package/dist/tools-DZk2Jn64.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.137.0",
3
+ "version": "0.139.0",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {
@@ -150,7 +150,7 @@
150
150
  "access": "public"
151
151
  },
152
152
  "scripts": {
153
- "build": "tsdown && pnpm openapi",
153
+ "build": "pnpm check:analyst-benchmark && tsdown && pnpm openapi",
154
154
  "dev": "tsdown --watch",
155
155
  "prepare": "husky",
156
156
  "prepublishOnly": "pnpm build",
@@ -161,12 +161,12 @@
161
161
  "lint": "biome check src",
162
162
  "format": "biome format --write src",
163
163
  "check:skill": "node scripts/check-skill.mjs",
164
+ "check:analyst-benchmark": "node scripts/check-analyst-benchmark-implementation.mjs",
164
165
  "openapi": "node dist/cli.js openapi --out dist/openapi.json",
165
- "verify:package": "pnpm run check:skill && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs"
166
+ "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs"
166
167
  },
167
168
  "dependencies": {
168
169
  "@asteasolutions/zod-to-openapi": "^9.1.0",
169
- "@ax-llm/ax": "^23.0.5",
170
170
  "@hono/node-server": "^2.0.12",
171
171
  "@tangle-network/agent-core": "0.4.28",
172
172
  "@tangle-network/agent-interface": "0.39.0",
@@ -201,6 +201,7 @@
201
201
  "vite"
202
202
  ],
203
203
  "overrides": {
204
+ "@arizeai/openinference-core>@opentelemetry/core": "^2.10.0",
204
205
  "esbuild@>=0.27.3 <0.28.1": "^0.28.1",
205
206
  "postcss@<8.5.18": "^8.5.18",
206
207
  "vite@>=7.0.0 <=7.3.4": "^7.3.5",
@@ -1,72 +0,0 @@
1
- import { a as RunRecord } from "./run-record-BPCa2rQ8.js";
2
- import { n as AnalystRegistry } from "./registry-BdM7SuTr.js";
3
- import { a as DatasetScenario } from "./dataset-v_Y5902-.js";
4
- import "./summary-report-DHipz9Kx.js";
5
- import { o as InsightReport, r as ExecutionInsight, t as CostProvenanceSummary } from "./insight-report-B9ooYH_g.js";
6
- //#region src/contract/analyze-runs.d.ts
7
- interface AnalyzeRunsOptions {
8
- /** The runs to analyze. */
9
- runs: RunRecord[];
10
- /** Which split to score against when reading composite from RunOutcome.
11
- * Default: holdout when ANY run has a `holdoutScore`, else search. */
12
- split?: 'search' | 'holdout' | 'auto';
13
- /** Pairwise analysis configuration. When both `baselineCandidateId` and
14
- * `candidateCandidateId` are present, lift is computed on paired
15
- * (experimentId, scenarioId, seed) identities shared between the two sides.
16
- * Unmatched rows remain visible in the lift result. */
17
- baselineCandidateId?: string;
18
- candidateCandidateId?: string;
19
- /** Canary scenarios — checked against every run's raw output for
20
- * holdout contamination. */
21
- canaryScenarios?: DatasetScenario[];
22
- /** Analyst registry for failure clustering. When omitted, the
23
- * `failureClusters` section is left undefined. */
24
- analyst?: AnalystRegistry;
25
- /** Downstream outcome metric per run (e.g. engagement rate, approval
26
- * rate, downstream pass rate). When present, the report includes
27
- * `outcomeCorrelation` + a simple linear reward model fit. */
28
- outcomeSignal?: {
29
- metric: string;
30
- valueByRunId: Record<string, number>;
31
- };
32
- /** Multi-rater feedback for inter-rater agreement. Each entry is one
33
- * rater's score for one run. Two or more raters → kappa + disagreement
34
- * triage list. */
35
- raterScores?: Array<{
36
- runId: string;
37
- rater: string;
38
- score: number;
39
- }>;
40
- /** Number of histogram bins for distributional summaries. Default 12. */
41
- histogramBins?: number;
42
- /** Decision threshold — the smallest composite lift the caller cares
43
- * about. Used by the recommendations engine to call ship vs hold.
44
- * Default 0.02. */
45
- decisionThreshold?: number;
46
- /** Optional prior-period runs. When set, the report includes
47
- * `priorPeriodComparison` with per-metric Welch-CI deltas and
48
- * recommendations fire on statistically significant regressions.
49
- * The two windows do NOT have to share scenarios — the comparison
50
- * is two-sample unpaired (the substrate's `lift` field uses paired
51
- * bootstrap on shared (experimentId, scenarioId, seed) identities; this is the
52
- * shape for "this week vs last week" rather than "candidate vs
53
- * baseline within a campaign"). */
54
- baselineRuns?: RunRecord[];
55
- /** Human-readable label for the baseline window, e.g. "vs prior 7
56
- * days", "vs v3.1 release". Surfaces in recommendations + UI. */
57
- baselineLabel?: string;
58
- }
59
- interface SummarizeExecutionOptions {
60
- runs: RunRecord[];
61
- histogramBins?: number;
62
- }
63
- interface ExecutionReport {
64
- execution: ExecutionInsight;
65
- costProvenance: CostProvenanceSummary;
66
- }
67
- /** Summarize runtime facts without interpreting task quality or promotion readiness. */
68
- declare function summarizeExecution(opts: SummarizeExecutionOptions): ExecutionReport;
69
- declare function analyzeRuns(opts: AnalyzeRunsOptions): Promise<InsightReport>;
70
- //#endregion
71
- export { summarizeExecution as a, analyzeRuns as i, ExecutionReport as n, SummarizeExecutionOptions as r, AnalyzeRunsOptions as t };
72
- //# sourceMappingURL=analyze-runs-PVtnfjvA.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"analyze-runs-PVtnfjvA.d.ts","names":[],"sources":["../src/contract/analyze-runs.ts"],"mappings":";;;;;;UAoEiB;;EAEf,MAAM;;;EAGN;;;;;EAKA;EACA;;;EAGA,kBAAkB;;;EAGlB,UAAU;;;;EAIV;IACE;IACA,cAAc;;;;;EAKhB,cAAc;IAAQ;IAAe;IAAe;;;EAEpD;;;;EAIA;;;;;;;;;EASA,eAAe;;;EAGf;;UAGe;EACf,MAAM;EACN;;UAGe;EACf,WAAW;EACX,gBAAgB;;;iBAIF,mBAAmB,MAAM,4BAA4B;iBAgB/C,YAAY,MAAM,qBAAqB,QAAQ"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"benchmark-CHX4orG7.d.ts","names":[],"sources":["../src/analyst/benchmark.ts"],"mappings":";;;UAYiB;EACf;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,4BAA4B;;UAGb,qBAAqB;EACpC;EACA,OAAO;EACP,yBAAyB;;EAEzB,2BAA2B;EAC3B;EACA,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA,mBAAmB;EACnB;EACA;EACA;EACA;;EAEA;;EAEA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;UAGe;EACf;EACA;EACA,oBAAoB;EACpB,QAAQ;;EAER;;KAGU,wBAAwB,qBAAqB;EACvD;EACA,WAAW;EACX,UAAU;EACV,SAAS;gBACK;;;;;iBAMA,2BAA2B,QACzC,WAAW,OAAO,WAAW,qBAC5B,wBAAwB;UAiBV;EACf,mBAAmB;EACnB,QAAQ;EACR,WAAW;;UAGI,uBAAuB;EACtC;EACA,QACE,OAAO,QACP;IAAW;IAAgB;IAAoB,SAAS;MACvD,yBAAyB,QAAQ;;UAGrB;EACf;EACA;EACA;EACA;EACA;EACA,mBAAmB;EACnB,OAAO;EACP,qBAAqB;EACrB;EACA,eAAe;EACf,QAAQ;EACR,iBAAiB;EACjB;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,UAAU;EACV;EACA,cAAc;EACd,WAAW;;UAGI,mCAAmC;EAClD;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,YAAY;EACZ,cAAc;EACd,WAAW;;UAGI,2BAA2B;EAC1C,gBAAgB,qBAAqB;EACrC,kBAAkB,uBAAuB;EACzC;EACA;EACA;EACA,kBAAkB,wBAAwB;EAC1C,YAAY;EACZ,SAAS;;iBAGK,qBACd,UAAU,KAAK,oEACf,mBAAmB,mBAClB;iBA0FmB,oBAAoB,QACxC,SAAS,2BAA2B,UACnC,QAAQ;iBAiDK,wBAAwB;EACtC;EACA,UAAU;EACV,aAAa,KAAK;IAChB,uBAAuB"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"benchmark-YDrpumqB.js","names":[],"sources":["../src/analyst/benchmark.ts"],"sourcesContent":["import { performance } from 'node:perf_hooks'\nimport { linearSumAssignment } from 'linear-sum-assignment'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { AnalystRegistry, RegistryRunOpts } from './registry'\nimport type {\n AnalystFinding,\n AnalystRunInputs,\n AnalystRunResult,\n AnalystUsageReceipt,\n EvidenceRef,\n} from './types'\n\nexport interface AnalystEvidenceExpectation {\n uri: string\n kind?: EvidenceRef['kind']\n}\n\nexport interface AnalystIssueExpectation {\n id: string\n findingIds?: readonly string[]\n areas?: readonly string[]\n subjects?: readonly string[]\n evidence?: readonly AnalystEvidenceExpectation[]\n evidenceMode?: 'any' | 'all'\n /** Exact evidence location for the first unrecoverable or causal step. */\n criticalEvidence?: readonly AnalystEvidenceExpectation[]\n}\n\nexport interface AnalystBenchmarkCase<TInput = unknown> {\n id: string\n input: TInput\n expectedIssues: readonly AnalystIssueExpectation[]\n /** Complete set of labeled locations used to measure label-location agreement. */\n labeledEvidence?: readonly AnalystEvidenceExpectation[]\n tags?: readonly string[]\n metadata?: Record<string, unknown>\n}\n\nexport interface AnalystFindingScore {\n expectedIssueCount: number\n matchedIssueIds: string[]\n missedIssueIds: string[]\n supportedFindingIndexes: number[]\n unsupportedFindingIndexes: number[]\n unlabeledEvidence: EvidenceRef[]\n issueRecall: number\n findingPrecision: number\n f1: number\n criticalStepAccuracy: number | null\n /** Share of findings that cite at least one evidence location. */\n citationCoverage: number | null\n /** Share of citations that agree with a labeled case location. */\n citationLabelAgreement: number | null\n cleanFalsePositive: boolean\n}\n\nexport interface AnalystEvidenceResolutionError {\n evidence: EvidenceRef\n class: string\n message: string\n}\n\nexport interface AnalystEvidenceResolution {\n checked: number\n resolved: number\n unresolvedEvidence: EvidenceRef[]\n errors: AnalystEvidenceResolutionError[]\n /** Null when no citations were checked or any resolution attempt failed. */\n validity: number | null\n}\n\nexport type AnalystEvidenceResolver<TInput = unknown> = (input: {\n caseId: string\n caseInput: TInput\n evidence: EvidenceRef\n signal?: AbortSignal\n}) => boolean | Promise<boolean>\n\n/**\n * Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.\n * Other evidence kinds and URI schemes require a caller-supplied resolver.\n */\nexport function traceStoreEvidenceResolver<TInput>(\n getStore: (input: TInput) => TraceAnalysisStore,\n): AnalystEvidenceResolver<TInput> {\n return async ({ caseInput, evidence }) => {\n if (evidence.kind !== 'span') return false\n const location = parseTraceSpanUri(evidence.uri)\n if (!location) return false\n const result = await getStore(caseInput).viewSpans({\n trace_id: location.traceId,\n span_ids: [location.spanId],\n })\n return (\n result.trace_id === location.traceId &&\n result.missing_span_ids.length === 0 &&\n result.spans.some((span) => span.span_id === location.spanId)\n )\n }\n}\n\nexport interface AnalystBenchmarkOutput {\n findings: readonly AnalystFinding[]\n usage?: AnalystUsageReceipt\n metadata?: Record<string, unknown>\n}\n\nexport interface AnalystBenchmarkRunner<TInput = unknown> {\n id: string\n analyze(\n input: TInput,\n context: { caseId: string; repetition: number; signal?: AbortSignal },\n ): AnalystBenchmarkOutput | Promise<AnalystBenchmarkOutput>\n}\n\nexport interface AnalystBenchmarkObservation {\n runnerId: string\n caseId: string\n repetition: number\n executionIndex: number\n latencyMs: number\n findings: readonly AnalystFinding[]\n score: AnalystFindingScore\n evidenceResolution?: AnalystEvidenceResolution\n caseTags: readonly string[]\n caseMetadata?: Record<string, unknown>\n usage?: AnalystUsageReceipt\n runnerMetadata?: Record<string, unknown>\n error?: { class: string; message: string }\n}\n\nexport interface AnalystLatencyDistribution {\n min: number\n mean: number\n p50: number\n p95: number\n max: number\n}\n\nexport interface AnalystBenchmarkSummary {\n runnerId: string\n plannedRuns: number\n completedRuns: number\n failedRuns: number\n issueRecall: number | null\n findingPrecision: number | null\n f1: number | null\n criticalStepAccuracy: number | null\n citationCoverage: number | null\n citationLabelAgreement: number | null\n citationResolution: number | null\n citationResolutionUnknownRuns: number\n unresolvedCitations: number\n citationResolutionErrors: number\n cleanCaseFalsePositiveRate: number | null\n cleanCaseFailureRate: number | null\n runAgreement: number | null\n latencyMs: AnalystLatencyDistribution\n calls: number\n callsUnknownRuns: number\n inputTokens: number\n outputTokens: number\n reasoningTokens: number\n cachedTokens: number\n cacheWriteTokens: number\n tokenUsageUnknownRuns: number\n reasoningTokenUsageUnknownRuns: number\n cachedTokenUsageUnknownRuns: number\n cacheWriteTokenUsageUnknownRuns: number\n knownCostUsd: number\n costUnknownRuns: number\n}\n\nexport interface AnalystBenchmarkDatasetRef {\n id: string\n revision: string\n split?: string\n}\n\nexport interface AnalystBenchmarkDescriptor {\n id?: string\n dataset?: AnalystBenchmarkDatasetRef\n command?: string\n environment?: Record<string, string>\n metadata?: Record<string, unknown>\n}\n\nexport interface AnalystBenchmarkProvenance extends AnalystBenchmarkDescriptor {\n startedAt: string\n endedAt: string\n caseCount: number\n runnerIds: string[]\n repetitions: number\n maxConcurrency: number\n runnerOrderSeed: number\n}\n\nexport interface AnalystBenchmarkResult {\n provenance: AnalystBenchmarkProvenance\n observations: AnalystBenchmarkObservation[]\n summaries: AnalystBenchmarkSummary[]\n}\n\nexport interface RunAnalystBenchmarkOptions<TInput> {\n cases: readonly AnalystBenchmarkCase<TInput>[]\n runners: readonly AnalystBenchmarkRunner<TInput>[]\n repetitions?: number\n maxConcurrency?: number\n runnerOrderSeed?: number\n resolveEvidence?: AnalystEvidenceResolver<TInput>\n benchmark?: AnalystBenchmarkDescriptor\n signal?: AbortSignal\n}\n\nexport function scoreAnalystFindings(\n testCase: Pick<AnalystBenchmarkCase, 'id' | 'expectedIssues' | 'labeledEvidence'>,\n findings: readonly AnalystFinding[],\n): AnalystFindingScore {\n validateCase(testCase)\n const matchedFindingByIssue = matchFindingsToIssues(testCase.expectedIssues, findings)\n const matchedIssueIds = testCase.expectedIssues\n .filter((_, index) => matchedFindingByIssue.has(index))\n .map((issue) => issue.id)\n const missedIssueIds = testCase.expectedIssues\n .filter((_, index) => !matchedFindingByIssue.has(index))\n .map((issue) => issue.id)\n const supportedFindingIndexes = new Set(matchedFindingByIssue.values())\n\n const unsupportedFindingIndexes = findings\n .map((_, index) => index)\n .filter((index) => !supportedFindingIndexes.has(index))\n const expectedIssueCount = testCase.expectedIssues.length\n const issueRecall = expectedIssueCount === 0 ? 1 : matchedIssueIds.length / expectedIssueCount\n const findingPrecision =\n findings.length === 0\n ? expectedIssueCount === 0\n ? 1\n : 0\n : supportedFindingIndexes.size / findings.length\n const f1 = harmonicMean(findingPrecision, issueRecall)\n const allEvidence = findings.flatMap((finding) => finding.evidence_refs)\n\n const criticalIssues = testCase.expectedIssues.filter(\n (issue) => (issue.criticalEvidence?.length ?? 0) > 0,\n )\n const criticalHits = testCase.expectedIssues.filter((issue) => {\n if ((issue.criticalEvidence?.length ?? 0) === 0) return false\n return matchesEvidence(allEvidence, issue.criticalEvidence ?? [], 'any')\n }).length\n\n const findingsWithEvidence = findings.filter((finding) => finding.evidence_refs.length > 0).length\n const unlabeledEvidence = testCase.labeledEvidence\n ? allEvidence.filter(\n (ref) => !testCase.labeledEvidence!.some((expected) => evidenceMatches(ref, expected)),\n )\n : []\n\n return {\n expectedIssueCount,\n matchedIssueIds,\n missedIssueIds,\n supportedFindingIndexes: [...supportedFindingIndexes].sort((a, b) => a - b),\n unsupportedFindingIndexes,\n unlabeledEvidence,\n issueRecall,\n findingPrecision,\n f1,\n criticalStepAccuracy: criticalIssues.length === 0 ? null : criticalHits / criticalIssues.length,\n citationCoverage: findings.length === 0 ? null : findingsWithEvidence / findings.length,\n citationLabelAgreement:\n testCase.labeledEvidence === undefined\n ? null\n : allEvidence.length === 0\n ? findings.length === 0\n ? null\n : 0\n : (allEvidence.length - unlabeledEvidence.length) / allEvidence.length,\n cleanFalsePositive: expectedIssueCount === 0 && findings.length > 0,\n }\n}\n\nfunction matchFindingsToIssues(\n issues: readonly AnalystIssueExpectation[],\n findings: readonly AnalystFinding[],\n): Map<number, number> {\n if (issues.length === 0) return new Map()\n const cardinalityWeight = issues.length + 1\n const scores = issues.map((issue) => [\n ...findings.map((finding) => {\n if (!findingMatchesIssue(finding, issue)) return -1\n const criticalHit =\n (issue.criticalEvidence?.length ?? 0) > 0 &&\n matchesEvidence(finding.evidence_refs, issue.criticalEvidence ?? [], 'any')\n return cardinalityWeight + Number(criticalHit)\n }),\n ...Array.from({ length: issues.length }, () => 0),\n ])\n const assignment = linearSumAssignment(scores, { maximaze: true }).rowAssignments\n const matches = new Map<number, number>()\n for (const [issueIndex, column] of assignment.entries()) {\n if (column < 0 || column >= findings.length) continue\n if (!findingMatchesIssue(findings[column]!, issues[issueIndex]!)) continue\n matches.set(issueIndex, column)\n }\n return matches\n}\n\nexport async function runAnalystBenchmark<TInput>(\n options: RunAnalystBenchmarkOptions<TInput>,\n): Promise<AnalystBenchmarkResult> {\n validateBenchmarkOptions(options)\n const startedAt = new Date().toISOString()\n const repetitions = options.repetitions ?? 1\n const runnerOrderSeed = options.runnerOrderSeed ?? 0\n const maxConcurrency = Math.min(\n options.maxConcurrency ?? 1,\n options.runners.length * options.cases.length * repetitions,\n )\n const jobs = benchmarkJobs(options.cases, options.runners, repetitions, runnerOrderSeed)\n const observations: AnalystBenchmarkObservation[] = []\n let cursor = 0\n const worker = async (): Promise<void> => {\n while (cursor < jobs.length) {\n options.signal?.throwIfAborted()\n const job = jobs[cursor++]!\n observations.push(await runBenchmarkJob(job, options.signal, options.resolveEvidence))\n }\n }\n await Promise.all(Array.from({ length: maxConcurrency }, worker))\n const runnerOrder = new Map(options.runners.map((runner, index) => [runner.id, index]))\n const caseOrder = new Map(options.cases.map((testCase, index) => [testCase.id, index]))\n observations.sort(\n (a, b) =>\n (runnerOrder.get(a.runnerId) ?? 0) - (runnerOrder.get(b.runnerId) ?? 0) ||\n (caseOrder.get(a.caseId) ?? 0) - (caseOrder.get(b.caseId) ?? 0) ||\n a.repetition - b.repetition,\n )\n return {\n provenance: {\n ...options.benchmark,\n startedAt,\n endedAt: new Date().toISOString(),\n caseCount: options.cases.length,\n runnerIds: options.runners.map((runner) => runner.id),\n repetitions,\n maxConcurrency,\n runnerOrderSeed,\n },\n observations,\n summaries: options.runners.map((runner) =>\n summarizeRunner(\n runner.id,\n observations.filter((observation) => observation.runnerId === runner.id),\n ),\n ),\n }\n}\n\nexport function registryBenchmarkRunner(options: {\n id: string\n registry: AnalystRegistry\n runOptions?: Omit<RegistryRunOpts, 'signal'>\n}): AnalystBenchmarkRunner<AnalystRunInputs> {\n return {\n id: options.id,\n async analyze(input, context) {\n const result = await options.registry.run(\n `${options.id}:${context.caseId}:${context.repetition}`,\n input,\n { ...options.runOptions, signal: context.signal },\n )\n return {\n findings: result.findings,\n usage: mergeRegistryUsage(result),\n metadata: { analystRun: result },\n }\n },\n }\n}\n\nasync function runBenchmarkJob<TInput>(\n job: {\n runner: AnalystBenchmarkRunner<TInput>\n testCase: AnalystBenchmarkCase<TInput>\n repetition: number\n executionIndex: number\n },\n signal?: AbortSignal,\n resolveEvidence?: AnalystEvidenceResolver<TInput>,\n): Promise<AnalystBenchmarkObservation> {\n const started = performance.now()\n try {\n const output = await job.runner.analyze(job.testCase.input, {\n caseId: job.testCase.id,\n repetition: job.repetition,\n signal,\n })\n return {\n runnerId: job.runner.id,\n caseId: job.testCase.id,\n repetition: job.repetition,\n executionIndex: job.executionIndex,\n latencyMs: performance.now() - started,\n findings: output.findings,\n score: scoreAnalystFindings(job.testCase, output.findings),\n evidenceResolution: resolveEvidence\n ? await resolveFindingEvidence(job.testCase, output.findings, resolveEvidence, signal)\n : undefined,\n caseTags: [...(job.testCase.tags ?? [])],\n caseMetadata: job.testCase.metadata,\n usage: output.usage,\n runnerMetadata: output.metadata,\n }\n } catch (error) {\n if (signal?.aborted) throw error\n const findings: AnalystFinding[] = []\n return {\n runnerId: job.runner.id,\n caseId: job.testCase.id,\n repetition: job.repetition,\n executionIndex: job.executionIndex,\n latencyMs: performance.now() - started,\n findings,\n score: scoreAnalystFindings(job.testCase, findings),\n caseTags: [...(job.testCase.tags ?? [])],\n caseMetadata: job.testCase.metadata,\n error: {\n class: error instanceof Error ? error.constructor.name : 'Error',\n message: error instanceof Error ? error.message : String(error),\n },\n }\n }\n}\n\nfunction benchmarkJobs<TInput>(\n cases: readonly AnalystBenchmarkCase<TInput>[],\n runners: readonly AnalystBenchmarkRunner<TInput>[],\n repetitions: number,\n runnerOrderSeed: number,\n): Array<{\n runner: AnalystBenchmarkRunner<TInput>\n testCase: AnalystBenchmarkCase<TInput>\n repetition: number\n executionIndex: number\n}> {\n const seededRunners = [...runners].sort(\n (left, right) =>\n stableHash(`${runnerOrderSeed}\\u0000${left.id}`) -\n stableHash(`${runnerOrderSeed}\\u0000${right.id}`) || left.id.localeCompare(right.id),\n )\n let executionIndex = 0\n return cases.flatMap((testCase, caseIndex) =>\n Array.from({ length: repetitions }, (_, repetition) => {\n const blockIndex = caseIndex * repetitions + repetition\n const rotation = blockIndex % seededRunners.length\n const ordered = [...seededRunners.slice(rotation), ...seededRunners.slice(0, rotation)]\n return ordered.map((runner) => ({\n runner,\n testCase,\n repetition,\n executionIndex: executionIndex++,\n }))\n }).flat(),\n )\n}\n\nasync function resolveFindingEvidence<TInput>(\n testCase: AnalystBenchmarkCase<TInput>,\n findings: readonly AnalystFinding[],\n resolver: AnalystEvidenceResolver<TInput>,\n signal?: AbortSignal,\n): Promise<AnalystEvidenceResolution> {\n const evidence = findings.flatMap((finding) => finding.evidence_refs)\n const resolved: EvidenceRef[] = []\n const unresolvedEvidence: EvidenceRef[] = []\n const errors: AnalystEvidenceResolutionError[] = []\n for (const ref of evidence) {\n signal?.throwIfAborted()\n try {\n if (\n await resolver({\n caseId: testCase.id,\n caseInput: testCase.input,\n evidence: ref,\n signal,\n })\n ) {\n resolved.push(ref)\n } else {\n unresolvedEvidence.push(ref)\n }\n } catch (error) {\n if (signal?.aborted) throw error\n errors.push({\n evidence: ref,\n class: error instanceof Error ? error.constructor.name : 'Error',\n message: error instanceof Error ? error.message : String(error),\n })\n }\n }\n return {\n checked: evidence.length,\n resolved: resolved.length,\n unresolvedEvidence,\n errors,\n validity: evidence.length === 0 || errors.length > 0 ? null : resolved.length / evidence.length,\n }\n}\n\nfunction summarizeRunner(\n runnerId: string,\n observations: readonly AnalystBenchmarkObservation[],\n): AnalystBenchmarkSummary {\n const issueBearing = observations.filter(\n (observation) => observation.score.expectedIssueCount > 0,\n )\n const expectedIssues = issueBearing.reduce(\n (sum, observation) => sum + observation.score.expectedIssueCount,\n 0,\n )\n const matchedIssues = issueBearing.reduce(\n (sum, observation) => sum + observation.score.matchedIssueIds.length,\n 0,\n )\n const issueFindings = issueBearing.reduce(\n (sum, observation) => sum + observation.findings.length,\n 0,\n )\n const supportedFindings = issueBearing.reduce(\n (sum, observation) => sum + observation.score.supportedFindingIndexes.length,\n 0,\n )\n const issueRecall = expectedIssues === 0 ? null : matchedIssues / expectedIssues\n const findingPrecision =\n expectedIssues === 0 ? null : issueFindings === 0 ? 0 : supportedFindings / issueFindings\n const critical = observations\n .map((observation) => observation.score.criticalStepAccuracy)\n .filter((value): value is number => value !== null)\n const findingsWithEvidence = observations.reduce(\n (sum, observation) =>\n sum + observation.findings.filter((finding) => finding.evidence_refs.length > 0).length,\n 0,\n )\n const allFindings = observations.reduce(\n (sum, observation) => sum + observation.findings.length,\n 0,\n )\n const citationObservations = observations.filter(\n (observation) => observation.score.citationLabelAgreement !== null,\n )\n const citationCount = citationObservations.reduce(\n (sum, observation) =>\n sum +\n observation.findings.reduce((count, finding) => count + finding.evidence_refs.length, 0),\n 0,\n )\n const invalidCitationCount = citationObservations.reduce(\n (sum, observation) => sum + observation.score.unlabeledEvidence.length,\n 0,\n )\n const clean = observations.filter((observation) => observation.score.expectedIssueCount === 0)\n const completedClean = clean.filter((observation) => !observation.error)\n const resolvedCitations = observations.reduce(\n (sum, observation) => sum + (observation.evidenceResolution?.resolved ?? 0),\n 0,\n )\n const unresolvedCitations = observations.reduce(\n (sum, observation) => sum + (observation.evidenceResolution?.unresolvedEvidence.length ?? 0),\n 0,\n )\n const citationResolutionErrors = observations.reduce(\n (sum, observation) => sum + (observation.evidenceResolution?.errors.length ?? 0),\n 0,\n )\n const resolutionAttempts = observations.filter((observation) =>\n observation.findings.some((finding) => finding.evidence_refs.length > 0),\n )\n const citationResolutionUnknownRuns = resolutionAttempts.filter(\n (observation) =>\n !observation.evidenceResolution || observation.evidenceResolution.errors.length > 0,\n ).length\n const usages = observations.map((observation) => observation.usage)\n const knownCostUsd = usages.reduce((sum, usage) => {\n if (!usage) return sum\n return sum + (usage.cost.kind === 'uncaptured' ? (usage.knownCostUsd ?? 0) : usage.cost.usd)\n }, 0)\n return {\n runnerId,\n plannedRuns: observations.length,\n completedRuns: observations.filter((observation) => !observation.error).length,\n failedRuns: observations.filter((observation) => Boolean(observation.error)).length,\n issueRecall,\n findingPrecision,\n f1:\n findingPrecision === null || issueRecall === null\n ? null\n : harmonicMean(findingPrecision, issueRecall),\n criticalStepAccuracy: critical.length === 0 ? null : mean(critical),\n citationCoverage: allFindings === 0 ? null : findingsWithEvidence / allFindings,\n citationLabelAgreement:\n citationObservations.length === 0\n ? null\n : citationCount === 0\n ? 0\n : (citationCount - invalidCitationCount) / citationCount,\n citationResolution:\n resolutionAttempts.length === 0 ||\n citationResolutionUnknownRuns > 0 ||\n resolvedCitations + unresolvedCitations === 0\n ? null\n : resolvedCitations / (resolvedCitations + unresolvedCitations),\n citationResolutionUnknownRuns,\n unresolvedCitations,\n citationResolutionErrors,\n cleanCaseFalsePositiveRate:\n completedClean.length === 0\n ? null\n : completedClean.filter((observation) => observation.score.cleanFalsePositive).length /\n completedClean.length,\n cleanCaseFailureRate:\n clean.length === 0\n ? null\n : clean.filter((observation) => Boolean(observation.error)).length / clean.length,\n runAgreement: issueAgreement(observations.filter((observation) => !observation.error)),\n latencyMs: latencyDistribution(observations.map((observation) => observation.latencyMs)),\n calls: usages.reduce((sum, usage) => sum + (usage?.calls ?? 0), 0),\n callsUnknownRuns: usages.filter((usage) => !usage || usage.calls === null).length,\n inputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.input ?? 0), 0),\n outputTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.output ?? 0), 0),\n reasoningTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.reasoning ?? 0), 0),\n cachedTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cached ?? 0), 0),\n cacheWriteTokens: usages.reduce((sum, usage) => sum + (usage?.tokens?.cacheWrite ?? 0), 0),\n tokenUsageUnknownRuns: usages.filter((usage) => !usage?.tokens).length,\n reasoningTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.reasoning === undefined)\n .length,\n cachedTokenUsageUnknownRuns: usages.filter((usage) => usage?.tokens?.cached === undefined)\n .length,\n cacheWriteTokenUsageUnknownRuns: usages.filter(\n (usage) => usage?.tokens?.cacheWrite === undefined,\n ).length,\n knownCostUsd,\n costUnknownRuns: usages.filter((usage) => !usage || usage.cost.kind === 'uncaptured').length,\n }\n}\n\nfunction findingMatchesIssue(finding: AnalystFinding, issue: AnalystIssueExpectation): boolean {\n if (issue.findingIds && !issue.findingIds.includes(finding.finding_id)) return false\n if (issue.areas && !issue.areas.includes(finding.area)) return false\n if (issue.subjects && (!finding.subject || !issue.subjects.includes(finding.subject)))\n return false\n if (\n issue.evidence &&\n !matchesEvidence(finding.evidence_refs, issue.evidence, issue.evidenceMode ?? 'any')\n ) {\n return false\n }\n return true\n}\n\nfunction matchesEvidence(\n actual: readonly EvidenceRef[],\n expected: readonly AnalystEvidenceExpectation[],\n mode: 'any' | 'all',\n): boolean {\n if (expected.length === 0) return true\n const match = (target: AnalystEvidenceExpectation) =>\n actual.some((ref) => evidenceMatches(ref, target))\n return mode === 'all' ? expected.every(match) : expected.some(match)\n}\n\nfunction evidenceMatches(actual: EvidenceRef, expected: AnalystEvidenceExpectation): boolean {\n return (\n actual.uri === expected.uri && (expected.kind === undefined || actual.kind === expected.kind)\n )\n}\n\nfunction parseTraceSpanUri(uri: string): { traceId: string; spanId: string } | null {\n const match = /^trace:\\/\\/([^/]+)\\/span\\/([^/]+)$/.exec(uri)\n if (!match) return null\n try {\n const traceId = decodeURIComponent(match[1]!)\n const spanId = decodeURIComponent(match[2]!)\n return traceId && spanId ? { traceId, spanId } : null\n } catch {\n return null\n }\n}\n\nfunction validateCase(\n testCase: Pick<AnalystBenchmarkCase, 'id' | 'expectedIssues' | 'labeledEvidence'>,\n): void {\n if (!testCase.id.trim()) throw new TypeError('analyst benchmark case id must not be empty')\n const ids = new Set<string>()\n for (const issue of testCase.expectedIssues) {\n if (!issue.id.trim()) throw new TypeError(`${testCase.id}: expected issue id must not be empty`)\n if (ids.has(issue.id))\n throw new TypeError(`${testCase.id}: duplicate expected issue id '${issue.id}'`)\n ids.add(issue.id)\n if (\n !issue.findingIds?.length &&\n !issue.areas?.length &&\n !issue.subjects?.length &&\n !issue.evidence?.length\n ) {\n throw new TypeError(\n `${testCase.id}/${issue.id}: expected issue must identify a finding by id, area, subject, or evidence`,\n )\n }\n }\n for (const ref of testCase.labeledEvidence ?? []) {\n if (!ref.uri.trim()) {\n throw new TypeError(`${testCase.id}: labeled evidence URI must not be empty`)\n }\n }\n}\n\nfunction validateBenchmarkOptions<TInput>(options: RunAnalystBenchmarkOptions<TInput>): void {\n if (options.cases.length === 0) throw new TypeError('runAnalystBenchmark requires cases')\n if (options.runners.length === 0) throw new TypeError('runAnalystBenchmark requires runners')\n const repetitions = options.repetitions ?? 1\n const maxConcurrency = options.maxConcurrency ?? 1\n if (!Number.isSafeInteger(repetitions) || repetitions < 1) {\n throw new RangeError('runAnalystBenchmark repetitions must be a positive safe integer')\n }\n if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1) {\n throw new RangeError('runAnalystBenchmark maxConcurrency must be a positive safe integer')\n }\n if (!Number.isSafeInteger(options.runnerOrderSeed ?? 0)) {\n throw new RangeError('runAnalystBenchmark runnerOrderSeed must be a safe integer')\n }\n assertUniqueNonEmpty(\n options.cases.map((testCase) => testCase.id),\n 'case',\n )\n assertUniqueNonEmpty(\n options.runners.map((runner) => runner.id),\n 'runner',\n )\n for (const testCase of options.cases) validateCase(testCase)\n}\n\nfunction assertUniqueNonEmpty(values: readonly string[], label: string): void {\n const seen = new Set<string>()\n for (const value of values) {\n if (!value.trim()) throw new TypeError(`analyst benchmark ${label} id must not be empty`)\n if (seen.has(value)) throw new TypeError(`duplicate analyst benchmark ${label} id '${value}'`)\n seen.add(value)\n }\n}\n\nfunction harmonicMean(a: number, b: number): number {\n return a + b === 0 ? 0 : (2 * a * b) / (a + b)\n}\n\nfunction mean(values: readonly number[]): number {\n return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length\n}\n\nfunction latencyDistribution(values: readonly number[]): AnalystLatencyDistribution {\n const sorted = [...values].sort((a, b) => a - b)\n return {\n min: sorted[0] ?? 0,\n mean: mean(sorted),\n p50: percentile(sorted, 0.5),\n p95: percentile(sorted, 0.95),\n max: sorted.at(-1) ?? 0,\n }\n}\n\nfunction percentile(sorted: readonly number[], quantile: number): number {\n if (sorted.length === 0) return 0\n return sorted[Math.ceil(quantile * sorted.length) - 1] ?? sorted.at(-1) ?? 0\n}\n\nfunction stableHash(value: string): number {\n let hash = 2166136261\n for (let index = 0; index < value.length; index += 1) {\n hash ^= value.charCodeAt(index)\n hash = Math.imul(hash, 16777619)\n }\n return hash >>> 0\n}\n\nfunction issueAgreement(observations: readonly AnalystBenchmarkObservation[]): number | null {\n const byCase = new Map<string, AnalystBenchmarkObservation[]>()\n for (const observation of observations) {\n const rows = byCase.get(observation.caseId) ?? []\n rows.push(observation)\n byCase.set(observation.caseId, rows)\n }\n const agreements: number[] = []\n for (const rows of byCase.values()) {\n for (let left = 0; left < rows.length; left++) {\n for (let right = left + 1; right < rows.length; right++) {\n agreements.push(\n jaccard(rows[left]!.score.matchedIssueIds, rows[right]!.score.matchedIssueIds),\n )\n }\n }\n }\n return agreements.length === 0 ? null : mean(agreements)\n}\n\nfunction jaccard(left: readonly string[], right: readonly string[]): number {\n const a = new Set(left)\n const b = new Set(right)\n const union = new Set([...a, ...b])\n if (union.size === 0) return 1\n let intersection = 0\n for (const value of a) if (b.has(value)) intersection += 1\n return intersection / union.size\n}\n\nfunction mergeRegistryUsage(result: AnalystRunResult): AnalystUsageReceipt {\n const usages = result.per_analyst.map((summary) => summary.usage)\n const calls = usages.every((usage) => usage.calls !== null)\n ? usages.reduce((sum, usage) => sum + (usage.calls ?? 0), 0)\n : null\n const tokens = usages.every((usage) => usage.tokens !== null)\n ? usages.reduce(\n (sum, usage) => ({\n input: sum.input + (usage.tokens?.input ?? 0),\n output: sum.output + (usage.tokens?.output ?? 0),\n reasoning: sum.reasoning + (usage.tokens?.reasoning ?? 0),\n cached: sum.cached + (usage.tokens?.cached ?? 0),\n cacheWrite: sum.cacheWrite + (usage.tokens?.cacheWrite ?? 0),\n }),\n { input: 0, output: 0, reasoning: 0, cached: 0, cacheWrite: 0 },\n )\n : null\n const knownCostUsd = usages.reduce(\n (sum, usage) =>\n sum + (usage.cost.kind === 'uncaptured' ? (usage.knownCostUsd ?? 0) : usage.cost.usd),\n 0,\n )\n const cost = usages.some((usage) => usage.cost.kind === 'uncaptured')\n ? ({ kind: 'uncaptured', usd: null } as const)\n : usages.some((usage) => usage.cost.kind === 'estimated')\n ? ({ kind: 'estimated', usd: knownCostUsd } as const)\n : ({ kind: 'observed', usd: knownCostUsd } as const)\n return {\n calls,\n tokens,\n cost,\n ...(cost.kind === 'uncaptured' ? { knownCostUsd } : {}),\n }\n}\n"],"mappings":";;;;;;;AAkFA,SAAgB,2BACd,UACiC;CACjC,OAAO,OAAO,EAAE,WAAW,eAAe;EACxC,IAAI,SAAS,SAAS,QAAQ,OAAO;EACrC,MAAM,WAAW,kBAAkB,SAAS,GAAG;EAC/C,IAAI,CAAC,UAAU,OAAO;EACtB,MAAM,SAAS,MAAM,SAAS,SAAS,CAAC,CAAC,UAAU;GACjD,UAAU,SAAS;GACnB,UAAU,CAAC,SAAS,MAAM;EAC5B,CAAC;EACD,OACE,OAAO,aAAa,SAAS,WAC7B,OAAO,iBAAiB,WAAW,KACnC,OAAO,MAAM,MAAM,SAAS,KAAK,YAAY,SAAS,MAAM;CAEhE;AACF;AAmHA,SAAgB,qBACd,UACA,UACqB;CACrB,aAAa,QAAQ;CACrB,MAAM,wBAAwB,sBAAsB,SAAS,gBAAgB,QAAQ;CACrF,MAAM,kBAAkB,SAAS,eAC9B,QAAQ,GAAG,UAAU,sBAAsB,IAAI,KAAK,CAAC,CAAC,CACtD,KAAK,UAAU,MAAM,EAAE;CAC1B,MAAM,iBAAiB,SAAS,eAC7B,QAAQ,GAAG,UAAU,CAAC,sBAAsB,IAAI,KAAK,CAAC,CAAC,CACvD,KAAK,UAAU,MAAM,EAAE;CAC1B,MAAM,0BAA0B,IAAI,IAAI,sBAAsB,OAAO,CAAC;CAEtE,MAAM,4BAA4B,SAC/B,KAAK,GAAG,UAAU,KAAK,CAAC,CACxB,QAAQ,UAAU,CAAC,wBAAwB,IAAI,KAAK,CAAC;CACxD,MAAM,qBAAqB,SAAS,eAAe;CACnD,MAAM,cAAc,uBAAuB,IAAI,IAAI,gBAAgB,SAAS;CAC5E,MAAM,mBACJ,SAAS,WAAW,IAChB,uBAAuB,IACrB,IACA,IACF,wBAAwB,OAAO,SAAS;CAC9C,MAAM,KAAK,aAAa,kBAAkB,WAAW;CACrD,MAAM,cAAc,SAAS,SAAS,YAAY,QAAQ,aAAa;CAEvE,MAAM,iBAAiB,SAAS,eAAe,QAC5C,WAAW,MAAM,kBAAkB,UAAU,KAAK,CACrD;CACA,MAAM,eAAe,SAAS,eAAe,QAAQ,UAAU;EAC7D,KAAK,MAAM,kBAAkB,UAAU,OAAO,GAAG,OAAO;EACxD,OAAO,gBAAgB,aAAa,MAAM,oBAAoB,CAAC,GAAG,KAAK;CACzE,CAAC,CAAC,CAAC;CAEH,MAAM,uBAAuB,SAAS,QAAQ,YAAY,QAAQ,cAAc,SAAS,CAAC,CAAC,CAAC;CAC5F,MAAM,oBAAoB,SAAS,kBAC/B,YAAY,QACT,QAAQ,CAAC,SAAS,gBAAiB,MAAM,aAAa,gBAAgB,KAAK,QAAQ,CAAC,CACvF,IACA,CAAC;CAEL,OAAO;EACL;EACA;EACA;EACA,yBAAyB,CAAC,GAAG,uBAAuB,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;EAC1E;EACA;EACA;EACA;EACA;EACA,sBAAsB,eAAe,WAAW,IAAI,OAAO,eAAe,eAAe;EACzF,kBAAkB,SAAS,WAAW,IAAI,OAAO,uBAAuB,SAAS;EACjF,wBACE,SAAS,oBAAoB,KAAA,IACzB,OACA,YAAY,WAAW,IACrB,SAAS,WAAW,IAClB,OACA,KACD,YAAY,SAAS,kBAAkB,UAAU,YAAY;EACtE,oBAAoB,uBAAuB,KAAK,SAAS,SAAS;CACpE;AACF;AAEA,SAAS,sBACP,QACA,UACqB;CACrB,IAAI,OAAO,WAAW,GAAG,uBAAO,IAAI,IAAI;CACxC,MAAM,oBAAoB,OAAO,SAAS;CAW1C,MAAM,aAAa,oBAVJ,OAAO,KAAK,UAAU,CACnC,GAAG,SAAS,KAAK,YAAY;EAC3B,IAAI,CAAC,oBAAoB,SAAS,KAAK,GAAG,OAAO;EACjD,MAAM,eACH,MAAM,kBAAkB,UAAU,KAAK,KACxC,gBAAgB,QAAQ,eAAe,MAAM,oBAAoB,CAAC,GAAG,KAAK;EAC5E,OAAO,oBAAoB,OAAO,WAAW;CAC/C,CAAC,GACD,GAAG,MAAM,KAAK,EAAE,QAAQ,OAAO,OAAO,SAAS,CAAC,CAClD,CAC4C,GAAG,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC;CACnE,MAAM,0BAAU,IAAI,IAAoB;CACxC,KAAK,MAAM,CAAC,YAAY,WAAW,WAAW,QAAQ,GAAG;EACvD,IAAI,SAAS,KAAK,UAAU,SAAS,QAAQ;EAC7C,IAAI,CAAC,oBAAoB,SAAS,SAAU,OAAO,WAAY,GAAG;EAClE,QAAQ,IAAI,YAAY,MAAM;CAChC;CACA,OAAO;AACT;AAEA,eAAsB,oBACpB,SACiC;CACjC,yBAAyB,OAAO;CAChC,MAAM,6BAAY,IAAI,KAAK,EAAA,CAAE,YAAY;CACzC,MAAM,cAAc,QAAQ,eAAe;CAC3C,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,MAAM,iBAAiB,KAAK,IAC1B,QAAQ,kBAAkB,GAC1B,QAAQ,QAAQ,SAAS,QAAQ,MAAM,SAAS,WAClD;CACA,MAAM,OAAO,cAAc,QAAQ,OAAO,QAAQ,SAAS,aAAa,eAAe;CACvF,MAAM,eAA8C,CAAC;CACrD,IAAI,SAAS;CACb,MAAM,SAAS,YAA2B;EACxC,OAAO,SAAS,KAAK,QAAQ;GAC3B,QAAQ,QAAQ,eAAe;GAC/B,MAAM,MAAM,KAAK;GACjB,aAAa,KAAK,MAAM,gBAAgB,KAAK,QAAQ,QAAQ,QAAQ,eAAe,CAAC;EACvF;CACF;CACA,MAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,eAAe,GAAG,MAAM,CAAC;CAChE,MAAM,cAAc,IAAI,IAAI,QAAQ,QAAQ,KAAK,QAAQ,UAAU,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;CACtF,MAAM,YAAY,IAAI,IAAI,QAAQ,MAAM,KAAK,UAAU,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;CACtF,aAAa,MACV,GAAG,OACD,YAAY,IAAI,EAAE,QAAQ,KAAK,MAAM,YAAY,IAAI,EAAE,QAAQ,KAAK,OACpE,UAAU,IAAI,EAAE,MAAM,KAAK,MAAM,UAAU,IAAI,EAAE,MAAM,KAAK,MAC7D,EAAE,aAAa,EAAE,UACrB;CACA,OAAO;EACL,YAAY;GACV,GAAG,QAAQ;GACX;GACA,0BAAS,IAAI,KAAK,EAAA,CAAE,YAAY;GAChC,WAAW,QAAQ,MAAM;GACzB,WAAW,QAAQ,QAAQ,KAAK,WAAW,OAAO,EAAE;GACpD;GACA;GACA;EACF;EACA;EACA,WAAW,QAAQ,QAAQ,KAAK,WAC9B,gBACE,OAAO,IACP,aAAa,QAAQ,gBAAgB,YAAY,aAAa,OAAO,EAAE,CACzE,CACF;CACF;AACF;AAEA,SAAgB,wBAAwB,SAIK;CAC3C,OAAO;EACL,IAAI,QAAQ;EACZ,MAAM,QAAQ,OAAO,SAAS;GAC5B,MAAM,SAAS,MAAM,QAAQ,SAAS,IACpC,GAAG,QAAQ,GAAG,GAAG,QAAQ,OAAO,GAAG,QAAQ,cAC3C,OACA;IAAE,GAAG,QAAQ;IAAY,QAAQ,QAAQ;GAAO,CAClD;GACA,OAAO;IACL,UAAU,OAAO;IACjB,OAAO,mBAAmB,MAAM;IAChC,UAAU,EAAE,YAAY,OAAO;GACjC;EACF;CACF;AACF;AAEA,eAAe,gBACb,KAMA,QACA,iBACsC;CACtC,MAAM,UAAU,YAAY,IAAI;CAChC,IAAI;EACF,MAAM,SAAS,MAAM,IAAI,OAAO,QAAQ,IAAI,SAAS,OAAO;GAC1D,QAAQ,IAAI,SAAS;GACrB,YAAY,IAAI;GAChB;EACF,CAAC;EACD,OAAO;GACL,UAAU,IAAI,OAAO;GACrB,QAAQ,IAAI,SAAS;GACrB,YAAY,IAAI;GAChB,gBAAgB,IAAI;GACpB,WAAW,YAAY,IAAI,IAAI;GAC/B,UAAU,OAAO;GACjB,OAAO,qBAAqB,IAAI,UAAU,OAAO,QAAQ;GACzD,oBAAoB,kBAChB,MAAM,uBAAuB,IAAI,UAAU,OAAO,UAAU,iBAAiB,MAAM,IACnF,KAAA;GACJ,UAAU,CAAC,GAAI,IAAI,SAAS,QAAQ,CAAC,CAAE;GACvC,cAAc,IAAI,SAAS;GAC3B,OAAO,OAAO;GACd,gBAAgB,OAAO;EACzB;CACF,SAAS,OAAO;EACd,IAAI,QAAQ,SAAS,MAAM;EAC3B,MAAM,WAA6B,CAAC;EACpC,OAAO;GACL,UAAU,IAAI,OAAO;GACrB,QAAQ,IAAI,SAAS;GACrB,YAAY,IAAI;GAChB,gBAAgB,IAAI;GACpB,WAAW,YAAY,IAAI,IAAI;GAC/B;GACA,OAAO,qBAAqB,IAAI,UAAU,QAAQ;GAClD,UAAU,CAAC,GAAI,IAAI,SAAS,QAAQ,CAAC,CAAE;GACvC,cAAc,IAAI,SAAS;GAC3B,OAAO;IACL,OAAO,iBAAiB,QAAQ,MAAM,YAAY,OAAO;IACzD,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAChE;EACF;CACF;AACF;AAEA,SAAS,cACP,OACA,SACA,aACA,iBAMC;CACD,MAAM,gBAAgB,CAAC,GAAG,OAAO,CAAC,CAAC,MAChC,MAAM,UACL,WAAW,GAAG,gBAAgB,QAAQ,KAAK,IAAI,IAC7C,WAAW,GAAG,gBAAgB,QAAQ,MAAM,IAAI,KAAK,KAAK,GAAG,cAAc,MAAM,EAAE,CACzF;CACA,IAAI,iBAAiB;CACrB,OAAO,MAAM,SAAS,UAAU,cAC9B,MAAM,KAAK,EAAE,QAAQ,YAAY,IAAI,GAAG,eAAe;EAErD,MAAM,YADa,YAAY,cAAc,cACf,cAAc;EAE5C,OAAO,CADU,GAAG,cAAc,MAAM,QAAQ,GAAG,GAAG,cAAc,MAAM,GAAG,QAAQ,CACxE,CAAC,CAAC,KAAK,YAAY;GAC9B;GACA;GACA;GACA,gBAAgB;EAClB,EAAE;CACJ,CAAC,CAAC,CAAC,KAAK,CACV;AACF;AAEA,eAAe,uBACb,UACA,UACA,UACA,QACoC;CACpC,MAAM,WAAW,SAAS,SAAS,YAAY,QAAQ,aAAa;CACpE,MAAM,WAA0B,CAAC;CACjC,MAAM,qBAAoC,CAAC;CAC3C,MAAM,SAA2C,CAAC;CAClD,KAAK,MAAM,OAAO,UAAU;EAC1B,QAAQ,eAAe;EACvB,IAAI;GACF,IACE,MAAM,SAAS;IACb,QAAQ,SAAS;IACjB,WAAW,SAAS;IACpB,UAAU;IACV;GACF,CAAC,GAED,SAAS,KAAK,GAAG;QAEjB,mBAAmB,KAAK,GAAG;EAE/B,SAAS,OAAO;GACd,IAAI,QAAQ,SAAS,MAAM;GAC3B,OAAO,KAAK;IACV,UAAU;IACV,OAAO,iBAAiB,QAAQ,MAAM,YAAY,OAAO;IACzD,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAChE,CAAC;EACH;CACF;CACA,OAAO;EACL,SAAS,SAAS;EAClB,UAAU,SAAS;EACnB;EACA;EACA,UAAU,SAAS,WAAW,KAAK,OAAO,SAAS,IAAI,OAAO,SAAS,SAAS,SAAS;CAC3F;AACF;AAEA,SAAS,gBACP,UACA,cACyB;CACzB,MAAM,eAAe,aAAa,QAC/B,gBAAgB,YAAY,MAAM,qBAAqB,CAC1D;CACA,MAAM,iBAAiB,aAAa,QACjC,KAAK,gBAAgB,MAAM,YAAY,MAAM,oBAC9C,CACF;CACA,MAAM,gBAAgB,aAAa,QAChC,KAAK,gBAAgB,MAAM,YAAY,MAAM,gBAAgB,QAC9D,CACF;CACA,MAAM,gBAAgB,aAAa,QAChC,KAAK,gBAAgB,MAAM,YAAY,SAAS,QACjD,CACF;CACA,MAAM,oBAAoB,aAAa,QACpC,KAAK,gBAAgB,MAAM,YAAY,MAAM,wBAAwB,QACtE,CACF;CACA,MAAM,cAAc,mBAAmB,IAAI,OAAO,gBAAgB;CAClE,MAAM,mBACJ,mBAAmB,IAAI,OAAO,kBAAkB,IAAI,IAAI,oBAAoB;CAC9E,MAAM,WAAW,aACd,KAAK,gBAAgB,YAAY,MAAM,oBAAoB,CAAC,CAC5D,QAAQ,UAA2B,UAAU,IAAI;CACpD,MAAM,uBAAuB,aAAa,QACvC,KAAK,gBACJ,MAAM,YAAY,SAAS,QAAQ,YAAY,QAAQ,cAAc,SAAS,CAAC,CAAC,CAAC,QACnF,CACF;CACA,MAAM,cAAc,aAAa,QAC9B,KAAK,gBAAgB,MAAM,YAAY,SAAS,QACjD,CACF;CACA,MAAM,uBAAuB,aAAa,QACvC,gBAAgB,YAAY,MAAM,2BAA2B,IAChE;CACA,MAAM,gBAAgB,qBAAqB,QACxC,KAAK,gBACJ,MACA,YAAY,SAAS,QAAQ,OAAO,YAAY,QAAQ,QAAQ,cAAc,QAAQ,CAAC,GACzF,CACF;CACA,MAAM,uBAAuB,qBAAqB,QAC/C,KAAK,gBAAgB,MAAM,YAAY,MAAM,kBAAkB,QAChE,CACF;CACA,MAAM,QAAQ,aAAa,QAAQ,gBAAgB,YAAY,MAAM,uBAAuB,CAAC;CAC7F,MAAM,iBAAiB,MAAM,QAAQ,gBAAgB,CAAC,YAAY,KAAK;CACvE,MAAM,oBAAoB,aAAa,QACpC,KAAK,gBAAgB,OAAO,YAAY,oBAAoB,YAAY,IACzE,CACF;CACA,MAAM,sBAAsB,aAAa,QACtC,KAAK,gBAAgB,OAAO,YAAY,oBAAoB,mBAAmB,UAAU,IAC1F,CACF;CACA,MAAM,2BAA2B,aAAa,QAC3C,KAAK,gBAAgB,OAAO,YAAY,oBAAoB,OAAO,UAAU,IAC9E,CACF;CACA,MAAM,qBAAqB,aAAa,QAAQ,gBAC9C,YAAY,SAAS,MAAM,YAAY,QAAQ,cAAc,SAAS,CAAC,CACzE;CACA,MAAM,gCAAgC,mBAAmB,QACtD,gBACC,CAAC,YAAY,sBAAsB,YAAY,mBAAmB,OAAO,SAAS,CACtF,CAAC,CAAC;CACF,MAAM,SAAS,aAAa,KAAK,gBAAgB,YAAY,KAAK;CAClE,MAAM,eAAe,OAAO,QAAQ,KAAK,UAAU;EACjD,IAAI,CAAC,OAAO,OAAO;EACnB,OAAO,OAAO,MAAM,KAAK,SAAS,eAAgB,MAAM,gBAAgB,IAAK,MAAM,KAAK;CAC1F,GAAG,CAAC;CACJ,OAAO;EACL;EACA,aAAa,aAAa;EAC1B,eAAe,aAAa,QAAQ,gBAAgB,CAAC,YAAY,KAAK,CAAC,CAAC;EACxE,YAAY,aAAa,QAAQ,gBAAgB,QAAQ,YAAY,KAAK,CAAC,CAAC,CAAC;EAC7E;EACA;EACA,IACE,qBAAqB,QAAQ,gBAAgB,OACzC,OACA,aAAa,kBAAkB,WAAW;EAChD,sBAAsB,SAAS,WAAW,IAAI,OAAO,KAAK,QAAQ;EAClE,kBAAkB,gBAAgB,IAAI,OAAO,uBAAuB;EACpE,wBACE,qBAAqB,WAAW,IAC5B,OACA,kBAAkB,IAChB,KACC,gBAAgB,wBAAwB;EACjD,oBACE,mBAAmB,WAAW,KAC9B,gCAAgC,KAChC,oBAAoB,wBAAwB,IACxC,OACA,qBAAqB,oBAAoB;EAC/C;EACA;EACA;EACA,4BACE,eAAe,WAAW,IACtB,OACA,eAAe,QAAQ,gBAAgB,YAAY,MAAM,kBAAkB,CAAC,CAAC,SAC7E,eAAe;EACrB,sBACE,MAAM,WAAW,IACb,OACA,MAAM,QAAQ,gBAAgB,QAAQ,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,MAAM;EAC/E,cAAc,eAAe,aAAa,QAAQ,gBAAgB,CAAC,YAAY,KAAK,CAAC;EACrF,WAAW,oBAAoB,aAAa,KAAK,gBAAgB,YAAY,SAAS,CAAC;EACvF,OAAO,OAAO,QAAQ,KAAK,UAAU,OAAO,OAAO,SAAS,IAAI,CAAC;EACjE,kBAAkB,OAAO,QAAQ,UAAU,CAAC,SAAS,MAAM,UAAU,IAAI,CAAC,CAAC;EAC3E,aAAa,OAAO,QAAQ,KAAK,UAAU,OAAO,OAAO,QAAQ,SAAS,IAAI,CAAC;EAC/E,cAAc,OAAO,QAAQ,KAAK,UAAU,OAAO,OAAO,QAAQ,UAAU,IAAI,CAAC;EACjF,iBAAiB,OAAO,QAAQ,KAAK,UAAU,OAAO,OAAO,QAAQ,aAAa,IAAI,CAAC;EACvF,cAAc,OAAO,QAAQ,KAAK,UAAU,OAAO,OAAO,QAAQ,UAAU,IAAI,CAAC;EACjF,kBAAkB,OAAO,QAAQ,KAAK,UAAU,OAAO,OAAO,QAAQ,cAAc,IAAI,CAAC;EACzF,uBAAuB,OAAO,QAAQ,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;EAChE,gCAAgC,OAAO,QAAQ,UAAU,OAAO,QAAQ,cAAc,KAAA,CAAS,CAAC,CAC7F;EACH,6BAA6B,OAAO,QAAQ,UAAU,OAAO,QAAQ,WAAW,KAAA,CAAS,CAAC,CACvF;EACH,iCAAiC,OAAO,QACrC,UAAU,OAAO,QAAQ,eAAe,KAAA,CAC3C,CAAC,CAAC;EACF;EACA,iBAAiB,OAAO,QAAQ,UAAU,CAAC,SAAS,MAAM,KAAK,SAAS,YAAY,CAAC,CAAC;CACxF;AACF;AAEA,SAAS,oBAAoB,SAAyB,OAAyC;CAC7F,IAAI,MAAM,cAAc,CAAC,MAAM,WAAW,SAAS,QAAQ,UAAU,GAAG,OAAO;CAC/E,IAAI,MAAM,SAAS,CAAC,MAAM,MAAM,SAAS,QAAQ,IAAI,GAAG,OAAO;CAC/D,IAAI,MAAM,aAAa,CAAC,QAAQ,WAAW,CAAC,MAAM,SAAS,SAAS,QAAQ,OAAO,IACjF,OAAO;CACT,IACE,MAAM,YACN,CAAC,gBAAgB,QAAQ,eAAe,MAAM,UAAU,MAAM,gBAAgB,KAAK,GAEnF,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBACP,QACA,UACA,MACS;CACT,IAAI,SAAS,WAAW,GAAG,OAAO;CAClC,MAAM,SAAS,WACb,OAAO,MAAM,QAAQ,gBAAgB,KAAK,MAAM,CAAC;CACnD,OAAO,SAAS,QAAQ,SAAS,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK;AACrE;AAEA,SAAS,gBAAgB,QAAqB,UAA+C;CAC3F,OACE,OAAO,QAAQ,SAAS,QAAQ,SAAS,SAAS,KAAA,KAAa,OAAO,SAAS,SAAS;AAE5F;AAEA,SAAS,kBAAkB,KAAyD;CAClF,MAAM,QAAQ,qCAAqC,KAAK,GAAG;CAC3D,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI;EACF,MAAM,UAAU,mBAAmB,MAAM,EAAG;EAC5C,MAAM,SAAS,mBAAmB,MAAM,EAAG;EAC3C,OAAO,WAAW,SAAS;GAAE;GAAS;EAAO,IAAI;CACnD,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,aACP,UACM;CACN,IAAI,CAAC,SAAS,GAAG,KAAK,GAAG,MAAM,IAAI,UAAU,6CAA6C;CAC1F,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,MAAM,SAAS,SAAS,gBAAgB;EAC3C,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,MAAM,IAAI,UAAU,GAAG,SAAS,GAAG,sCAAsC;EAC/F,IAAI,IAAI,IAAI,MAAM,EAAE,GAClB,MAAM,IAAI,UAAU,GAAG,SAAS,GAAG,iCAAiC,MAAM,GAAG,EAAE;EACjF,IAAI,IAAI,MAAM,EAAE;EAChB,IACE,CAAC,MAAM,YAAY,UACnB,CAAC,MAAM,OAAO,UACd,CAAC,MAAM,UAAU,UACjB,CAAC,MAAM,UAAU,QAEjB,MAAM,IAAI,UACR,GAAG,SAAS,GAAG,GAAG,MAAM,GAAG,2EAC7B;CAEJ;CACA,KAAK,MAAM,OAAO,SAAS,mBAAmB,CAAC,GAC7C,IAAI,CAAC,IAAI,IAAI,KAAK,GAChB,MAAM,IAAI,UAAU,GAAG,SAAS,GAAG,yCAAyC;AAGlF;AAEA,SAAS,yBAAiC,SAAmD;CAC3F,IAAI,QAAQ,MAAM,WAAW,GAAG,MAAM,IAAI,UAAU,oCAAoC;CACxF,IAAI,QAAQ,QAAQ,WAAW,GAAG,MAAM,IAAI,UAAU,sCAAsC;CAC5F,MAAM,cAAc,QAAQ,eAAe;CAC3C,MAAM,iBAAiB,QAAQ,kBAAkB;CACjD,IAAI,CAAC,OAAO,cAAc,WAAW,KAAK,cAAc,GACtD,MAAM,IAAI,WAAW,iEAAiE;CAExF,IAAI,CAAC,OAAO,cAAc,cAAc,KAAK,iBAAiB,GAC5D,MAAM,IAAI,WAAW,oEAAoE;CAE3F,IAAI,CAAC,OAAO,cAAc,QAAQ,mBAAmB,CAAC,GACpD,MAAM,IAAI,WAAW,4DAA4D;CAEnF,qBACE,QAAQ,MAAM,KAAK,aAAa,SAAS,EAAE,GAC3C,MACF;CACA,qBACE,QAAQ,QAAQ,KAAK,WAAW,OAAO,EAAE,GACzC,QACF;CACA,KAAK,MAAM,YAAY,QAAQ,OAAO,aAAa,QAAQ;AAC7D;AAEA,SAAS,qBAAqB,QAA2B,OAAqB;CAC5E,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,SAAS,QAAQ;EAC1B,IAAI,CAAC,MAAM,KAAK,GAAG,MAAM,IAAI,UAAU,qBAAqB,MAAM,sBAAsB;EACxF,IAAI,KAAK,IAAI,KAAK,GAAG,MAAM,IAAI,UAAU,+BAA+B,MAAM,OAAO,MAAM,EAAE;EAC7F,KAAK,IAAI,KAAK;CAChB;AACF;AAEA,SAAS,aAAa,GAAW,GAAmB;CAClD,OAAO,IAAI,MAAM,IAAI,IAAK,IAAI,IAAI,KAAM,IAAI;AAC9C;AAEA,SAAS,KAAK,QAAmC;CAC/C,OAAO,OAAO,WAAW,IAAI,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAC1F;AAEA,SAAS,oBAAoB,QAAuD;CAClF,MAAM,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC/C,OAAO;EACL,KAAK,OAAO,MAAM;EAClB,MAAM,KAAK,MAAM;EACjB,KAAK,WAAW,QAAQ,EAAG;EAC3B,KAAK,WAAW,QAAQ,GAAI;EAC5B,KAAK,OAAO,GAAG,EAAE,KAAK;CACxB;AACF;AAEA,SAAS,WAAW,QAA2B,UAA0B;CACvE,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,OAAO,OAAO,KAAK,KAAK,WAAW,OAAO,MAAM,IAAI,MAAM,OAAO,GAAG,EAAE,KAAK;AAC7E;AAEA,SAAS,WAAW,OAAuB;CACzC,IAAI,OAAO;CACX,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,QAAQ,MAAM,WAAW,KAAK;EAC9B,OAAO,KAAK,KAAK,MAAM,QAAQ;CACjC;CACA,OAAO,SAAS;AAClB;AAEA,SAAS,eAAe,cAAqE;CAC3F,MAAM,yBAAS,IAAI,IAA2C;CAC9D,KAAK,MAAM,eAAe,cAAc;EACtC,MAAM,OAAO,OAAO,IAAI,YAAY,MAAM,KAAK,CAAC;EAChD,KAAK,KAAK,WAAW;EACrB,OAAO,IAAI,YAAY,QAAQ,IAAI;CACrC;CACA,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,QAAQ,OAAO,OAAO,GAC/B,KAAK,IAAI,OAAO,GAAG,OAAO,KAAK,QAAQ,QACrC,KAAK,IAAI,QAAQ,OAAO,GAAG,QAAQ,KAAK,QAAQ,SAC9C,WAAW,KACT,QAAQ,KAAK,KAAK,CAAE,MAAM,iBAAiB,KAAK,MAAM,CAAE,MAAM,eAAe,CAC/E;CAIN,OAAO,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU;AACzD;AAEA,SAAS,QAAQ,MAAyB,OAAkC;CAC1E,MAAM,IAAI,IAAI,IAAI,IAAI;CACtB,MAAM,IAAI,IAAI,IAAI,KAAK;CACvB,MAAM,wBAAQ,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;CAClC,IAAI,MAAM,SAAS,GAAG,OAAO;CAC7B,IAAI,eAAe;CACnB,KAAK,MAAM,SAAS,GAAG,IAAI,EAAE,IAAI,KAAK,GAAG,gBAAgB;CACzD,OAAO,eAAe,MAAM;AAC9B;AAEA,SAAS,mBAAmB,QAA+C;CACzE,MAAM,SAAS,OAAO,YAAY,KAAK,YAAY,QAAQ,KAAK;CAChE,MAAM,QAAQ,OAAO,OAAO,UAAU,MAAM,UAAU,IAAI,IACtD,OAAO,QAAQ,KAAK,UAAU,OAAO,MAAM,SAAS,IAAI,CAAC,IACzD;CACJ,MAAM,SAAS,OAAO,OAAO,UAAU,MAAM,WAAW,IAAI,IACxD,OAAO,QACJ,KAAK,WAAW;EACf,OAAO,IAAI,SAAS,MAAM,QAAQ,SAAS;EAC3C,QAAQ,IAAI,UAAU,MAAM,QAAQ,UAAU;EAC9C,WAAW,IAAI,aAAa,MAAM,QAAQ,aAAa;EACvD,QAAQ,IAAI,UAAU,MAAM,QAAQ,UAAU;EAC9C,YAAY,IAAI,cAAc,MAAM,QAAQ,cAAc;CAC5D,IACA;EAAE,OAAO;EAAG,QAAQ;EAAG,WAAW;EAAG,QAAQ;EAAG,YAAY;CAAE,CAChE,IACA;CACJ,MAAM,eAAe,OAAO,QACzB,KAAK,UACJ,OAAO,MAAM,KAAK,SAAS,eAAgB,MAAM,gBAAgB,IAAK,MAAM,KAAK,MACnF,CACF;CACA,MAAM,OAAO,OAAO,MAAM,UAAU,MAAM,KAAK,SAAS,YAAY,IAC/D;EAAE,MAAM;EAAc,KAAK;CAAK,IACjC,OAAO,MAAM,UAAU,MAAM,KAAK,SAAS,WAAW,IACnD;EAAE,MAAM;EAAa,KAAK;CAAa,IACvC;EAAE,MAAM;EAAY,KAAK;CAAa;CAC7C,OAAO;EACL;EACA;EACA;EACA,GAAI,KAAK,SAAS,eAAe,EAAE,aAAa,IAAI,CAAC;CACvD;AACF"}