@tangle-network/agent-eval 0.131.1 → 0.133.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/analyst/index.d.ts +5 -5
  3. package/dist/analyst/index.js +3 -3
  4. package/dist/{analyze-runs-CBnCgfse.d.ts → analyze-runs-DZr7JW-m.d.ts} +5 -5
  5. package/dist/{analyze-runs-CBnCgfse.d.ts.map → analyze-runs-DZr7JW-m.d.ts.map} +1 -1
  6. package/dist/benchmarks/index.d.ts +1 -1
  7. package/dist/benchmarks/index.js +1 -1
  8. package/dist/{benchmarks-BDtRiOvE.js → benchmarks-DncrVrSr.js} +3 -3
  9. package/dist/{benchmarks-BDtRiOvE.js.map → benchmarks-DncrVrSr.js.map} +1 -1
  10. package/dist/campaign/index.d.ts +4 -4
  11. package/dist/campaign/index.js +2 -2
  12. package/dist/{campaign-CLG6y9jF.js → campaign-CyQMCCEU.js} +4 -4
  13. package/dist/{campaign-CLG6y9jF.js.map → campaign-CyQMCCEU.js.map} +1 -1
  14. package/dist/cli.js +1 -1
  15. package/dist/{client-w90OcvvR.d.ts → client-D4F9hdzR.d.ts} +4 -4
  16. package/dist/{client-w90OcvvR.d.ts.map → client-D4F9hdzR.d.ts.map} +1 -1
  17. package/dist/contract/index.d.ts +188 -22
  18. package/dist/contract/index.d.ts.map +1 -1
  19. package/dist/contract/index.js +1663 -65
  20. package/dist/contract/index.js.map +1 -1
  21. package/dist/control.d.ts +1 -1
  22. package/dist/{cost-ledger-DIgQUFZZ.js → cost-ledger-BrJxbrMy.js} +278 -9
  23. package/dist/cost-ledger-BrJxbrMy.js.map +1 -0
  24. package/dist/{cost-ledger-Dye6jCgg.d.ts → cost-ledger-fGS_u_O1.d.ts} +18 -2
  25. package/dist/cost-ledger-fGS_u_O1.d.ts.map +1 -0
  26. package/dist/{default-registry-Cj1oUpLN.d.ts → default-registry-Cl3pHo4n.d.ts} +4 -4
  27. package/dist/{default-registry-Cj1oUpLN.d.ts.map → default-registry-Cl3pHo4n.d.ts.map} +1 -1
  28. package/dist/{default-registry-C-vFCSEc.js → default-registry-D3T9XbuY.js} +4 -13
  29. package/dist/default-registry-D3T9XbuY.js.map +1 -0
  30. package/dist/{eval-campaign-C2k-m4aY.js → eval-campaign-CYkUME2T.js} +2 -2
  31. package/dist/{eval-campaign-C2k-m4aY.js.map → eval-campaign-CYkUME2T.js.map} +1 -1
  32. package/dist/{task-failure-attributes-CQZlB3et.js → extract-usage-2j25whHw.js} +154 -2
  33. package/dist/extract-usage-2j25whHw.js.map +1 -0
  34. package/dist/fuzz.d.ts +1 -1
  35. package/dist/fuzz.js +1 -1
  36. package/dist/hosted/index.d.ts +2 -2
  37. package/dist/{index-B11XCkdf2.d.ts → index-3cdlURSk.d.ts} +2 -2
  38. package/dist/index-3cdlURSk.d.ts.map +1 -0
  39. package/dist/{index-VTypFU3t.d.ts → index-B_eUhU9K.d.ts} +7 -7
  40. package/dist/{index-VTypFU3t.d.ts.map → index-B_eUhU9K.d.ts.map} +1 -1
  41. package/dist/{index-DNXoNaFF.d.ts → index-C2fkZhv_.d.ts} +2 -2
  42. package/dist/{index-DNXoNaFF.d.ts.map → index-C2fkZhv_.d.ts.map} +1 -1
  43. package/dist/{index-NPeSWD98.d.ts → index-CiP2DRCj.d.ts} +5 -5
  44. package/dist/{index-NPeSWD98.d.ts.map → index-CiP2DRCj.d.ts.map} +1 -1
  45. package/dist/index.d.ts +69 -23
  46. package/dist/index.d.ts.map +1 -1
  47. package/dist/index.js +100 -12
  48. package/dist/index.js.map +1 -1
  49. package/dist/{llm-client-B_nIBlYo.d.ts → llm-client-BiK4HW0u.d.ts} +2 -2
  50. package/dist/{llm-client-B_nIBlYo.d.ts.map → llm-client-BiK4HW0u.d.ts.map} +1 -1
  51. package/dist/{llm-client--GR4JbZE.js → llm-client-ClPW-dWB.js} +2 -2
  52. package/dist/{llm-client--GR4JbZE.js.map → llm-client-ClPW-dWB.js.map} +1 -1
  53. package/dist/meta-eval/index.d.ts +216 -3
  54. package/dist/{index-D_F6VAKe.d.ts.map → meta-eval/index.d.ts.map} +1 -1
  55. package/dist/meta-eval/index.js +91 -1
  56. package/dist/meta-eval/index.js.map +1 -1
  57. package/dist/multishot/index.d.ts +1 -1
  58. package/dist/openapi.json +1 -1
  59. package/dist/{release-report-Crg9oFJ0.d.ts → release-report-DfmKSIEE.d.ts} +3 -3
  60. package/dist/{release-report-Crg9oFJ0.d.ts.map → release-report-DfmKSIEE.d.ts.map} +1 -1
  61. package/dist/{replay-RE97Ckjl.d.ts → replay-BI6CVKkp.d.ts} +2 -2
  62. package/dist/{replay-RE97Ckjl.d.ts.map → replay-BI6CVKkp.d.ts.map} +1 -1
  63. package/dist/{replay-D18-pBAA.js → replay-BMR4TEYY.js} +2 -3
  64. package/dist/{replay-D18-pBAA.js.map → replay-BMR4TEYY.js.map} +1 -1
  65. package/dist/reporting.d.ts +4 -4
  66. package/dist/{researcher-Q5rpPqZY.d.ts → researcher-DMimgHtN.d.ts} +4 -4
  67. package/dist/{researcher-Q5rpPqZY.d.ts.map → researcher-DMimgHtN.d.ts.map} +1 -1
  68. package/dist/{reward-hacking-CW-3HN0n.d.ts → reward-hacking-D-QqXvg-.d.ts} +2 -2
  69. package/dist/{reward-hacking-CW-3HN0n.d.ts.map → reward-hacking-D-QqXvg-.d.ts.map} +1 -1
  70. package/dist/rl.d.ts +156 -6
  71. package/dist/rl.d.ts.map +1 -1
  72. package/dist/rl.js +217 -2
  73. package/dist/rl.js.map +1 -1
  74. package/dist/rollout/index.d.ts +1 -1
  75. package/dist/{rubric-predictive-validity-lXLmashy.d.ts → rubric-predictive-validity-C1dCLcvb.d.ts} +2 -2
  76. package/dist/{rubric-predictive-validity-lXLmashy.d.ts.map → rubric-predictive-validity-C1dCLcvb.d.ts.map} +1 -1
  77. package/dist/{run-evidence-oByzm-dE.d.ts → run-evidence-DokQtX0-.d.ts} +2 -2
  78. package/dist/{run-evidence-oByzm-dE.d.ts.map → run-evidence-DokQtX0-.d.ts.map} +1 -1
  79. package/dist/run-record-CN8Zd21B.js.map +1 -1
  80. package/dist/{run-record-BJnYdTxO.d.ts → run-record-DcObtIGh.d.ts} +4 -14
  81. package/dist/run-record-DcObtIGh.d.ts.map +1 -0
  82. package/dist/{semantic-concept-judge-b5m3irbR.js → semantic-concept-judge-BypLt6Fw.js} +4 -4
  83. package/dist/{semantic-concept-judge-b5m3irbR.js.map → semantic-concept-judge-BypLt6Fw.js.map} +1 -1
  84. package/dist/{server-m5D9cvnG.js → server-BPqlDBWK.js} +3 -3
  85. package/dist/{server-m5D9cvnG.js.map → server-BPqlDBWK.js.map} +1 -1
  86. package/dist/{skill-usage-D5mlWdAJ.d.ts → skill-usage-BaaxFSJR.d.ts} +5 -5
  87. package/dist/{skill-usage-D5mlWdAJ.d.ts.map → skill-usage-BaaxFSJR.d.ts.map} +1 -1
  88. package/dist/{skillopt-optimization-method-D0MVjJdP.d.ts → skillopt-optimization-method-CAASpcS3.d.ts} +9 -9
  89. package/dist/{skillopt-optimization-method-D0MVjJdP.d.ts.map → skillopt-optimization-method-CAASpcS3.d.ts.map} +1 -1
  90. package/dist/{skillopt-optimization-method-eOJL2570.js → skillopt-optimization-method-DgN7U9iR.js} +3 -3
  91. package/dist/{skillopt-optimization-method-eOJL2570.js.map → skillopt-optimization-method-DgN7U9iR.js.map} +1 -1
  92. package/dist/{statistics-Cmj6nynr.d.ts → statistics-DbvkkDPa.d.ts} +2 -2
  93. package/dist/{statistics-Cmj6nynr.d.ts.map → statistics-DbvkkDPa.d.ts.map} +1 -1
  94. package/dist/{summary-report-CWwB_LiV.d.ts → summary-report-DnUcjVpV.d.ts} +2 -2
  95. package/dist/{summary-report-CWwB_LiV.d.ts.map → summary-report-DnUcjVpV.d.ts.map} +1 -1
  96. package/dist/traces.d.ts +2 -2
  97. package/dist/traces.js +2 -2
  98. package/dist/{types-CsD5nTfV.d.ts → types-BokuXvOG.d.ts} +4 -4
  99. package/dist/{types-CsD5nTfV.d.ts.map → types-BokuXvOG.d.ts.map} +1 -1
  100. package/dist/{types-DGsxbAEd.d.ts → types-Cc3qbqzj.d.ts} +3 -3
  101. package/dist/{types-DGsxbAEd.d.ts.map → types-Cc3qbqzj.d.ts.map} +1 -1
  102. package/dist/wire/index.d.ts +2 -2
  103. package/dist/wire/index.js +1 -1
  104. package/package.json +3 -8
  105. package/dist/belief-state/index.d.ts +0 -622
  106. package/dist/belief-state/index.d.ts.map +0 -1
  107. package/dist/belief-state/index.js +0 -1819
  108. package/dist/belief-state/index.js.map +0 -1
  109. package/dist/calibration-CNWWA6K8.js +0 -94
  110. package/dist/calibration-CNWWA6K8.js.map +0 -1
  111. package/dist/code-agent-session-BjkMTQ7H.js +0 -1390
  112. package/dist/code-agent-session-BjkMTQ7H.js.map +0 -1
  113. package/dist/code-agent-session-aYa3SGKz.d.ts +0 -143
  114. package/dist/code-agent-session-aYa3SGKz.d.ts.map +0 -1
  115. package/dist/cost-ledger-DIgQUFZZ.js.map +0 -1
  116. package/dist/cost-ledger-Dye6jCgg.d.ts.map +0 -1
  117. package/dist/default-registry-C-vFCSEc.js.map +0 -1
  118. package/dist/extract-usage-BrQ8mCLX.js +0 -155
  119. package/dist/extract-usage-BrQ8mCLX.js.map +0 -1
  120. package/dist/index-B11XCkdf2.d.ts.map +0 -1
  121. package/dist/index-D_F6VAKe.d.ts +0 -217
  122. package/dist/metrics-C9YY1OcL.js +0 -239
  123. package/dist/metrics-C9YY1OcL.js.map +0 -1
  124. package/dist/off-policy-DvgzvtIx.js +0 -220
  125. package/dist/off-policy-DvgzvtIx.js.map +0 -1
  126. package/dist/off-policy-mskQw8Mb.d.ts +0 -153
  127. package/dist/off-policy-mskQw8Mb.d.ts.map +0 -1
  128. package/dist/run-record-BJnYdTxO.d.ts.map +0 -1
  129. package/dist/runtime-trajectory-1gyaTOoC.js +0 -93
  130. package/dist/runtime-trajectory-1gyaTOoC.js.map +0 -1
  131. package/dist/runtime-trajectory-BXxG4lyi.d.ts +0 -50
  132. package/dist/runtime-trajectory-BXxG4lyi.d.ts.map +0 -1
  133. package/dist/task-failure-attributes-CQZlB3et.js.map +0 -1
@@ -1,220 +0,0 @@
1
- import { s as ValidationError } from "./errors-8YnH8WlF.js";
2
- //#region src/rl/off-policy.ts
3
- /**
4
- * Off-policy evaluation primitives.
5
- *
6
- * Standard inverse-probability-weighted (IPS), self-normalized
7
- * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
8
- * value of a *target* policy given trajectories collected under a
9
- * *behavior* policy. This is the canonical RL eval task: "we have last
10
- * week's runs, we changed the policy — how would the new one do without
11
- * re-running?"
12
- *
13
- * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
14
- * & Joachims 2015 for SNIPS) but the *application* to LLM-agent
15
- * evaluation needs care:
16
- *
17
- * - The "policy" is the (prompt, tool config, model snapshot) triple.
18
- * Two policies have the same probability over an action *iff* their
19
- * LLM call would emit the same token with the same probability —
20
- * which is generally unknowable without the model log-probs.
21
- * - For LLM agents, propensity scores must be supplied by the caller
22
- * (logged in the trace, recovered from token log-probs, or estimated
23
- * via a learned propensity model). We do NOT estimate propensity here.
24
- * - Doubly-robust requires two outputs from a Q-function: its prediction
25
- * for the logged action and its expectation under the target policy.
26
- * Consumers compute these with a tabular estimate, regression fit, or
27
- * learned reward model before constructing the trajectories.
28
- *
29
- * Bias / variance tradeoffs:
30
- * - IPS: unbiased; high variance for small overlap, infinite variance
31
- * when target has support outside behavior.
32
- * - SNIPS: lower variance, slight bias; usually preferred in practice.
33
- * - DR: doubly-robust — unbiased if either propensity OR Q-function is
34
- * correct. Lowest practical variance when Q is decent. Use this.
35
- *
36
- * Caveat the panel will land: on the LLM-agent setting, propensity scores
37
- * recovered from token log-probs are noisy, the action space is enormous,
38
- * and overlap is often poor. These estimators are useful but not magic;
39
- * complement with `replayCampaign` (exact replay where the request hashes
40
- * match) for high-confidence answers and OPE for the gap.
41
- */
42
- /**
43
- * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
44
- * of E[reward under target policy]. Variance scales with the spread of
45
- * target/behavior ratios.
46
- */
47
- function inverseProbabilityWeighting(trajectories, opts = {}) {
48
- const cap = opts.weightCap ?? Infinity;
49
- const clip = opts.rewardClip ?? {
50
- low: 0,
51
- high: 1
52
- };
53
- if (trajectories.length === 0) return zeroEstimate();
54
- const weights = [];
55
- const weightedRewards = [];
56
- let maxW = 0;
57
- for (const t of trajectories) {
58
- if (t.behaviorProb <= 0) throw new ValidationError(`inverseProbabilityWeighting: behaviorProb must be > 0 (runId=${t.runId})`);
59
- const w = Math.min(cap, t.targetProb / t.behaviorProb);
60
- const r = clamp(t.reward, clip.low, clip.high);
61
- weights.push(w);
62
- weightedRewards.push(w * r);
63
- if (w > maxW) maxW = w;
64
- }
65
- const n = weights.length;
66
- const value = weightedRewards.reduce((s, x) => s + x, 0) / n;
67
- const variance = weightedRewards.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1);
68
- const sumW = weights.reduce((s, w) => s + w, 0);
69
- const sumW2 = weights.reduce((s, w) => s + w * w, 0);
70
- const effN = sumW === 0 ? 0 : sumW * sumW / sumW2;
71
- return {
72
- value,
73
- standardError: Math.sqrt(variance / n),
74
- effectiveSampleSize: effN,
75
- n,
76
- maxImportanceWeight: maxW
77
- };
78
- }
79
- /**
80
- * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
81
- * the cost of small bias (vanishing as N grows). The right default for
82
- * LLM-agent evaluation where overlap is often poor.
83
- */
84
- function selfNormalizedImportanceWeighting(trajectories, opts = {}) {
85
- const cap = opts.weightCap ?? Infinity;
86
- const clip = opts.rewardClip ?? {
87
- low: 0,
88
- high: 1
89
- };
90
- if (trajectories.length === 0) return zeroEstimate();
91
- const weights = [];
92
- const rewards = [];
93
- let maxW = 0;
94
- for (const t of trajectories) {
95
- if (t.behaviorProb <= 0) throw new ValidationError(`selfNormalizedImportanceWeighting: behaviorProb must be > 0 (runId=${t.runId})`);
96
- const w = Math.min(cap, t.targetProb / t.behaviorProb);
97
- weights.push(w);
98
- rewards.push(clamp(t.reward, clip.low, clip.high));
99
- if (w > maxW) maxW = w;
100
- }
101
- const sumW = weights.reduce((s, w) => s + w, 0);
102
- const sumWR = weights.reduce((s, w, i) => s + w * rewards[i], 0);
103
- const value = sumW === 0 ? 0 : sumWR / sumW;
104
- const sumW2 = weights.reduce((s, w) => s + w * w, 0);
105
- const effN = sumW === 0 ? 0 : sumW * sumW / sumW2;
106
- const variance = weights.map((w, i) => w * (rewards[i] - value)).reduce((s, x) => s + x * x, 0) / Math.max(1, sumW * sumW);
107
- return {
108
- value,
109
- standardError: Math.sqrt(variance),
110
- effectiveSampleSize: effN,
111
- n: trajectories.length,
112
- maxImportanceWeight: maxW
113
- };
114
- }
115
- /**
116
- * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
117
- *
118
- * V_DR = (1/N) * sum_i [ v_hat_target_i
119
- * + (target_prob_i / behavior_prob_i) * (r_i - q_hat_chosen_i) ]
120
- *
121
- * Unbiased if EITHER:
122
- * - the importance ratios are correct (IPS-style validity), OR
123
- * - the Q-hat function is correct (model-based validity).
124
- *
125
- * In practice both are imperfect, but the residual bias is the *product*
126
- * of both errors — much smaller than either alone. This is why DR is the
127
- * default in production OPE pipelines.
128
- *
129
- * `qHatChosen` and `vHatTarget` must be supplied together. Rows with neither
130
- * use the exact IPS contribution. `contributionCounts` makes the mix explicit
131
- * in the result.
132
- * Callers must cross-fit the Q-function or train it on independent rows;
133
- * fitting and evaluating Q on the same outcomes leaks the answer.
134
- */
135
- function doublyRobust(trajectories, opts = {}) {
136
- const cap = opts.weightCap ?? Infinity;
137
- const clip = opts.rewardClip ?? {
138
- low: 0,
139
- high: 1
140
- };
141
- if (trajectories.length === 0) return {
142
- ...zeroEstimate(),
143
- contributionCounts: {
144
- dr: 0,
145
- ipsFallback: 0
146
- }
147
- };
148
- const contributions = [];
149
- const contributionCounts = {
150
- dr: 0,
151
- ipsFallback: 0
152
- };
153
- let maxW = 0;
154
- let sumW = 0;
155
- let sumW2 = 0;
156
- for (const t of trajectories) {
157
- if (t.behaviorProb <= 0) throw new ValidationError(`doublyRobust: behaviorProb must be > 0 (runId=${t.runId})`);
158
- const w = Math.min(cap, t.targetProb / t.behaviorProb);
159
- const r = clamp(t.reward, clip.low, clip.high);
160
- const rawQHatChosen = t.qHatChosen;
161
- const rawVHatTarget = t.vHatTarget;
162
- const hasQHatChosen = rawQHatChosen !== null && rawQHatChosen !== void 0;
163
- const hasVHatTarget = rawVHatTarget !== null && rawVHatTarget !== void 0;
164
- if (hasQHatChosen !== hasVHatTarget) throw new ValidationError(`doublyRobust: qHatChosen and vHatTarget must be supplied together (runId=${t.runId})`);
165
- if (hasQHatChosen && hasVHatTarget) {
166
- if (!Number.isFinite(rawQHatChosen) || !Number.isFinite(rawVHatTarget)) throw new ValidationError(`doublyRobust: qHatChosen and vHatTarget must be finite (runId=${t.runId})`);
167
- const qHatChosen = clamp(rawQHatChosen, clip.low, clip.high);
168
- const vHatTarget = clamp(rawVHatTarget, clip.low, clip.high);
169
- contributions.push(vHatTarget + w * (r - qHatChosen));
170
- contributionCounts.dr += 1;
171
- } else {
172
- contributions.push(w * r);
173
- contributionCounts.ipsFallback += 1;
174
- }
175
- if (w > maxW) maxW = w;
176
- sumW += w;
177
- sumW2 += w * w;
178
- }
179
- const n = contributions.length;
180
- const value = contributions.reduce((s, x) => s + x, 0) / n;
181
- const variance = contributions.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1);
182
- const effN = sumW === 0 ? 0 : sumW * sumW / sumW2;
183
- return {
184
- value,
185
- standardError: Math.sqrt(variance / n),
186
- effectiveSampleSize: effN,
187
- n,
188
- maxImportanceWeight: maxW,
189
- contributionCounts
190
- };
191
- }
192
- /**
193
- * Convenience: run all three estimators and return them side-by-side.
194
- * The recommended diagnostic — agreement across estimators is a much
195
- * stronger signal than any single one.
196
- */
197
- function offPolicyEstimateAll(trajectories, opts = {}) {
198
- return {
199
- ips: inverseProbabilityWeighting(trajectories, opts),
200
- snips: selfNormalizedImportanceWeighting(trajectories, opts),
201
- dr: doublyRobust(trajectories, opts)
202
- };
203
- }
204
- function zeroEstimate() {
205
- return {
206
- value: 0,
207
- standardError: 0,
208
- effectiveSampleSize: 0,
209
- n: 0,
210
- maxImportanceWeight: 0
211
- };
212
- }
213
- function clamp(x, lo, hi) {
214
- if (!Number.isFinite(x)) return lo;
215
- return Math.max(lo, Math.min(hi, x));
216
- }
217
- //#endregion
218
- export { selfNormalizedImportanceWeighting as i, inverseProbabilityWeighting as n, offPolicyEstimateAll as r, doublyRobust as t };
219
-
220
- //# sourceMappingURL=off-policy-DvgzvtIx.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"off-policy-DvgzvtIx.js","names":[],"sources":["../src/rl/off-policy.ts"],"sourcesContent":["/**\n * Off-policy evaluation primitives.\n *\n * Standard inverse-probability-weighted (IPS), self-normalized\n * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the\n * value of a *target* policy given trajectories collected under a\n * *behavior* policy. This is the canonical RL eval task: \"we have last\n * week's runs, we changed the policy — how would the new one do without\n * re-running?\"\n *\n * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan\n * & Joachims 2015 for SNIPS) but the *application* to LLM-agent\n * evaluation needs care:\n *\n * - The \"policy\" is the (prompt, tool config, model snapshot) triple.\n * Two policies have the same probability over an action *iff* their\n * LLM call would emit the same token with the same probability —\n * which is generally unknowable without the model log-probs.\n * - For LLM agents, propensity scores must be supplied by the caller\n * (logged in the trace, recovered from token log-probs, or estimated\n * via a learned propensity model). We do NOT estimate propensity here.\n * - Doubly-robust requires two outputs from a Q-function: its prediction\n * for the logged action and its expectation under the target policy.\n * Consumers compute these with a tabular estimate, regression fit, or\n * learned reward model before constructing the trajectories.\n *\n * Bias / variance tradeoffs:\n * - IPS: unbiased; high variance for small overlap, infinite variance\n * when target has support outside behavior.\n * - SNIPS: lower variance, slight bias; usually preferred in practice.\n * - DR: doubly-robust — unbiased if either propensity OR Q-function is\n * correct. Lowest practical variance when Q is decent. Use this.\n *\n * Caveat the panel will land: on the LLM-agent setting, propensity scores\n * recovered from token log-probs are noisy, the action space is enormous,\n * and overlap is often poor. These estimators are useful but not magic;\n * complement with `replayCampaign` (exact replay where the request hashes\n * match) for high-confidence answers and OPE for the gap.\n */\n\nimport { ValidationError } from '../errors'\n\nexport interface OffPolicyTrajectory {\n /** Stable id, for traceability through the dataset. */\n runId: string\n /** Reward observed under the behavior policy (the realized outcome). */\n reward: number\n /**\n * Behavior-policy probability of the action that was taken. For LLM\n * agents this is typically `exp(sum(token_log_probs))` over the chosen\n * trajectory. Must be in (0, 1].\n */\n behaviorProb: number\n /**\n * Target-policy probability of the same action. For replay-style\n * counterfactual evaluation this is what the *new* policy would have\n * assigned to the *old* trajectory. Must be in [0, 1].\n */\n targetProb: number\n /**\n * Model-based reward prediction for the action selected by the behavior\n * policy: `Q_hat(context, loggedAction)`. Supply this together with\n * `vHatTarget` for contextual-bandit doubly-robust estimation.\n */\n qHatChosen?: number | null\n /**\n * Expected model-based reward under the target policy:\n * `sum_action targetPolicy(action | context) * Q_hat(context, action)`.\n * Supply this together with `qHatChosen`. For an honest evaluation, both\n * values must come from a model cross-fitted or trained outside this row.\n */\n vHatTarget?: number | null\n}\n\nexport interface OffPolicyContributionCounts {\n /** Contributions using the contextual-bandit doubly-robust formula. */\n dr: number\n /** Contributions using exact IPS because no reward-model estimate was supplied. */\n ipsFallback: number\n}\n\nexport interface OffPolicyEstimate {\n /** Estimated value of the target policy. */\n value: number\n /** Standard error of the estimate. */\n standardError: number\n /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */\n effectiveSampleSize: number\n /** Number of trajectories used. */\n n: number\n /**\n * Diagnostic: maximum importance weight observed. Large values (>>10x\n * mean) are a red flag — variance is dominated by a few outliers.\n */\n maxImportanceWeight: number\n /** Populated by `doublyRobust` to expose which formula each row used. */\n contributionCounts?: OffPolicyContributionCounts\n}\n\nexport interface OffPolicyOptions {\n /**\n * Cap importance weights at this value (Ionides 2008 truncated IS) to\n * trade unbiasedness for variance reduction. Default `Infinity` (no cap).\n * Set e.g. `10` for stable estimates when the policies are close.\n */\n weightCap?: number\n /** Reward clipping range. Default `[0, 1]`. */\n rewardClip?: { low: number; high: number }\n}\n\n/**\n * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator\n * of E[reward under target policy]. Variance scales with the spread of\n * target/behavior ratios.\n */\nexport function inverseProbabilityWeighting(\n trajectories: OffPolicyTrajectory[],\n opts: OffPolicyOptions = {},\n): OffPolicyEstimate {\n const cap = opts.weightCap ?? Infinity\n const clip = opts.rewardClip ?? { low: 0, high: 1 }\n\n if (trajectories.length === 0) {\n return zeroEstimate()\n }\n\n const weights: number[] = []\n const weightedRewards: number[] = []\n let maxW = 0\n for (const t of trajectories) {\n if (t.behaviorProb <= 0) {\n throw new ValidationError(\n `inverseProbabilityWeighting: behaviorProb must be > 0 (runId=${t.runId})`,\n )\n }\n const w = Math.min(cap, t.targetProb / t.behaviorProb)\n const r = clamp(t.reward, clip.low, clip.high)\n weights.push(w)\n weightedRewards.push(w * r)\n if (w > maxW) maxW = w\n }\n const n = weights.length\n const value = weightedRewards.reduce((s, x) => s + x, 0) / n\n const variance = weightedRewards.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1)\n const sumW = weights.reduce((s, w) => s + w, 0)\n const sumW2 = weights.reduce((s, w) => s + w * w, 0)\n const effN = sumW === 0 ? 0 : (sumW * sumW) / sumW2\n\n return {\n value,\n standardError: Math.sqrt(variance / n),\n effectiveSampleSize: effN,\n n,\n maxImportanceWeight: maxW,\n }\n}\n\n/**\n * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at\n * the cost of small bias (vanishing as N grows). The right default for\n * LLM-agent evaluation where overlap is often poor.\n */\nexport function selfNormalizedImportanceWeighting(\n trajectories: OffPolicyTrajectory[],\n opts: OffPolicyOptions = {},\n): OffPolicyEstimate {\n const cap = opts.weightCap ?? Infinity\n const clip = opts.rewardClip ?? { low: 0, high: 1 }\n if (trajectories.length === 0) return zeroEstimate()\n\n const weights: number[] = []\n const rewards: number[] = []\n let maxW = 0\n for (const t of trajectories) {\n if (t.behaviorProb <= 0) {\n throw new ValidationError(\n `selfNormalizedImportanceWeighting: behaviorProb must be > 0 (runId=${t.runId})`,\n )\n }\n const w = Math.min(cap, t.targetProb / t.behaviorProb)\n weights.push(w)\n rewards.push(clamp(t.reward, clip.low, clip.high))\n if (w > maxW) maxW = w\n }\n const sumW = weights.reduce((s, w) => s + w, 0)\n const sumWR = weights.reduce((s, w, i) => s + w * rewards[i]!, 0)\n const value = sumW === 0 ? 0 : sumWR / sumW\n const sumW2 = weights.reduce((s, w) => s + w * w, 0)\n const effN = sumW === 0 ? 0 : (sumW * sumW) / sumW2\n // Influence-function-based SE for SNIPS (Owen 2013, Ch. 9).\n const phi = weights.map((w, i) => w * (rewards[i]! - value))\n const variance = phi.reduce((s, x) => s + x * x, 0) / Math.max(1, sumW * sumW)\n return {\n value,\n standardError: Math.sqrt(variance),\n effectiveSampleSize: effN,\n n: trajectories.length,\n maxImportanceWeight: maxW,\n }\n}\n\n/**\n * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).\n *\n * V_DR = (1/N) * sum_i [ v_hat_target_i\n * + (target_prob_i / behavior_prob_i) * (r_i - q_hat_chosen_i) ]\n *\n * Unbiased if EITHER:\n * - the importance ratios are correct (IPS-style validity), OR\n * - the Q-hat function is correct (model-based validity).\n *\n * In practice both are imperfect, but the residual bias is the *product*\n * of both errors — much smaller than either alone. This is why DR is the\n * default in production OPE pipelines.\n *\n * `qHatChosen` and `vHatTarget` must be supplied together. Rows with neither\n * use the exact IPS contribution. `contributionCounts` makes the mix explicit\n * in the result.\n * Callers must cross-fit the Q-function or train it on independent rows;\n * fitting and evaluating Q on the same outcomes leaks the answer.\n */\nexport function doublyRobust(\n trajectories: OffPolicyTrajectory[],\n opts: OffPolicyOptions = {},\n): OffPolicyEstimate {\n const cap = opts.weightCap ?? Infinity\n const clip = opts.rewardClip ?? { low: 0, high: 1 }\n if (trajectories.length === 0) {\n return {\n ...zeroEstimate(),\n contributionCounts: { dr: 0, ipsFallback: 0 },\n }\n }\n\n const contributions: number[] = []\n const contributionCounts: OffPolicyContributionCounts = {\n dr: 0,\n ipsFallback: 0,\n }\n let maxW = 0\n let sumW = 0\n let sumW2 = 0\n for (const t of trajectories) {\n if (t.behaviorProb <= 0) {\n throw new ValidationError(`doublyRobust: behaviorProb must be > 0 (runId=${t.runId})`)\n }\n const w = Math.min(cap, t.targetProb / t.behaviorProb)\n const r = clamp(t.reward, clip.low, clip.high)\n const rawQHatChosen = t.qHatChosen\n const rawVHatTarget = t.vHatTarget\n const hasQHatChosen = rawQHatChosen !== null && rawQHatChosen !== undefined\n const hasVHatTarget = rawVHatTarget !== null && rawVHatTarget !== undefined\n if (hasQHatChosen !== hasVHatTarget) {\n throw new ValidationError(\n `doublyRobust: qHatChosen and vHatTarget must be supplied together (runId=${t.runId})`,\n )\n }\n\n if (hasQHatChosen && hasVHatTarget) {\n if (!Number.isFinite(rawQHatChosen) || !Number.isFinite(rawVHatTarget)) {\n throw new ValidationError(\n `doublyRobust: qHatChosen and vHatTarget must be finite (runId=${t.runId})`,\n )\n }\n const qHatChosen = clamp(rawQHatChosen, clip.low, clip.high)\n const vHatTarget = clamp(rawVHatTarget, clip.low, clip.high)\n contributions.push(vHatTarget + w * (r - qHatChosen))\n contributionCounts.dr += 1\n } else {\n contributions.push(w * r)\n contributionCounts.ipsFallback += 1\n }\n if (w > maxW) maxW = w\n sumW += w\n sumW2 += w * w\n }\n const n = contributions.length\n const value = contributions.reduce((s, x) => s + x, 0) / n\n const variance = contributions.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1)\n const effN = sumW === 0 ? 0 : (sumW * sumW) / sumW2\n return {\n value,\n standardError: Math.sqrt(variance / n),\n effectiveSampleSize: effN,\n n,\n maxImportanceWeight: maxW,\n contributionCounts,\n }\n}\n\n/**\n * Convenience: run all three estimators and return them side-by-side.\n * The recommended diagnostic — agreement across estimators is a much\n * stronger signal than any single one.\n */\nexport function offPolicyEstimateAll(\n trajectories: OffPolicyTrajectory[],\n opts: OffPolicyOptions = {},\n): { ips: OffPolicyEstimate; snips: OffPolicyEstimate; dr: OffPolicyEstimate } {\n return {\n ips: inverseProbabilityWeighting(trajectories, opts),\n snips: selfNormalizedImportanceWeighting(trajectories, opts),\n dr: doublyRobust(trajectories, opts),\n }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction zeroEstimate(): OffPolicyEstimate {\n return { value: 0, standardError: 0, effectiveSampleSize: 0, n: 0, maxImportanceWeight: 0 }\n}\n\nfunction clamp(x: number, lo: number, hi: number): number {\n if (!Number.isFinite(x)) return lo\n return Math.max(lo, Math.min(hi, x))\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmHA,SAAgB,4BACd,cACA,OAAyB,CAAC,GACP;CACnB,MAAM,MAAM,KAAK,aAAa;CAC9B,MAAM,OAAO,KAAK,cAAc;EAAE,KAAK;EAAG,MAAM;CAAE;CAElD,IAAI,aAAa,WAAW,GAC1B,OAAO,aAAa;CAGtB,MAAM,UAAoB,CAAC;CAC3B,MAAM,kBAA4B,CAAC;CACnC,IAAI,OAAO;CACX,KAAK,MAAM,KAAK,cAAc;EAC5B,IAAI,EAAE,gBAAgB,GACpB,MAAM,IAAI,gBACR,gEAAgE,EAAE,MAAM,EAC1E;EAEF,MAAM,IAAI,KAAK,IAAI,KAAK,EAAE,aAAa,EAAE,YAAY;EACrD,MAAM,IAAI,MAAM,EAAE,QAAQ,KAAK,KAAK,KAAK,IAAI;EAC7C,QAAQ,KAAK,CAAC;EACd,gBAAgB,KAAK,IAAI,CAAC;EAC1B,IAAI,IAAI,MAAM,OAAO;CACvB;CACA,MAAM,IAAI,QAAQ;CAClB,MAAM,QAAQ,gBAAgB,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CAC3D,MAAM,WAAW,gBAAgB,QAAQ,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,CAAC;CAC9F,MAAM,OAAO,QAAQ,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC;CAC9C,MAAM,QAAQ,QAAQ,QAAQ,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC;CACnD,MAAM,OAAO,SAAS,IAAI,IAAK,OAAO,OAAQ;CAE9C,OAAO;EACL;EACA,eAAe,KAAK,KAAK,WAAW,CAAC;EACrC,qBAAqB;EACrB;EACA,qBAAqB;CACvB;AACF;;;;;;AAOA,SAAgB,kCACd,cACA,OAAyB,CAAC,GACP;CACnB,MAAM,MAAM,KAAK,aAAa;CAC9B,MAAM,OAAO,KAAK,cAAc;EAAE,KAAK;EAAG,MAAM;CAAE;CAClD,IAAI,aAAa,WAAW,GAAG,OAAO,aAAa;CAEnD,MAAM,UAAoB,CAAC;CAC3B,MAAM,UAAoB,CAAC;CAC3B,IAAI,OAAO;CACX,KAAK,MAAM,KAAK,cAAc;EAC5B,IAAI,EAAE,gBAAgB,GACpB,MAAM,IAAI,gBACR,sEAAsE,EAAE,MAAM,EAChF;EAEF,MAAM,IAAI,KAAK,IAAI,KAAK,EAAE,aAAa,EAAE,YAAY;EACrD,QAAQ,KAAK,CAAC;EACd,QAAQ,KAAK,MAAM,EAAE,QAAQ,KAAK,KAAK,KAAK,IAAI,CAAC;EACjD,IAAI,IAAI,MAAM,OAAO;CACvB;CACA,MAAM,OAAO,QAAQ,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC;CAC9C,MAAM,QAAQ,QAAQ,QAAQ,GAAG,GAAG,MAAM,IAAI,IAAI,QAAQ,IAAK,CAAC;CAChE,MAAM,QAAQ,SAAS,IAAI,IAAI,QAAQ;CACvC,MAAM,QAAQ,QAAQ,QAAQ,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC;CACnD,MAAM,OAAO,SAAS,IAAI,IAAK,OAAO,OAAQ;CAG9C,MAAM,WADM,QAAQ,KAAK,GAAG,MAAM,KAAK,QAAQ,KAAM,MAClC,CAAC,CAAC,QAAQ,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,OAAO,IAAI;CAC7E,OAAO;EACL;EACA,eAAe,KAAK,KAAK,QAAQ;EACjC,qBAAqB;EACrB,GAAG,aAAa;EAChB,qBAAqB;CACvB;AACF;;;;;;;;;;;;;;;;;;;;;AAsBA,SAAgB,aACd,cACA,OAAyB,CAAC,GACP;CACnB,MAAM,MAAM,KAAK,aAAa;CAC9B,MAAM,OAAO,KAAK,cAAc;EAAE,KAAK;EAAG,MAAM;CAAE;CAClD,IAAI,aAAa,WAAW,GAC1B,OAAO;EACL,GAAG,aAAa;EAChB,oBAAoB;GAAE,IAAI;GAAG,aAAa;EAAE;CAC9C;CAGF,MAAM,gBAA0B,CAAC;CACjC,MAAM,qBAAkD;EACtD,IAAI;EACJ,aAAa;CACf;CACA,IAAI,OAAO;CACX,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,KAAK,MAAM,KAAK,cAAc;EAC5B,IAAI,EAAE,gBAAgB,GACpB,MAAM,IAAI,gBAAgB,iDAAiD,EAAE,MAAM,EAAE;EAEvF,MAAM,IAAI,KAAK,IAAI,KAAK,EAAE,aAAa,EAAE,YAAY;EACrD,MAAM,IAAI,MAAM,EAAE,QAAQ,KAAK,KAAK,KAAK,IAAI;EAC7C,MAAM,gBAAgB,EAAE;EACxB,MAAM,gBAAgB,EAAE;EACxB,MAAM,gBAAgB,kBAAkB,QAAQ,kBAAkB,KAAA;EAClE,MAAM,gBAAgB,kBAAkB,QAAQ,kBAAkB,KAAA;EAClE,IAAI,kBAAkB,eACpB,MAAM,IAAI,gBACR,4EAA4E,EAAE,MAAM,EACtF;EAGF,IAAI,iBAAiB,eAAe;GAClC,IAAI,CAAC,OAAO,SAAS,aAAa,KAAK,CAAC,OAAO,SAAS,aAAa,GACnE,MAAM,IAAI,gBACR,iEAAiE,EAAE,MAAM,EAC3E;GAEF,MAAM,aAAa,MAAM,eAAe,KAAK,KAAK,KAAK,IAAI;GAC3D,MAAM,aAAa,MAAM,eAAe,KAAK,KAAK,KAAK,IAAI;GAC3D,cAAc,KAAK,aAAa,KAAK,IAAI,WAAW;GACpD,mBAAmB,MAAM;EAC3B,OAAO;GACL,cAAc,KAAK,IAAI,CAAC;GACxB,mBAAmB,eAAe;EACpC;EACA,IAAI,IAAI,MAAM,OAAO;EACrB,QAAQ;EACR,SAAS,IAAI;CACf;CACA,MAAM,IAAI,cAAc;CACxB,MAAM,QAAQ,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CACzD,MAAM,WAAW,cAAc,QAAQ,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,CAAC;CAC5F,MAAM,OAAO,SAAS,IAAI,IAAK,OAAO,OAAQ;CAC9C,OAAO;EACL;EACA,eAAe,KAAK,KAAK,WAAW,CAAC;EACrC,qBAAqB;EACrB;EACA,qBAAqB;EACrB;CACF;AACF;;;;;;AAOA,SAAgB,qBACd,cACA,OAAyB,CAAC,GACmD;CAC7E,OAAO;EACL,KAAK,4BAA4B,cAAc,IAAI;EACnD,OAAO,kCAAkC,cAAc,IAAI;EAC3D,IAAI,aAAa,cAAc,IAAI;CACrC;AACF;AAIA,SAAS,eAAkC;CACzC,OAAO;EAAE,OAAO;EAAG,eAAe;EAAG,qBAAqB;EAAG,GAAG;EAAG,qBAAqB;CAAE;AAC5F;AAEA,SAAS,MAAM,GAAW,IAAY,IAAoB;CACxD,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG,OAAO;CAChC,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC;AACrC"}
@@ -1,153 +0,0 @@
1
- //#region src/rl/off-policy.d.ts
2
- /**
3
- * Off-policy evaluation primitives.
4
- *
5
- * Standard inverse-probability-weighted (IPS), self-normalized
6
- * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
7
- * value of a *target* policy given trajectories collected under a
8
- * *behavior* policy. This is the canonical RL eval task: "we have last
9
- * week's runs, we changed the policy — how would the new one do without
10
- * re-running?"
11
- *
12
- * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
13
- * & Joachims 2015 for SNIPS) but the *application* to LLM-agent
14
- * evaluation needs care:
15
- *
16
- * - The "policy" is the (prompt, tool config, model snapshot) triple.
17
- * Two policies have the same probability over an action *iff* their
18
- * LLM call would emit the same token with the same probability —
19
- * which is generally unknowable without the model log-probs.
20
- * - For LLM agents, propensity scores must be supplied by the caller
21
- * (logged in the trace, recovered from token log-probs, or estimated
22
- * via a learned propensity model). We do NOT estimate propensity here.
23
- * - Doubly-robust requires two outputs from a Q-function: its prediction
24
- * for the logged action and its expectation under the target policy.
25
- * Consumers compute these with a tabular estimate, regression fit, or
26
- * learned reward model before constructing the trajectories.
27
- *
28
- * Bias / variance tradeoffs:
29
- * - IPS: unbiased; high variance for small overlap, infinite variance
30
- * when target has support outside behavior.
31
- * - SNIPS: lower variance, slight bias; usually preferred in practice.
32
- * - DR: doubly-robust — unbiased if either propensity OR Q-function is
33
- * correct. Lowest practical variance when Q is decent. Use this.
34
- *
35
- * Caveat the panel will land: on the LLM-agent setting, propensity scores
36
- * recovered from token log-probs are noisy, the action space is enormous,
37
- * and overlap is often poor. These estimators are useful but not magic;
38
- * complement with `replayCampaign` (exact replay where the request hashes
39
- * match) for high-confidence answers and OPE for the gap.
40
- */
41
- interface OffPolicyTrajectory {
42
- /** Stable id, for traceability through the dataset. */
43
- runId: string;
44
- /** Reward observed under the behavior policy (the realized outcome). */
45
- reward: number;
46
- /**
47
- * Behavior-policy probability of the action that was taken. For LLM
48
- * agents this is typically `exp(sum(token_log_probs))` over the chosen
49
- * trajectory. Must be in (0, 1].
50
- */
51
- behaviorProb: number;
52
- /**
53
- * Target-policy probability of the same action. For replay-style
54
- * counterfactual evaluation this is what the *new* policy would have
55
- * assigned to the *old* trajectory. Must be in [0, 1].
56
- */
57
- targetProb: number;
58
- /**
59
- * Model-based reward prediction for the action selected by the behavior
60
- * policy: `Q_hat(context, loggedAction)`. Supply this together with
61
- * `vHatTarget` for contextual-bandit doubly-robust estimation.
62
- */
63
- qHatChosen?: number | null;
64
- /**
65
- * Expected model-based reward under the target policy:
66
- * `sum_action targetPolicy(action | context) * Q_hat(context, action)`.
67
- * Supply this together with `qHatChosen`. For an honest evaluation, both
68
- * values must come from a model cross-fitted or trained outside this row.
69
- */
70
- vHatTarget?: number | null;
71
- }
72
- interface OffPolicyContributionCounts {
73
- /** Contributions using the contextual-bandit doubly-robust formula. */
74
- dr: number;
75
- /** Contributions using exact IPS because no reward-model estimate was supplied. */
76
- ipsFallback: number;
77
- }
78
- interface OffPolicyEstimate {
79
- /** Estimated value of the target policy. */
80
- value: number;
81
- /** Standard error of the estimate. */
82
- standardError: number;
83
- /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */
84
- effectiveSampleSize: number;
85
- /** Number of trajectories used. */
86
- n: number;
87
- /**
88
- * Diagnostic: maximum importance weight observed. Large values (>>10x
89
- * mean) are a red flag — variance is dominated by a few outliers.
90
- */
91
- maxImportanceWeight: number;
92
- /** Populated by `doublyRobust` to expose which formula each row used. */
93
- contributionCounts?: OffPolicyContributionCounts;
94
- }
95
- interface OffPolicyOptions {
96
- /**
97
- * Cap importance weights at this value (Ionides 2008 truncated IS) to
98
- * trade unbiasedness for variance reduction. Default `Infinity` (no cap).
99
- * Set e.g. `10` for stable estimates when the policies are close.
100
- */
101
- weightCap?: number;
102
- /** Reward clipping range. Default `[0, 1]`. */
103
- rewardClip?: {
104
- low: number;
105
- high: number;
106
- };
107
- }
108
- /**
109
- * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
110
- * of E[reward under target policy]. Variance scales with the spread of
111
- * target/behavior ratios.
112
- */
113
- declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
114
- /**
115
- * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
116
- * the cost of small bias (vanishing as N grows). The right default for
117
- * LLM-agent evaluation where overlap is often poor.
118
- */
119
- declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
120
- /**
121
- * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
122
- *
123
- * V_DR = (1/N) * sum_i [ v_hat_target_i
124
- * + (target_prob_i / behavior_prob_i) * (r_i - q_hat_chosen_i) ]
125
- *
126
- * Unbiased if EITHER:
127
- * - the importance ratios are correct (IPS-style validity), OR
128
- * - the Q-hat function is correct (model-based validity).
129
- *
130
- * In practice both are imperfect, but the residual bias is the *product*
131
- * of both errors — much smaller than either alone. This is why DR is the
132
- * default in production OPE pipelines.
133
- *
134
- * `qHatChosen` and `vHatTarget` must be supplied together. Rows with neither
135
- * use the exact IPS contribution. `contributionCounts` makes the mix explicit
136
- * in the result.
137
- * Callers must cross-fit the Q-function or train it on independent rows;
138
- * fitting and evaluating Q on the same outcomes leaks the answer.
139
- */
140
- declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
141
- /**
142
- * Convenience: run all three estimators and return them side-by-side.
143
- * The recommended diagnostic — agreement across estimators is a much
144
- * stronger signal than any single one.
145
- */
146
- declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
147
- ips: OffPolicyEstimate;
148
- snips: OffPolicyEstimate;
149
- dr: OffPolicyEstimate;
150
- };
151
- //#endregion
152
- export { doublyRobust as a, selfNormalizedImportanceWeighting as c, OffPolicyTrajectory as i, OffPolicyEstimate as n, inverseProbabilityWeighting as o, OffPolicyOptions as r, offPolicyEstimateAll as s, OffPolicyContributionCounts as t };
153
- //# sourceMappingURL=off-policy-mskQw8Mb.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"off-policy-mskQw8Mb.d.ts","names":[],"sources":["../src/rl/off-policy.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UA0CiB;;EAEf;;EAEA;;;;;;EAMA;;;;;;EAMA;;;;;;EAMA;;;;;;;EAOA;;UAGe;;EAEf;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;;;;EAKA;;EAEA,qBAAqB;;UAGN;;;;;;EAMf;;EAEA;IAAe;IAAa;;;;;;;;iBAQd,4BACd,cAAc,uBACd,OAAM,mBACL;;;;;;iBA4Ca,kCACd,cAAc,uBACd,OAAM,mBACL;;;;;;;;;;;;;;;;;;;;;iBAwDa,aACd,cAAc,uBACd,OAAM,mBACL;;;;;;iBAuEa,qBACd,cAAc,uBACd,OAAM;EACH,KAAK;EAAmB,OAAO;EAAmB,IAAI"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"run-record-BJnYdTxO.d.ts","names":[],"sources":["../src/run-record.ts"],"mappings":";;;;;;KAqCY;;;;;;;KAQA;UAEK;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;KAMU;EACN;EAAkB;;EAClB;EAAmB;;EACnB;EAAoB;;UAET;EACf;EACA;;;EAGA;;;EAGA;;;;;;;;;;;;;;;;;;;;;UAsBe;;EAEf,UAAU,eAAe;;EAEzB,YAAY;;;EAGZ;;;;EAIA;;;EAGA;;UAGe;;;EAGf;;;EAGA;;;;EAIA,KAAK;;;;;;EAML,cAAc;;;;;;;EAOd;IAAa;IAAe;IAAgB;;;;;;;;;;;;;;;;;;;UAmB7B;;EAEf;;;EAGA;;;EAGA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB,YAAY;;EAEZ,iBAAiB;;;EAGjB;;EAEA,gBAAgB;;EAEhB,SAAS;;;;;EAKT,eAAe;;;EAGf;;EAEA,UAAU;;;;;EAKV;;;;;;;;EAQA,eAAe;;;;;;;;;KAUL;EACN;EAA0B;;EAC1B;EAAyB;;EAEzB,cAAc,QAAQ;EACtB;;;;;;;;;;iBAWU,aAAa,QAAQ;cAmCxB,iCAAiC;WACnC;EACT,YAAY,iBAAiB;;;;;;;iBAWf,kBAAkB,iBAAiB;;iBAiOnC,YAAY,iBAAiB,SAAS;;iBAUtC,mBACd;EACG;EAAU,OAAO;;EAAgB;EAAW,OAAO;;;iBAUxC,mBAAmB,QAAQ,YAAY;;;;;;;;;;;iBA2FvC,iBAAiB"}
@@ -1,93 +0,0 @@
1
- //#region src/runtime-trajectory.ts
2
- const DEFAULT_SPLIT_TAG = "search";
3
- function projectRuntimeTrajectoryEvidence(options) {
4
- const diagnostics = [];
5
- const runsById = /* @__PURE__ */ new Map();
6
- const events = [];
7
- let recordWithRuntimeEventsCount = 0;
8
- let defaultedSplitCount = 0;
9
- for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {
10
- const record = options.records[recordIndex];
11
- const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf);
12
- const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG;
13
- if (record.splitTag === void 0) defaultedSplitCount += 1;
14
- const rawEvents = record.runtimeEvents;
15
- if (!Array.isArray(rawEvents)) {
16
- diagnostics.push(`${key}: runtimeEvents is not an array; no runtime run join can be extracted`);
17
- continue;
18
- }
19
- if (rawEvents.length === 0) {
20
- diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`);
21
- continue;
22
- }
23
- recordWithRuntimeEventsCount += 1;
24
- for (let index = 0; index < rawEvents.length; index += 1) {
25
- const event = parseRuntimeTrajectoryHookEvent(rawEvents[index]);
26
- if (!event) {
27
- diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`);
28
- continue;
29
- }
30
- events.push(event);
31
- const scenarioId = event.scenarioId ?? stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ?? stringOrUndefined(record.scenarioId);
32
- const prior = runsById.get(event.runId);
33
- if (!prior) {
34
- runsById.set(event.runId, {
35
- runId: event.runId,
36
- scenarioId,
37
- splitTag
38
- });
39
- continue;
40
- }
41
- if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`);
42
- }
43
- }
44
- const runs = [...runsById.values()];
45
- return {
46
- runs,
47
- events,
48
- summary: {
49
- recordCount: options.records.length,
50
- recordWithRuntimeEventsCount,
51
- runtimeRunCount: runs.length,
52
- lifecycleEventCount: events.length,
53
- defaultedSplitCount
54
- },
55
- diagnostics
56
- };
57
- }
58
- function parseRuntimeTrajectoryHookEvent(input) {
59
- if (!isRecord(input)) return null;
60
- if (typeof input.id !== "string" || input.id.length === 0) return null;
61
- if (typeof input.runId !== "string" || input.runId.length === 0) return null;
62
- if (typeof input.target !== "string" || input.target.length === 0) return null;
63
- if (typeof input.phase !== "string" || input.phase.length === 0) return null;
64
- if (typeof input.timestamp !== "number" || !Number.isFinite(input.timestamp)) return null;
65
- return {
66
- id: input.id,
67
- runId: input.runId,
68
- scenarioId: stringOrUndefined(input.scenarioId),
69
- target: input.target,
70
- phase: input.phase,
71
- timestamp: input.timestamp,
72
- stepIndex: finiteNumberOrUndefined(input.stepIndex),
73
- parentId: stringOrUndefined(input.parentId),
74
- payload: input.payload,
75
- metadata: isRecord(input.metadata) ? { ...input.metadata } : void 0
76
- };
77
- }
78
- function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
79
- return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
80
- }
81
- function isRecord(value) {
82
- return typeof value === "object" && value !== null && !Array.isArray(value);
83
- }
84
- function stringOrUndefined(value) {
85
- return typeof value === "string" && value.length > 0 ? value : void 0;
86
- }
87
- function finiteNumberOrUndefined(value) {
88
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
89
- }
90
- //#endregion
91
- export { projectRuntimeTrajectoryEvidence as n, parseRuntimeTrajectoryHookEvent as t };
92
-
93
- //# sourceMappingURL=runtime-trajectory-1gyaTOoC.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"runtime-trajectory-1gyaTOoC.js","names":[],"sources":["../src/runtime-trajectory.ts"],"sourcesContent":["import type { RunSplitTag } from './run-record'\n\nexport interface RuntimeTrajectoryHookEvent {\n id: string\n runId: string\n scenarioId?: string\n target: string\n phase: string\n timestamp: number\n stepIndex?: number\n parentId?: string\n payload?: unknown\n metadata?: Record<string, unknown>\n}\n\nexport interface RuntimeTrajectoryRecord {\n id?: string\n scenarioId?: string\n splitTag?: RunSplitTag\n runtimeEvents?: unknown\n [key: string]: unknown\n}\n\nexport interface RuntimeTrajectoryRunRecord {\n runId: string\n scenarioId?: string\n splitTag: RunSplitTag\n}\n\nexport interface RuntimeTrajectoryEvidenceSummary {\n recordCount: number\n recordWithRuntimeEventsCount: number\n runtimeRunCount: number\n lifecycleEventCount: number\n defaultedSplitCount: number\n}\n\nexport interface RuntimeTrajectoryEvidenceProjection {\n runs: RuntimeTrajectoryRunRecord[]\n events: RuntimeTrajectoryHookEvent[]\n summary: RuntimeTrajectoryEvidenceSummary\n diagnostics: string[]\n}\n\nexport interface ProjectRuntimeTrajectoryEvidenceOptions<\n TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord,\n> {\n records: TRecord[]\n defaultSplitTag?: RunSplitTag\n recordIdOf?: (record: TRecord, index: number) => string | undefined\n scenarioIdOf?: (record: TRecord, index: number) => string | undefined\n}\n\nconst DEFAULT_SPLIT_TAG: RunSplitTag = 'search'\n\nexport function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(\n options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>,\n): RuntimeTrajectoryEvidenceProjection {\n const diagnostics: string[] = []\n const runsById = new Map<string, RuntimeTrajectoryRunRecord>()\n const events: RuntimeTrajectoryHookEvent[] = []\n let recordWithRuntimeEventsCount = 0\n let defaultedSplitCount = 0\n\n for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {\n const record = options.records[recordIndex]!\n const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf)\n const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG\n if (record.splitTag === undefined) defaultedSplitCount += 1\n\n const rawEvents = record.runtimeEvents\n if (!Array.isArray(rawEvents)) {\n diagnostics.push(\n `${key}: runtimeEvents is not an array; no runtime run join can be extracted`,\n )\n continue\n }\n if (rawEvents.length === 0) {\n diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`)\n continue\n }\n recordWithRuntimeEventsCount += 1\n\n for (let index = 0; index < rawEvents.length; index += 1) {\n const event = parseRuntimeTrajectoryHookEvent(rawEvents[index])\n if (!event) {\n diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`)\n continue\n }\n events.push(event)\n\n const scenarioId =\n event.scenarioId ??\n stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ??\n stringOrUndefined(record.scenarioId)\n const prior = runsById.get(event.runId)\n if (!prior) {\n runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag })\n continue\n }\n if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {\n diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`)\n }\n }\n }\n\n const runs = [...runsById.values()]\n return {\n runs,\n events,\n summary: {\n recordCount: options.records.length,\n recordWithRuntimeEventsCount,\n runtimeRunCount: runs.length,\n lifecycleEventCount: events.length,\n defaultedSplitCount,\n },\n diagnostics,\n }\n}\n\nexport function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null {\n if (!isRecord(input)) return null\n if (typeof input.id !== 'string' || input.id.length === 0) return null\n if (typeof input.runId !== 'string' || input.runId.length === 0) return null\n if (typeof input.target !== 'string' || input.target.length === 0) return null\n if (typeof input.phase !== 'string' || input.phase.length === 0) return null\n if (typeof input.timestamp !== 'number' || !Number.isFinite(input.timestamp)) return null\n\n return {\n id: input.id,\n runId: input.runId,\n scenarioId: stringOrUndefined(input.scenarioId),\n target: input.target,\n phase: input.phase,\n timestamp: input.timestamp,\n stepIndex: finiteNumberOrUndefined(input.stepIndex),\n parentId: stringOrUndefined(input.parentId),\n payload: input.payload,\n metadata: isRecord(input.metadata) ? { ...input.metadata } : undefined,\n }\n}\n\nfunction runtimeTrajectoryRecordKey<TRecord extends RuntimeTrajectoryRecord>(\n record: TRecord,\n index: number,\n recordIdOf?: (record: TRecord, index: number) => string | undefined,\n): string {\n return (\n stringOrUndefined(recordIdOf?.(record, index)) ??\n stringOrUndefined(record.id) ??\n `record[${index}]`\n )\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction stringOrUndefined(value: unknown): string | undefined {\n return typeof value === 'string' && value.length > 0 ? value : undefined\n}\n\nfunction finiteNumberOrUndefined(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n"],"mappings":";AAqDA,MAAM,oBAAiC;AAEvC,SAAgB,iCACd,SACqC;CACrC,MAAM,cAAwB,CAAC;CAC/B,MAAM,2BAAW,IAAI,IAAwC;CAC7D,MAAM,SAAuC,CAAC;CAC9C,IAAI,+BAA+B;CACnC,IAAI,sBAAsB;CAE1B,KAAK,IAAI,cAAc,GAAG,cAAc,QAAQ,QAAQ,QAAQ,eAAe,GAAG;EAChF,MAAM,SAAS,QAAQ,QAAQ;EAC/B,MAAM,MAAM,2BAA2B,QAAQ,aAAa,QAAQ,UAAU;EAC9E,MAAM,WAAW,OAAO,YAAY,QAAQ,mBAAmB;EAC/D,IAAI,OAAO,aAAa,KAAA,GAAW,uBAAuB;EAE1D,MAAM,YAAY,OAAO;EACzB,IAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;GAC7B,YAAY,KACV,GAAG,IAAI,sEACT;GACA;EACF;EACA,IAAI,UAAU,WAAW,GAAG;GAC1B,YAAY,KAAK,GAAG,IAAI,yDAAyD;GACjF;EACF;EACA,gCAAgC;EAEhC,KAAK,IAAI,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;GACxD,MAAM,QAAQ,gCAAgC,UAAU,MAAM;GAC9D,IAAI,CAAC,OAAO;IACV,YAAY,KAAK,GAAG,IAAI,kBAAkB,MAAM,4BAA4B;IAC5E;GACF;GACA,OAAO,KAAK,KAAK;GAEjB,MAAM,aACJ,MAAM,cACN,kBAAkB,QAAQ,eAAe,QAAQ,WAAW,CAAC,KAC7D,kBAAkB,OAAO,UAAU;GACrC,MAAM,QAAQ,SAAS,IAAI,MAAM,KAAK;GACtC,IAAI,CAAC,OAAO;IACV,SAAS,IAAI,MAAM,OAAO;KAAE,OAAO,MAAM;KAAO;KAAY;IAAS,CAAC;IACtE;GACF;GACA,IAAI,MAAM,eAAe,cAAc,MAAM,aAAa,UACxD,YAAY,KAAK,GAAG,IAAI,UAAU,MAAM,MAAM,yCAAyC;EAE3F;CACF;CAEA,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;CAClC,OAAO;EACL;EACA;EACA,SAAS;GACP,aAAa,QAAQ,QAAQ;GAC7B;GACA,iBAAiB,KAAK;GACtB,qBAAqB,OAAO;GAC5B;EACF;EACA;CACF;AACF;AAEA,SAAgB,gCAAgC,OAAmD;CACjG,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;CAC7B,IAAI,OAAO,MAAM,OAAO,YAAY,MAAM,GAAG,WAAW,GAAG,OAAO;CAClE,IAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,GAAG,OAAO;CACxE,IAAI,OAAO,MAAM,WAAW,YAAY,MAAM,OAAO,WAAW,GAAG,OAAO;CAC1E,IAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,GAAG,OAAO;CACxE,IAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,GAAG,OAAO;CAErF,OAAO;EACL,IAAI,MAAM;EACV,OAAO,MAAM;EACb,YAAY,kBAAkB,MAAM,UAAU;EAC9C,QAAQ,MAAM;EACd,OAAO,MAAM;EACb,WAAW,MAAM;EACjB,WAAW,wBAAwB,MAAM,SAAS;EAClD,UAAU,kBAAkB,MAAM,QAAQ;EAC1C,SAAS,MAAM;EACf,UAAU,SAAS,MAAM,QAAQ,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI,KAAA;CAC/D;AACF;AAEA,SAAS,2BACP,QACA,OACA,YACQ;CACR,OACE,kBAAkB,aAAa,QAAQ,KAAK,CAAC,KAC7C,kBAAkB,OAAO,EAAE,KAC3B,UAAU,MAAM;AAEpB;AAEA,SAAS,SAAS,OAAkD;CAClE,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,kBAAkB,OAAoC;CAC7D,OAAO,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ,KAAA;AACjE;AAEA,SAAS,wBAAwB,OAAoC;CACnE,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AACvE"}
@@ -1,50 +0,0 @@
1
- import { s as RunSplitTag } from "./run-record-BJnYdTxO.js";
2
- //#region src/runtime-trajectory.d.ts
3
- interface RuntimeTrajectoryHookEvent {
4
- id: string;
5
- runId: string;
6
- scenarioId?: string;
7
- target: string;
8
- phase: string;
9
- timestamp: number;
10
- stepIndex?: number;
11
- parentId?: string;
12
- payload?: unknown;
13
- metadata?: Record<string, unknown>;
14
- }
15
- interface RuntimeTrajectoryRecord {
16
- id?: string;
17
- scenarioId?: string;
18
- splitTag?: RunSplitTag;
19
- runtimeEvents?: unknown;
20
- [key: string]: unknown;
21
- }
22
- interface RuntimeTrajectoryRunRecord {
23
- runId: string;
24
- scenarioId?: string;
25
- splitTag: RunSplitTag;
26
- }
27
- interface RuntimeTrajectoryEvidenceSummary {
28
- recordCount: number;
29
- recordWithRuntimeEventsCount: number;
30
- runtimeRunCount: number;
31
- lifecycleEventCount: number;
32
- defaultedSplitCount: number;
33
- }
34
- interface RuntimeTrajectoryEvidenceProjection {
35
- runs: RuntimeTrajectoryRunRecord[];
36
- events: RuntimeTrajectoryHookEvent[];
37
- summary: RuntimeTrajectoryEvidenceSummary;
38
- diagnostics: string[];
39
- }
40
- interface ProjectRuntimeTrajectoryEvidenceOptions<TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord> {
41
- records: TRecord[];
42
- defaultSplitTag?: RunSplitTag;
43
- recordIdOf?: (record: TRecord, index: number) => string | undefined;
44
- scenarioIdOf?: (record: TRecord, index: number) => string | undefined;
45
- }
46
- declare function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>): RuntimeTrajectoryEvidenceProjection;
47
- declare function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null;
48
- //#endregion
49
- export { RuntimeTrajectoryRecord as a, projectRuntimeTrajectoryEvidence as c, RuntimeTrajectoryHookEvent as i, RuntimeTrajectoryEvidenceProjection as n, RuntimeTrajectoryRunRecord as o, RuntimeTrajectoryEvidenceSummary as r, parseRuntimeTrajectoryHookEvent as s, ProjectRuntimeTrajectoryEvidenceOptions as t };
50
- //# sourceMappingURL=runtime-trajectory-BXxG4lyi.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"runtime-trajectory-BXxG4lyi.d.ts","names":[],"sources":["../src/runtime-trajectory.ts"],"mappings":";;UAEiB;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;EACX;GACC;;UAGc;EACf;EACA;EACA,UAAU;;UAGK;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf,MAAM;EACN,QAAQ;EACR,SAAS;EACT;;UAGe,wCACf,gBAAgB,0BAA0B;EAE1C,SAAS;EACT,kBAAkB;EAClB,cAAc,QAAQ,SAAS;EAC/B,gBAAgB,QAAQ,SAAS;;iBAKnB,iCAAiC,gBAAgB,yBAC/D,SAAS,wCAAwC,WAChD;iBAgEa,gCAAgC,iBAAiB"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"task-failure-attributes-CQZlB3et.js","names":[],"sources":["../src/trace/error-classification.ts","../src/trace/execution-measurements.ts","../src/trace/task-failure-attributes.ts"],"sourcesContent":["import type { OtlpSpanRole } from './otlp-attributes'\n\nexport type TraceErrorRole = OtlpSpanRole\n\nexport interface TraceErrorSignal {\n id: string\n parentId?: string\n role: TraceErrorRole\n error: boolean\n processRoot: boolean\n}\n\nexport interface TraceErrorSummary {\n total: number\n execution: number\n process: number\n guardrail: number\n evaluation: number\n propagated: number\n unclassified: number\n}\n\n/**\n * Classify errored spans without counting a propagated parent status as a\n * second execution failure.\n */\nexport function summarizeTraceErrors(signals: readonly TraceErrorSignal[]): TraceErrorSummary {\n const byId = new Map<string, TraceErrorSignal>()\n for (const signal of signals) {\n if (byId.has(signal.id)) {\n throw new Error(`summarizeTraceErrors: duplicate span id '${signal.id}'`)\n }\n byId.set(signal.id, signal)\n }\n\n const propagated = new Set<string>()\n for (const signal of signals) {\n if (!signal.error) continue\n const visited = new Set<string>()\n let parentId = signal.parentId\n while (parentId && !visited.has(parentId)) {\n visited.add(parentId)\n const parent = byId.get(parentId)\n if (!parent) break\n if (parent.error) propagated.add(parent.id)\n parentId = parent.parentId\n }\n }\n\n const summary: TraceErrorSummary = {\n total: 0,\n execution: 0,\n process: 0,\n guardrail: 0,\n evaluation: 0,\n propagated: 0,\n unclassified: 0,\n }\n\n for (const signal of signals) {\n if (!signal.error) continue\n summary.total += 1\n if (signal.role === 'GUARDRAIL') {\n summary.guardrail += 1\n } else if (signal.role === 'EVALUATOR') {\n summary.evaluation += 1\n } else if (signal.processRoot) {\n summary.process += 1\n } else if (propagated.has(signal.id)) {\n summary.propagated += 1\n } else if (\n signal.role === 'AGENT' ||\n signal.role === 'CHAIN' ||\n signal.role === 'LLM' ||\n signal.role === 'TOOL'\n ) {\n summary.execution += 1\n } else {\n summary.unclassified += 1\n }\n }\n\n return summary\n}\n","import type { RunTokenUsage } from '../run-record'\nimport {\n LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,\n LLM_CACHED_TOKEN_ATTR_KEYS,\n LLM_COST_ATTR_KEYS,\n LLM_INPUT_TOKEN_ATTR_KEYS,\n LLM_OUTPUT_TOKEN_ATTR_KEYS,\n LLM_REASONING_TOKEN_ATTR_KEYS,\n RUN_COST_ATTR_KEYS,\n} from './otlp-attributes'\n\nexport interface ExecutionMeasurementSpan {\n id: string\n parentId?: string\n attributes: Record<string, unknown>\n modelCall: boolean\n aggregate: boolean\n}\n\nexport interface MeasurementCoverage {\n value?: number\n reportingCalls: number\n complete: boolean\n}\n\nexport interface ExecutionMeasurements {\n tokenUsage: RunTokenUsage\n modelCallCount: number\n callSpanIds: string[]\n cost: MeasurementCoverage\n aggregate?: {\n tokenUsage: RunTokenUsage\n costUsd?: number\n }\n}\n\nconst TOKEN_MEASUREMENT_KEY_GROUPS = [\n LLM_INPUT_TOKEN_ATTR_KEYS,\n LLM_OUTPUT_TOKEN_ATTR_KEYS,\n LLM_REASONING_TOKEN_ATTR_KEYS,\n LLM_CACHED_TOKEN_ATTR_KEYS,\n LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,\n] as const\n\nconst EXECUTION_MEASUREMENT_KEY_GROUPS = [\n ...TOKEN_MEASUREMENT_KEY_GROUPS,\n LLM_COST_ATTR_KEYS,\n] as const\n\ninterface RetainedCallSummary {\n callCount: number\n measurements: Array<{\n total: number\n reportingCalls: number\n }>\n}\n\n/**\n * Reconcile execution measurements across nested telemetry wrappers.\n * A measured parent is used only when a descendant call does not report the\n * same field, so aggregate wrappers neither duplicate complete child data nor\n * erase complementary parent fields.\n */\nexport function summarizeExecutionMeasurements(\n spans: ExecutionMeasurementSpan[],\n): ExecutionMeasurements {\n const byId = new Map<string, ExecutionMeasurementSpan>()\n for (const span of spans) {\n if (byId.has(span.id)) {\n throw new Error(`summarizeExecutionMeasurements: duplicate span id \"${span.id}\"`)\n }\n byId.set(span.id, span)\n }\n const tokenMeasurementKeys = TOKEN_MEASUREMENT_KEY_GROUPS.flat()\n const candidates = spans.filter(\n (span) =>\n span.modelCall ||\n (!span.aggregate && readNumber(span.attributes, tokenMeasurementKeys) !== undefined),\n )\n const candidateIds = new Set(candidates.map((span) => span.id))\n const candidateChildren = new Map<string, ExecutionMeasurementSpan[]>()\n for (const candidate of candidates) {\n const parentId = nearestCandidateParent(candidate, byId, candidateIds)\n if (!parentId) continue\n const children = candidateChildren.get(parentId) ?? []\n children.push(candidate)\n candidateChildren.set(parentId, children)\n }\n const aggregateIds = classifyAggregateSpans(candidates, candidateChildren)\n const untypedRunCostIds = new Set(\n spans\n .filter(\n (span) =>\n !span.modelCall &&\n !span.aggregate &&\n !candidateIds.has(span.id) &&\n readNumber(span.attributes, RUN_COST_ATTR_KEYS) !== undefined,\n )\n .map((span) => span.id),\n )\n const aggregateSourceIds = new Set(\n spans\n .filter(\n (span) => span.aggregate || aggregateIds.has(span.id) || untypedRunCostIds.has(span.id),\n )\n .map((span) => span.id),\n )\n\n const calls = candidates.filter((span) => !aggregateIds.has(span.id))\n const input = reconcileMeasurement(calls, byId, aggregateSourceIds, LLM_INPUT_TOKEN_ATTR_KEYS)\n const reasoning = reconcileMeasurement(\n calls,\n byId,\n aggregateSourceIds,\n LLM_REASONING_TOKEN_ATTR_KEYS,\n )\n const output = reconcileMeasurement(\n calls,\n byId,\n aggregateSourceIds,\n LLM_OUTPUT_TOKEN_ATTR_KEYS,\n LLM_REASONING_TOKEN_ATTR_KEYS,\n )\n const cached = reconcileMeasurement(calls, byId, aggregateSourceIds, LLM_CACHED_TOKEN_ATTR_KEYS)\n const cacheWrite = reconcileMeasurement(\n calls,\n byId,\n aggregateSourceIds,\n LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,\n )\n const aggregate = summarizeAggregateMeasurements(\n spans,\n byId,\n new Set([...aggregateIds, ...untypedRunCostIds]),\n )\n\n return {\n tokenUsage: {\n input: input.value ?? 0,\n output: Math.max(output.value ?? 0, reasoning.value ?? 0),\n ...(reasoning.value !== undefined ? { reasoning: reasoning.value } : {}),\n ...(cached.value !== undefined ? { cached: cached.value } : {}),\n ...(cacheWrite.value !== undefined ? { cacheWrite: cacheWrite.value } : {}),\n },\n modelCallCount: calls.length,\n callSpanIds: calls.map((span) => span.id),\n cost: reconcileMeasurement(calls, byId, aggregateSourceIds, LLM_COST_ATTR_KEYS),\n ...(aggregate ? { aggregate } : {}),\n }\n}\n\nexport function recordAggregateMeasurements(\n raw: Record<string, number>,\n aggregate: ExecutionMeasurements['aggregate'],\n): void {\n if (!aggregate) return\n raw.aggregate_prompt_tokens = aggregate.tokenUsage.input\n raw.aggregate_completion_tokens = aggregate.tokenUsage.output\n if (aggregate.tokenUsage.reasoning !== undefined)\n raw.aggregate_reasoning_tokens = aggregate.tokenUsage.reasoning\n if (aggregate.tokenUsage.cached !== undefined)\n raw.aggregate_cached_tokens = aggregate.tokenUsage.cached\n if (aggregate.tokenUsage.cacheWrite !== undefined)\n raw.aggregate_cache_write_tokens = aggregate.tokenUsage.cacheWrite\n if (aggregate.costUsd !== undefined) raw.aggregate_cost_usd = aggregate.costUsd\n}\n\nfunction summarizeAggregateMeasurements(\n spans: ExecutionMeasurementSpan[],\n byId: Map<string, ExecutionMeasurementSpan>,\n aggregateIds: Set<string>,\n): ExecutionMeasurements['aggregate'] {\n const aggregates = spans.filter((span) => span.aggregate || aggregateIds.has(span.id))\n const input = reconcileTopLevelMeasurement(aggregates, byId, LLM_INPUT_TOKEN_ATTR_KEYS)\n const output = reconcileTopLevelMeasurement(aggregates, byId, LLM_OUTPUT_TOKEN_ATTR_KEYS)\n const reasoning = reconcileTopLevelMeasurement(aggregates, byId, LLM_REASONING_TOKEN_ATTR_KEYS)\n const cached = reconcileTopLevelMeasurement(aggregates, byId, LLM_CACHED_TOKEN_ATTR_KEYS)\n const cacheWrite = reconcileTopLevelMeasurement(aggregates, byId, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS)\n const costUsd = reconcileTopLevelMeasurement(aggregates, byId, LLM_COST_ATTR_KEYS)\n if (\n input === undefined &&\n output === undefined &&\n reasoning === undefined &&\n cached === undefined &&\n cacheWrite === undefined &&\n costUsd === undefined\n )\n return undefined\n return {\n tokenUsage: {\n input: input ?? 0,\n output: output ?? reasoning ?? 0,\n ...(reasoning !== undefined ? { reasoning } : {}),\n ...(cached !== undefined ? { cached } : {}),\n ...(cacheWrite !== undefined ? { cacheWrite } : {}),\n },\n ...(costUsd !== undefined ? { costUsd } : {}),\n }\n}\n\nfunction reconcileTopLevelMeasurement(\n spans: ExecutionMeasurementSpan[],\n byId: Map<string, ExecutionMeasurementSpan>,\n keys: readonly string[],\n): number | undefined {\n const selected = new Map<string, number>()\n for (const span of spans) {\n const value = readNumber(span.attributes, keys)\n if (value !== undefined) selected.set(span.id, value)\n }\n for (const spanId of [...selected.keys()]) {\n const span = byId.get(spanId)\n if (!span) continue\n if (ancestorIds(span, byId).some((ancestorId) => selected.has(ancestorId))) {\n selected.delete(spanId)\n }\n }\n return selected.size > 0\n ? [...selected.values()].reduce((total, value) => total + value, 0)\n : undefined\n}\n\nfunction reconcileMeasurement(\n calls: ExecutionMeasurementSpan[],\n byId: Map<string, ExecutionMeasurementSpan>,\n aggregateSourceIds: Set<string>,\n keys: readonly string[],\n fallbackKeys?: readonly string[],\n): MeasurementCoverage {\n const selected = new Map<string, number>()\n const callIds = new Set(calls.map((call) => call.id))\n let reportingCalls = 0\n\n for (const call of calls) {\n const primary = nearestMeasurement(call, byId, callIds, aggregateSourceIds, keys)\n const fallback = fallbackKeys\n ? nearestMeasurement(call, byId, callIds, aggregateSourceIds, fallbackKeys)\n : undefined\n const source =\n primary && fallback && primary.span.id === fallback.span.id\n ? { span: primary.span, value: Math.max(primary.value, fallback.value) }\n : (primary ?? fallback)\n if (!source) continue\n reportingCalls += 1\n selected.set(source.span.id, source.value)\n }\n\n for (const spanId of [...selected.keys()]) {\n const span = byId.get(spanId)\n if (!span) continue\n if (\n ancestorIds(span, byId).some(\n (ancestorId) => selected.has(ancestorId) && !callIds.has(ancestorId),\n )\n ) {\n selected.delete(spanId)\n }\n }\n\n return {\n ...(selected.size > 0\n ? { value: [...selected.values()].reduce((total, value) => total + value, 0) }\n : {}),\n reportingCalls,\n complete: calls.length > 0 && reportingCalls === calls.length,\n }\n}\n\nfunction nearestMeasurement(\n call: ExecutionMeasurementSpan,\n byId: Map<string, ExecutionMeasurementSpan>,\n callIds: Set<string>,\n aggregateSourceIds: Set<string>,\n keys: readonly string[],\n): { span: ExecutionMeasurementSpan; value: number } | undefined {\n let current: ExecutionMeasurementSpan | undefined = call\n const seen = new Set<string>()\n while (current && !seen.has(current.id)) {\n seen.add(current.id)\n const value = readNumber(current.attributes, keys)\n if (\n value !== undefined &&\n (current.id === call.id || (!callIds.has(current.id) && aggregateSourceIds.has(current.id)))\n ) {\n return { span: current, value }\n }\n current = current.parentId ? byId.get(current.parentId) : undefined\n }\n return undefined\n}\n\nfunction classifyAggregateSpans(\n candidates: ExecutionMeasurementSpan[],\n childrenById: Map<string, ExecutionMeasurementSpan[]>,\n): Set<string> {\n const aggregateIds = new Set<string>()\n const summaries = new Map<string, RetainedCallSummary>()\n const visiting = new Set<string>()\n\n const visit = (span: ExecutionMeasurementSpan): RetainedCallSummary => {\n const cached = summaries.get(span.id)\n if (cached) return cached\n if (visiting.has(span.id)) return emptyRetainedCallSummary()\n visiting.add(span.id)\n\n const descendants = emptyRetainedCallSummary()\n for (const child of childrenById.get(span.id) ?? []) {\n const childDescendants = visit(child)\n if (!aggregateIds.has(child.id)) addRetainedCall(descendants, child)\n addRetainedCallSummary(descendants, childDescendants)\n }\n\n if (\n descendants.callCount > 0 &&\n (!span.modelCall || hasCompatibleDescendantMeasurements(span, descendants))\n ) {\n aggregateIds.add(span.id)\n }\n\n visiting.delete(span.id)\n summaries.set(span.id, descendants)\n return descendants\n }\n\n for (const candidate of candidates) visit(candidate)\n return aggregateIds\n}\n\nfunction emptyRetainedCallSummary(): RetainedCallSummary {\n return {\n callCount: 0,\n measurements: EXECUTION_MEASUREMENT_KEY_GROUPS.map(() => ({\n total: 0,\n reportingCalls: 0,\n })),\n }\n}\n\nfunction addRetainedCall(summary: RetainedCallSummary, span: ExecutionMeasurementSpan): void {\n summary.callCount += 1\n for (let index = 0; index < EXECUTION_MEASUREMENT_KEY_GROUPS.length; index += 1) {\n const value = readNumber(span.attributes, EXECUTION_MEASUREMENT_KEY_GROUPS[index]!)\n if (value === undefined) continue\n const measurement = summary.measurements[index]!\n measurement.total += value\n measurement.reportingCalls += 1\n }\n}\n\nfunction addRetainedCallSummary(target: RetainedCallSummary, source: RetainedCallSummary): void {\n target.callCount += source.callCount\n for (let index = 0; index < target.measurements.length; index += 1) {\n const measurement = target.measurements[index]!\n const sourceMeasurement = source.measurements[index]!\n measurement.total += sourceMeasurement.total\n measurement.reportingCalls += sourceMeasurement.reportingCalls\n }\n}\n\nfunction hasCompatibleDescendantMeasurements(\n span: ExecutionMeasurementSpan,\n descendants: RetainedCallSummary,\n): boolean {\n let parentMeasurements = 0\n let descendantMeasurements = 0\n for (let index = 0; index < EXECUTION_MEASUREMENT_KEY_GROUPS.length; index += 1) {\n const keys = EXECUTION_MEASUREMENT_KEY_GROUPS[index]!\n const parentValue = readNumber(span.attributes, keys)\n const measurement = descendants.measurements[index]!\n if (parentValue !== undefined) parentMeasurements += 1\n if (measurement.reportingCalls > 0) descendantMeasurements += 1\n if (parentValue === undefined || measurement.reportingCalls === 0) continue\n if (measurement.reportingCalls !== descendants.callCount) continue\n if (Math.abs(parentValue - measurement.total) > 1e-12) return false\n }\n return parentMeasurements === 0 || descendantMeasurements > 0\n}\n\nfunction ancestorIds(\n span: ExecutionMeasurementSpan,\n byId: Map<string, ExecutionMeasurementSpan>,\n): string[] {\n const ids: string[] = []\n const seen = new Set<string>()\n let parentId = span.parentId\n while (parentId && !seen.has(parentId)) {\n ids.push(parentId)\n seen.add(parentId)\n parentId = byId.get(parentId)?.parentId\n }\n return ids\n}\n\nfunction nearestCandidateParent(\n span: ExecutionMeasurementSpan,\n byId: Map<string, ExecutionMeasurementSpan>,\n candidateIds: Set<string>,\n): string | undefined {\n const seen = new Set<string>()\n let parentId = span.parentId\n while (parentId && !seen.has(parentId)) {\n if (candidateIds.has(parentId)) return parentId\n seen.add(parentId)\n parentId = byId.get(parentId)?.parentId\n }\n return undefined\n}\n\nfunction readNumber(\n attributes: Record<string, unknown>,\n keys: readonly string[],\n): number | undefined {\n for (const key of keys) {\n const value = attributes[key]\n const parsed =\n typeof value === 'number'\n ? value\n : typeof value === 'string' && value.length > 0\n ? Number(value)\n : Number.NaN\n if (Number.isFinite(parsed) && parsed >= 0) return parsed\n }\n return undefined\n}\n","import { ValidationError } from '../errors'\nimport { FAILURE_CLASSES, type FailureClass } from './schema'\n\nconst TASK_FAILURE_CLASS_ATTR = 'tangle.task.failure_class'\nconst TASK_FAILURE_MODE_ATTR = 'tangle.task.failure_mode'\n\ninterface AttributeCarrier {\n attributes: Record<string, unknown>\n}\n\nexport type TaskFailureLabels =\n | { failureClass?: undefined; failureMode?: undefined }\n | { failureClass: 'success'; failureMode?: undefined }\n | { failureClass: Exclude<FailureClass, 'success'>; failureMode?: string }\n\nexport function readTaskFailureLabels(\n roots: readonly AttributeCarrier[],\n context: string,\n): TaskFailureLabels {\n const failureClass = readConsistentRootString(roots, TASK_FAILURE_CLASS_ATTR, context)\n const failureMode = readConsistentRootString(roots, TASK_FAILURE_MODE_ATTR, context)\n\n if (failureClass !== undefined && !FAILURE_CLASSES.includes(failureClass as FailureClass)) {\n throw new ValidationError(\n `${context}: ${TASK_FAILURE_CLASS_ATTR} must be one of ${FAILURE_CLASSES.join(', ')}`,\n )\n }\n if (failureMode !== undefined && (failureClass === undefined || failureClass === 'success')) {\n throw new ValidationError(\n `${context}: ${TASK_FAILURE_MODE_ATTR} requires a non-success ${TASK_FAILURE_CLASS_ATTR}`,\n )\n }\n\n if (failureClass === undefined) return {}\n if (failureClass === 'success') return { failureClass }\n return {\n failureClass: failureClass as Exclude<FailureClass, 'success'>,\n ...(failureMode ? { failureMode } : {}),\n }\n}\n\nfunction readConsistentRootString(\n roots: readonly AttributeCarrier[],\n key: string,\n context: string,\n): string | undefined {\n const values = new Set<string>()\n for (const root of roots) {\n if (!Object.hasOwn(root.attributes, key)) continue\n const value = root.attributes[key]\n if (typeof value !== 'string' || value.trim().length === 0) {\n throw new ValidationError(`${context}: ${key} must be a non-empty string`)\n }\n values.add(value)\n }\n\n if (values.size > 1) {\n throw new ValidationError(\n `${context}: conflicting ${key} values: ${[...values].sort().join(', ')}`,\n )\n }\n return values.values().next().value\n}\n"],"mappings":";;;;;;;;;AA0BA,SAAgB,qBAAqB,SAAyD;CAC5F,MAAM,uBAAO,IAAI,IAA8B;CAC/C,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,KAAK,IAAI,OAAO,EAAE,GACpB,MAAM,IAAI,MAAM,4CAA4C,OAAO,GAAG,EAAE;EAE1E,KAAK,IAAI,OAAO,IAAI,MAAM;CAC5B;CAEA,MAAM,6BAAa,IAAI,IAAY;CACnC,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,CAAC,OAAO,OAAO;EACnB,MAAM,0BAAU,IAAI,IAAY;EAChC,IAAI,WAAW,OAAO;EACtB,OAAO,YAAY,CAAC,QAAQ,IAAI,QAAQ,GAAG;GACzC,QAAQ,IAAI,QAAQ;GACpB,MAAM,SAAS,KAAK,IAAI,QAAQ;GAChC,IAAI,CAAC,QAAQ;GACb,IAAI,OAAO,OAAO,WAAW,IAAI,OAAO,EAAE;GAC1C,WAAW,OAAO;EACpB;CACF;CAEA,MAAM,UAA6B;EACjC,OAAO;EACP,WAAW;EACX,SAAS;EACT,WAAW;EACX,YAAY;EACZ,YAAY;EACZ,cAAc;CAChB;CAEA,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,CAAC,OAAO,OAAO;EACnB,QAAQ,SAAS;EACjB,IAAI,OAAO,SAAS,aAClB,QAAQ,aAAa;OAChB,IAAI,OAAO,SAAS,aACzB,QAAQ,cAAc;OACjB,IAAI,OAAO,aAChB,QAAQ,WAAW;OACd,IAAI,WAAW,IAAI,OAAO,EAAE,GACjC,QAAQ,cAAc;OACjB,IACL,OAAO,SAAS,WAChB,OAAO,SAAS,WAChB,OAAO,SAAS,SAChB,OAAO,SAAS,QAEhB,QAAQ,aAAa;OAErB,QAAQ,gBAAgB;CAE5B;CAEA,OAAO;AACT;;;AC/CA,MAAM,+BAA+B;CACnC;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,mCAAmC,CACvC,GAAG,8BACH,kBACF;;;;;;;AAgBA,SAAgB,+BACd,OACuB;CACvB,MAAM,uBAAO,IAAI,IAAsC;CACvD,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,KAAK,IAAI,KAAK,EAAE,GAClB,MAAM,IAAI,MAAM,sDAAsD,KAAK,GAAG,EAAE;EAElF,KAAK,IAAI,KAAK,IAAI,IAAI;CACxB;CACA,MAAM,uBAAuB,6BAA6B,KAAK;CAC/D,MAAM,aAAa,MAAM,QACtB,SACC,KAAK,aACJ,CAAC,KAAK,aAAa,WAAW,KAAK,YAAY,oBAAoB,MAAM,KAAA,CAC9E;CACA,MAAM,eAAe,IAAI,IAAI,WAAW,KAAK,SAAS,KAAK,EAAE,CAAC;CAC9D,MAAM,oCAAoB,IAAI,IAAwC;CACtE,KAAK,MAAM,aAAa,YAAY;EAClC,MAAM,WAAW,uBAAuB,WAAW,MAAM,YAAY;EACrE,IAAI,CAAC,UAAU;EACf,MAAM,WAAW,kBAAkB,IAAI,QAAQ,KAAK,CAAC;EACrD,SAAS,KAAK,SAAS;EACvB,kBAAkB,IAAI,UAAU,QAAQ;CAC1C;CACA,MAAM,eAAe,uBAAuB,YAAY,iBAAiB;CACzE,MAAM,oBAAoB,IAAI,IAC5B,MACG,QACE,SACC,CAAC,KAAK,aACN,CAAC,KAAK,aACN,CAAC,aAAa,IAAI,KAAK,EAAE,KACzB,WAAW,KAAK,YAAY,kBAAkB,MAAM,KAAA,CACxD,CAAC,CACA,KAAK,SAAS,KAAK,EAAE,CAC1B;CACA,MAAM,qBAAqB,IAAI,IAC7B,MACG,QACE,SAAS,KAAK,aAAa,aAAa,IAAI,KAAK,EAAE,KAAK,kBAAkB,IAAI,KAAK,EAAE,CACxF,CAAC,CACA,KAAK,SAAS,KAAK,EAAE,CAC1B;CAEA,MAAM,QAAQ,WAAW,QAAQ,SAAS,CAAC,aAAa,IAAI,KAAK,EAAE,CAAC;CACpE,MAAM,QAAQ,qBAAqB,OAAO,MAAM,oBAAoB,yBAAyB;CAC7F,MAAM,YAAY,qBAChB,OACA,MACA,oBACA,6BACF;CACA,MAAM,SAAS,qBACb,OACA,MACA,oBACA,4BACA,6BACF;CACA,MAAM,SAAS,qBAAqB,OAAO,MAAM,oBAAoB,0BAA0B;CAC/F,MAAM,aAAa,qBACjB,OACA,MACA,oBACA,+BACF;CACA,MAAM,YAAY,+BAChB,OACA,sBACA,IAAI,IAAI,CAAC,GAAG,cAAc,GAAG,iBAAiB,CAAC,CACjD;CAEA,OAAO;EACL,YAAY;GACV,OAAO,MAAM,SAAS;GACtB,QAAQ,KAAK,IAAI,OAAO,SAAS,GAAG,UAAU,SAAS,CAAC;GACxD,GAAI,UAAU,UAAU,KAAA,IAAY,EAAE,WAAW,UAAU,MAAM,IAAI,CAAC;GACtE,GAAI,OAAO,UAAU,KAAA,IAAY,EAAE,QAAQ,OAAO,MAAM,IAAI,CAAC;GAC7D,GAAI,WAAW,UAAU,KAAA,IAAY,EAAE,YAAY,WAAW,MAAM,IAAI,CAAC;EAC3E;EACA,gBAAgB,MAAM;EACtB,aAAa,MAAM,KAAK,SAAS,KAAK,EAAE;EACxC,MAAM,qBAAqB,OAAO,MAAM,oBAAoB,kBAAkB;EAC9E,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;CACnC;AACF;AAEA,SAAgB,4BACd,KACA,WACM;CACN,IAAI,CAAC,WAAW;CAChB,IAAI,0BAA0B,UAAU,WAAW;CACnD,IAAI,8BAA8B,UAAU,WAAW;CACvD,IAAI,UAAU,WAAW,cAAc,KAAA,GACrC,IAAI,6BAA6B,UAAU,WAAW;CACxD,IAAI,UAAU,WAAW,WAAW,KAAA,GAClC,IAAI,0BAA0B,UAAU,WAAW;CACrD,IAAI,UAAU,WAAW,eAAe,KAAA,GACtC,IAAI,+BAA+B,UAAU,WAAW;CAC1D,IAAI,UAAU,YAAY,KAAA,GAAW,IAAI,qBAAqB,UAAU;AAC1E;AAEA,SAAS,+BACP,OACA,MACA,cACoC;CACpC,MAAM,aAAa,MAAM,QAAQ,SAAS,KAAK,aAAa,aAAa,IAAI,KAAK,EAAE,CAAC;CACrF,MAAM,QAAQ,6BAA6B,YAAY,MAAM,yBAAyB;CACtF,MAAM,SAAS,6BAA6B,YAAY,MAAM,0BAA0B;CACxF,MAAM,YAAY,6BAA6B,YAAY,MAAM,6BAA6B;CAC9F,MAAM,SAAS,6BAA6B,YAAY,MAAM,0BAA0B;CACxF,MAAM,aAAa,6BAA6B,YAAY,MAAM,+BAA+B;CACjG,MAAM,UAAU,6BAA6B,YAAY,MAAM,kBAAkB;CACjF,IACE,UAAU,KAAA,KACV,WAAW,KAAA,KACX,cAAc,KAAA,KACd,WAAW,KAAA,KACX,eAAe,KAAA,KACf,YAAY,KAAA,GAEZ,OAAO,KAAA;CACT,OAAO;EACL,YAAY;GACV,OAAO,SAAS;GAChB,QAAQ,UAAU,aAAa;GAC/B,GAAI,cAAc,KAAA,IAAY,EAAE,UAAU,IAAI,CAAC;GAC/C,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;GACzC,GAAI,eAAe,KAAA,IAAY,EAAE,WAAW,IAAI,CAAC;EACnD;EACA,GAAI,YAAY,KAAA,IAAY,EAAE,QAAQ,IAAI,CAAC;CAC7C;AACF;AAEA,SAAS,6BACP,OACA,MACA,MACoB;CACpB,MAAM,2BAAW,IAAI,IAAoB;CACzC,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,QAAQ,WAAW,KAAK,YAAY,IAAI;EAC9C,IAAI,UAAU,KAAA,GAAW,SAAS,IAAI,KAAK,IAAI,KAAK;CACtD;CACA,KAAK,MAAM,UAAU,CAAC,GAAG,SAAS,KAAK,CAAC,GAAG;EACzC,MAAM,OAAO,KAAK,IAAI,MAAM;EAC5B,IAAI,CAAC,MAAM;EACX,IAAI,YAAY,MAAM,IAAI,CAAC,CAAC,MAAM,eAAe,SAAS,IAAI,UAAU,CAAC,GACvE,SAAS,OAAO,MAAM;CAE1B;CACA,OAAO,SAAS,OAAO,IACnB,CAAC,GAAG,SAAS,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,UAAU,QAAQ,OAAO,CAAC,IAChE,KAAA;AACN;AAEA,SAAS,qBACP,OACA,MACA,oBACA,MACA,cACqB;CACrB,MAAM,2BAAW,IAAI,IAAoB;CACzC,MAAM,UAAU,IAAI,IAAI,MAAM,KAAK,SAAS,KAAK,EAAE,CAAC;CACpD,IAAI,iBAAiB;CAErB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,UAAU,mBAAmB,MAAM,MAAM,SAAS,oBAAoB,IAAI;EAChF,MAAM,WAAW,eACb,mBAAmB,MAAM,MAAM,SAAS,oBAAoB,YAAY,IACxE,KAAA;EACJ,MAAM,SACJ,WAAW,YAAY,QAAQ,KAAK,OAAO,SAAS,KAAK,KACrD;GAAE,MAAM,QAAQ;GAAM,OAAO,KAAK,IAAI,QAAQ,OAAO,SAAS,KAAK;EAAE,IACpE,WAAW;EAClB,IAAI,CAAC,QAAQ;EACb,kBAAkB;EAClB,SAAS,IAAI,OAAO,KAAK,IAAI,OAAO,KAAK;CAC3C;CAEA,KAAK,MAAM,UAAU,CAAC,GAAG,SAAS,KAAK,CAAC,GAAG;EACzC,MAAM,OAAO,KAAK,IAAI,MAAM;EAC5B,IAAI,CAAC,MAAM;EACX,IACE,YAAY,MAAM,IAAI,CAAC,CAAC,MACrB,eAAe,SAAS,IAAI,UAAU,KAAK,CAAC,QAAQ,IAAI,UAAU,CACrE,GAEA,SAAS,OAAO,MAAM;CAE1B;CAEA,OAAO;EACL,GAAI,SAAS,OAAO,IAChB,EAAE,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,UAAU,QAAQ,OAAO,CAAC,EAAE,IAC3E,CAAC;EACL;EACA,UAAU,MAAM,SAAS,KAAK,mBAAmB,MAAM;CACzD;AACF;AAEA,SAAS,mBACP,MACA,MACA,SACA,oBACA,MAC+D;CAC/D,IAAI,UAAgD;CACpD,MAAM,uBAAO,IAAI,IAAY;CAC7B,OAAO,WAAW,CAAC,KAAK,IAAI,QAAQ,EAAE,GAAG;EACvC,KAAK,IAAI,QAAQ,EAAE;EACnB,MAAM,QAAQ,WAAW,QAAQ,YAAY,IAAI;EACjD,IACE,UAAU,KAAA,MACT,QAAQ,OAAO,KAAK,MAAO,CAAC,QAAQ,IAAI,QAAQ,EAAE,KAAK,mBAAmB,IAAI,QAAQ,EAAE,IAEzF,OAAO;GAAE,MAAM;GAAS;EAAM;EAEhC,UAAU,QAAQ,WAAW,KAAK,IAAI,QAAQ,QAAQ,IAAI,KAAA;CAC5D;AAEF;AAEA,SAAS,uBACP,YACA,cACa;CACb,MAAM,+BAAe,IAAI,IAAY;CACrC,MAAM,4BAAY,IAAI,IAAiC;CACvD,MAAM,2BAAW,IAAI,IAAY;CAEjC,MAAM,SAAS,SAAwD;EACrE,MAAM,SAAS,UAAU,IAAI,KAAK,EAAE;EACpC,IAAI,QAAQ,OAAO;EACnB,IAAI,SAAS,IAAI,KAAK,EAAE,GAAG,OAAO,yBAAyB;EAC3D,SAAS,IAAI,KAAK,EAAE;EAEpB,MAAM,cAAc,yBAAyB;EAC7C,KAAK,MAAM,SAAS,aAAa,IAAI,KAAK,EAAE,KAAK,CAAC,GAAG;GACnD,MAAM,mBAAmB,MAAM,KAAK;GACpC,IAAI,CAAC,aAAa,IAAI,MAAM,EAAE,GAAG,gBAAgB,aAAa,KAAK;GACnE,uBAAuB,aAAa,gBAAgB;EACtD;EAEA,IACE,YAAY,YAAY,MACvB,CAAC,KAAK,aAAa,oCAAoC,MAAM,WAAW,IAEzE,aAAa,IAAI,KAAK,EAAE;EAG1B,SAAS,OAAO,KAAK,EAAE;EACvB,UAAU,IAAI,KAAK,IAAI,WAAW;EAClC,OAAO;CACT;CAEA,KAAK,MAAM,aAAa,YAAY,MAAM,SAAS;CACnD,OAAO;AACT;AAEA,SAAS,2BAAgD;CACvD,OAAO;EACL,WAAW;EACX,cAAc,iCAAiC,WAAW;GACxD,OAAO;GACP,gBAAgB;EAClB,EAAE;CACJ;AACF;AAEA,SAAS,gBAAgB,SAA8B,MAAsC;CAC3F,QAAQ,aAAa;CACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,iCAAiC,QAAQ,SAAS,GAAG;EAC/E,MAAM,QAAQ,WAAW,KAAK,YAAY,iCAAiC,MAAO;EAClF,IAAI,UAAU,KAAA,GAAW;EACzB,MAAM,cAAc,QAAQ,aAAa;EACzC,YAAY,SAAS;EACrB,YAAY,kBAAkB;CAChC;AACF;AAEA,SAAS,uBAAuB,QAA6B,QAAmC;CAC9F,OAAO,aAAa,OAAO;CAC3B,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,aAAa,QAAQ,SAAS,GAAG;EAClE,MAAM,cAAc,OAAO,aAAa;EACxC,MAAM,oBAAoB,OAAO,aAAa;EAC9C,YAAY,SAAS,kBAAkB;EACvC,YAAY,kBAAkB,kBAAkB;CAClD;AACF;AAEA,SAAS,oCACP,MACA,aACS;CACT,IAAI,qBAAqB;CACzB,IAAI,yBAAyB;CAC7B,KAAK,IAAI,QAAQ,GAAG,QAAQ,iCAAiC,QAAQ,SAAS,GAAG;EAC/E,MAAM,OAAO,iCAAiC;EAC9C,MAAM,cAAc,WAAW,KAAK,YAAY,IAAI;EACpD,MAAM,cAAc,YAAY,aAAa;EAC7C,IAAI,gBAAgB,KAAA,GAAW,sBAAsB;EACrD,IAAI,YAAY,iBAAiB,GAAG,0BAA0B;EAC9D,IAAI,gBAAgB,KAAA,KAAa,YAAY,mBAAmB,GAAG;EACnE,IAAI,YAAY,mBAAmB,YAAY,WAAW;EAC1D,IAAI,KAAK,IAAI,cAAc,YAAY,KAAK,IAAI,OAAO,OAAO;CAChE;CACA,OAAO,uBAAuB,KAAK,yBAAyB;AAC9D;AAEA,SAAS,YACP,MACA,MACU;CACV,MAAM,MAAgB,CAAC;CACvB,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,WAAW,KAAK;CACpB,OAAO,YAAY,CAAC,KAAK,IAAI,QAAQ,GAAG;EACtC,IAAI,KAAK,QAAQ;EACjB,KAAK,IAAI,QAAQ;EACjB,WAAW,KAAK,IAAI,QAAQ,CAAC,EAAE;CACjC;CACA,OAAO;AACT;AAEA,SAAS,uBACP,MACA,MACA,cACoB;CACpB,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,WAAW,KAAK;CACpB,OAAO,YAAY,CAAC,KAAK,IAAI,QAAQ,GAAG;EACtC,IAAI,aAAa,IAAI,QAAQ,GAAG,OAAO;EACvC,KAAK,IAAI,QAAQ;EACjB,WAAW,KAAK,IAAI,QAAQ,CAAC,EAAE;CACjC;AAEF;AAEA,SAAS,WACP,YACA,MACoB;CACpB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,WAAW;EACzB,MAAM,SACJ,OAAO,UAAU,WACb,QACA,OAAO,UAAU,YAAY,MAAM,SAAS,IAC1C,OAAO,KAAK,IACZ;EACR,IAAI,OAAO,SAAS,MAAM,KAAK,UAAU,GAAG,OAAO;CACrD;AAEF;;;ACpaA,MAAM,0BAA0B;AAChC,MAAM,yBAAyB;AAW/B,SAAgB,sBACd,OACA,SACmB;CACnB,MAAM,eAAe,yBAAyB,OAAO,yBAAyB,OAAO;CACrF,MAAM,cAAc,yBAAyB,OAAO,wBAAwB,OAAO;CAEnF,IAAI,iBAAiB,KAAA,KAAa,CAAC,gBAAgB,SAAS,YAA4B,GACtF,MAAM,IAAI,gBACR,GAAG,QAAQ,IAAI,wBAAwB,kBAAkB,gBAAgB,KAAK,IAAI,GACpF;CAEF,IAAI,gBAAgB,KAAA,MAAc,iBAAiB,KAAA,KAAa,iBAAiB,YAC/E,MAAM,IAAI,gBACR,GAAG,QAAQ,IAAI,uBAAuB,0BAA0B,yBAClE;CAGF,IAAI,iBAAiB,KAAA,GAAW,OAAO,CAAC;CACxC,IAAI,iBAAiB,WAAW,OAAO,EAAE,aAAa;CACtD,OAAO;EACS;EACd,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;CACvC;AACF;AAEA,SAAS,yBACP,OACA,KACA,SACoB;CACpB,MAAM,yBAAS,IAAI,IAAY;CAC/B,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,OAAO,OAAO,KAAK,YAAY,GAAG,GAAG;EAC1C,MAAM,QAAQ,KAAK,WAAW;EAC9B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,WAAW,GACvD,MAAM,IAAI,gBAAgB,GAAG,QAAQ,IAAI,IAAI,4BAA4B;EAE3E,OAAO,IAAI,KAAK;CAClB;CAEA,IAAI,OAAO,OAAO,GAChB,MAAM,IAAI,gBACR,GAAG,QAAQ,gBAAgB,IAAI,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,GACxE;CAEF,OAAO,OAAO,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAChC"}