@opensearch-project/agent-health 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/cli/dist/index.js +722 -330
  2. package/dist/assets/index-D-Np_l_T.js +246 -0
  3. package/dist/assets/index-vZt9QZKf.css +1 -0
  4. package/dist/index.html +2 -2
  5. package/docs/CLI.md +138 -1
  6. package/docs/CONNECTORS.md +1 -1
  7. package/docs/SDK.md +126 -2
  8. package/docs/skills/add-connector/SKILL.md +5 -1
  9. package/lib/dist/lib/agentTrends.d.ts +210 -0
  10. package/lib/dist/lib/agentTrends.d.ts.map +1 -0
  11. package/lib/dist/lib/agentTrends.js +360 -0
  12. package/lib/dist/lib/agentTrends.js.map +1 -0
  13. package/lib/dist/lib/benchmarkCaseReview.d.ts +114 -0
  14. package/lib/dist/lib/benchmarkCaseReview.d.ts.map +1 -0
  15. package/lib/dist/lib/benchmarkCaseReview.js +177 -0
  16. package/lib/dist/lib/benchmarkCaseReview.js.map +1 -0
  17. package/lib/dist/lib/benchmarkRunsTable.d.ts +109 -0
  18. package/lib/dist/lib/benchmarkRunsTable.d.ts.map +1 -0
  19. package/lib/dist/lib/benchmarkRunsTable.js +212 -0
  20. package/lib/dist/lib/benchmarkRunsTable.js.map +1 -0
  21. package/lib/dist/lib/comparisonInsights.d.ts +49 -2
  22. package/lib/dist/lib/comparisonInsights.d.ts.map +1 -1
  23. package/lib/dist/lib/comparisonInsights.js +65 -7
  24. package/lib/dist/lib/comparisonInsights.js.map +1 -1
  25. package/lib/dist/lib/config/loader.d.ts.map +1 -1
  26. package/lib/dist/lib/config/loader.js +11 -1
  27. package/lib/dist/lib/config/loader.js.map +1 -1
  28. package/lib/dist/lib/dashboardMetrics.d.ts +11 -2
  29. package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -1
  30. package/lib/dist/lib/dashboardMetrics.js +38 -3
  31. package/lib/dist/lib/dashboardMetrics.js.map +1 -1
  32. package/lib/dist/lib/evaluationRerun.d.ts +39 -0
  33. package/lib/dist/lib/evaluationRerun.d.ts.map +1 -1
  34. package/lib/dist/lib/evaluationRerun.js +49 -0
  35. package/lib/dist/lib/evaluationRerun.js.map +1 -1
  36. package/lib/dist/lib/judgeFailureSummary.d.ts +66 -0
  37. package/lib/dist/lib/judgeFailureSummary.d.ts.map +1 -0
  38. package/lib/dist/lib/judgeFailureSummary.js +68 -0
  39. package/lib/dist/lib/judgeFailureSummary.js.map +1 -0
  40. package/lib/dist/lib/judgeStrategies.d.ts +108 -0
  41. package/lib/dist/lib/judgeStrategies.d.ts.map +1 -0
  42. package/lib/dist/lib/judgeStrategies.js +135 -0
  43. package/lib/dist/lib/judgeStrategies.js.map +1 -0
  44. package/lib/dist/lib/matchers/expect.d.ts +21 -1
  45. package/lib/dist/lib/matchers/expect.d.ts.map +1 -1
  46. package/lib/dist/lib/matchers/expect.js +51 -0
  47. package/lib/dist/lib/matchers/expect.js.map +1 -1
  48. package/lib/dist/lib/matchers/judgeAccessor.d.ts +4 -0
  49. package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -1
  50. package/lib/dist/lib/matchers/judgeAccessor.js +13 -2
  51. package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -1
  52. package/lib/dist/lib/matchers/judgeReasoningParse.d.ts +49 -0
  53. package/lib/dist/lib/matchers/judgeReasoningParse.d.ts.map +1 -0
  54. package/lib/dist/lib/matchers/judgeReasoningParse.js +128 -0
  55. package/lib/dist/lib/matchers/judgeReasoningParse.js.map +1 -0
  56. package/lib/dist/lib/matchers/types.d.ts +25 -0
  57. package/lib/dist/lib/matchers/types.d.ts.map +1 -1
  58. package/lib/dist/lib/resolveCanonicalRun.d.ts +23 -0
  59. package/lib/dist/lib/resolveCanonicalRun.d.ts.map +1 -0
  60. package/lib/dist/lib/resolveCanonicalRun.js +26 -0
  61. package/lib/dist/lib/resolveCanonicalRun.js.map +1 -0
  62. package/lib/dist/lib/runActions.d.ts +120 -0
  63. package/lib/dist/lib/runActions.d.ts.map +1 -0
  64. package/lib/dist/lib/runActions.js +130 -0
  65. package/lib/dist/lib/runActions.js.map +1 -0
  66. package/lib/dist/lib/runInsights.d.ts +86 -0
  67. package/lib/dist/lib/runInsights.d.ts.map +1 -0
  68. package/lib/dist/lib/runInsights.js +185 -0
  69. package/lib/dist/lib/runInsights.js.map +1 -0
  70. package/lib/dist/lib/runName.d.ts +29 -0
  71. package/lib/dist/lib/runName.d.ts.map +1 -0
  72. package/lib/dist/lib/runName.js +38 -0
  73. package/lib/dist/lib/runName.js.map +1 -0
  74. package/lib/dist/lib/runReportPath.d.ts +16 -0
  75. package/lib/dist/lib/runReportPath.d.ts.map +1 -0
  76. package/lib/dist/lib/runReportPath.js +22 -0
  77. package/lib/dist/lib/runReportPath.js.map +1 -0
  78. package/lib/dist/lib/runSort.d.ts +27 -0
  79. package/lib/dist/lib/runSort.d.ts.map +1 -0
  80. package/lib/dist/lib/runSort.js +31 -0
  81. package/lib/dist/lib/runSort.js.map +1 -0
  82. package/lib/dist/lib/runStats.d.ts +86 -6
  83. package/lib/dist/lib/runStats.d.ts.map +1 -1
  84. package/lib/dist/lib/runStats.js +170 -19
  85. package/lib/dist/lib/runStats.js.map +1 -1
  86. package/lib/dist/lib/testCases/define.d.ts.map +1 -1
  87. package/lib/dist/lib/testCases/define.js +93 -46
  88. package/lib/dist/lib/testCases/define.js.map +1 -1
  89. package/lib/dist/lib/testCases/judge.d.ts.map +1 -1
  90. package/lib/dist/lib/testCases/judge.js +22 -4
  91. package/lib/dist/lib/testCases/judge.js.map +1 -1
  92. package/lib/dist/lib/testCases/loader.d.ts +24 -0
  93. package/lib/dist/lib/testCases/loader.d.ts.map +1 -1
  94. package/lib/dist/lib/testCases/loader.js +253 -36
  95. package/lib/dist/lib/testCases/loader.js.map +1 -1
  96. package/lib/dist/lib/trajectoryStepDisplay.d.ts +25 -0
  97. package/lib/dist/lib/trajectoryStepDisplay.d.ts.map +1 -0
  98. package/lib/dist/lib/trajectoryStepDisplay.js +42 -0
  99. package/lib/dist/lib/trajectoryStepDisplay.js.map +1 -0
  100. package/lib/dist/lib/utils.d.ts +19 -0
  101. package/lib/dist/lib/utils.d.ts.map +1 -1
  102. package/lib/dist/lib/utils.js +27 -0
  103. package/lib/dist/lib/utils.js.map +1 -1
  104. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +68 -32
  105. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -1
  106. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +148 -125
  107. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -1
  108. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +21 -13
  109. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -1
  110. package/lib/dist/services/connectors/kiro/KiroConnector.js +22 -25
  111. package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -1
  112. package/lib/dist/services/connectors/pi/PiConnector.d.ts +49 -10
  113. package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -1
  114. package/lib/dist/services/connectors/pi/PiConnector.js +102 -86
  115. package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -1
  116. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +59 -14
  117. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -1
  118. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +87 -61
  119. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -1
  120. package/lib/dist/services/evaluation/bedrockJudge.d.ts +11 -0
  121. package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -1
  122. package/lib/dist/services/evaluation/bedrockJudge.js +2 -0
  123. package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -1
  124. package/lib/dist/services/evaluation/index.d.ts +18 -1
  125. package/lib/dist/services/evaluation/index.d.ts.map +1 -1
  126. package/lib/dist/services/evaluation/index.js +156 -19
  127. package/lib/dist/services/evaluation/index.js.map +1 -1
  128. package/lib/dist/services/metrics.d.ts +55 -0
  129. package/lib/dist/services/metrics.d.ts.map +1 -0
  130. package/lib/dist/services/metrics.js +89 -0
  131. package/lib/dist/services/metrics.js.map +1 -0
  132. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -1
  133. package/lib/dist/services/storage/asyncBenchmarkStorage.js +16 -0
  134. package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -1
  135. package/lib/dist/services/storage/asyncRunStorage.d.ts +11 -0
  136. package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -1
  137. package/lib/dist/services/storage/asyncRunStorage.js +49 -2
  138. package/lib/dist/services/storage/asyncRunStorage.js.map +1 -1
  139. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +1 -1
  140. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -1
  141. package/lib/dist/services/storage/asyncTestCaseStorage.js +2 -1
  142. package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -1
  143. package/lib/dist/services/storage/opensearchClient.d.ts +12 -0
  144. package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -1
  145. package/lib/dist/services/storage/opensearchClient.js.map +1 -1
  146. package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -1
  147. package/lib/dist/services/traces/browserRecovery.js +3 -0
  148. package/lib/dist/services/traces/browserRecovery.js.map +1 -1
  149. package/lib/dist/services/traces/index.d.ts +8 -1
  150. package/lib/dist/services/traces/index.d.ts.map +1 -1
  151. package/lib/dist/services/traces/index.js +33 -12
  152. package/lib/dist/services/traces/index.js.map +1 -1
  153. package/lib/dist/services/traces/judgeAgentsHints.d.ts +98 -3
  154. package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -1
  155. package/lib/dist/services/traces/judgeAgentsHints.js +144 -3
  156. package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -1
  157. package/lib/dist/services/traces/spansToTrajectory.js +4 -4
  158. package/lib/dist/services/traces/spansToTrajectory.js.map +1 -1
  159. package/lib/dist/services/traces/tracePoller.d.ts.map +1 -1
  160. package/lib/dist/services/traces/tracePoller.js +17 -20
  161. package/lib/dist/services/traces/tracePoller.js.map +1 -1
  162. package/lib/dist/services/traces/trajectoryMerge.d.ts +79 -0
  163. package/lib/dist/services/traces/trajectoryMerge.d.ts.map +1 -0
  164. package/lib/dist/services/traces/trajectoryMerge.js +109 -0
  165. package/lib/dist/services/traces/trajectoryMerge.js.map +1 -0
  166. package/lib/dist/types/index.d.ts +195 -2
  167. package/lib/dist/types/index.d.ts.map +1 -1
  168. package/lib/dist/types/index.js +24 -0
  169. package/lib/dist/types/index.js.map +1 -1
  170. package/package.json +6 -5
  171. package/server/dist/app.js +3398 -926
  172. package/server/dist/index.js +3401 -929
  173. package/dist/assets/index-BfxtxmKc.css +0 -1
  174. package/dist/assets/index-CrjAfDHu.js +0 -243
@@ -0,0 +1,360 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ import { bucketRunResults } from '../lib/runStats.js';
6
+ import { formatCost, formatTokens } from '../services/metrics.js';
7
+ const GAP_BREAK_MS_DEFAULT = 7 * 24 * 60 * 60 * 1000; // 7 days
8
+ /**
9
+ * Deterministic, visually distinct color palette for the chart lines / chip
10
+ * left-borders. Assigned by sorted agentKey so colors stay stable across
11
+ * re-renders and time-range/benchmark filtering (not by first-seen order,
12
+ * which would shuffle as the visible run set changes).
13
+ *
14
+ * v3 note: the small-multiples redesign no longer overlays every agent's
15
+ * line on one chart, so per-agent hue no longer needs to scale to N agents
16
+ * (the old 10-color palette repeated after the 10th agent — "indistinguishable
17
+ * colors" in the owner's feedback with 14 agents in scope). Each row/detail
18
+ * chart is already labeled by name, so a single, theme-aware accent color
19
+ * plus semantic (up/down) delta coloring is used everywhere instead — see
20
+ * `TREND_LINE_COLOR` in AgentTrendSparkline/AgentTrendDetailChart.
21
+ */
22
+ /**
23
+ * Compute passed/failed/total/accuracyPct for a run, preferring the
24
+ * denormalized `run.stats` (fast path) and falling back to bucketing
25
+ * `run.results` directly (same canonical logic used by the runs list) —
26
+ * no report fetch required either way.
27
+ */
28
+ export function getRunAccuracy(run) {
29
+ if (run.stats && run.stats.total > 0) {
30
+ const evaluable = Math.max(0, run.stats.total - (run.stats.errored || 0));
31
+ const accuracyPct = evaluable > 0 ? (run.stats.passed / evaluable) * 100 : null;
32
+ return { passed: run.stats.passed, failed: run.stats.failed, total: run.stats.total, accuracyPct };
33
+ }
34
+ const bucketed = bucketRunResults(run.results);
35
+ const evaluable = Math.max(0, bucketed.total - bucketed.errored);
36
+ const accuracyPct = evaluable > 0 ? (bucketed.passed / evaluable) * 100 : null;
37
+ return { passed: bucketed.passed, failed: bucketed.failed, total: bucketed.total, accuracyPct };
38
+ }
39
+ export function timeRangeToSinceMs(range, nowMs = Date.now()) {
40
+ const days = range === '7d' ? 7 : range === '90d' ? 90 : 30;
41
+ return nowMs - days * 24 * 60 * 60 * 1000;
42
+ }
43
+ /**
44
+ * Build one AgentRunPoint per BenchmarkRun (skipping runs with zero test
45
+ * cases — e.g. still-provisioning or cancelled-before-start runs), sorted
46
+ * ascending by time.
47
+ */
48
+ export function buildAgentRunPoints(benchmarks, reports, metricsMap, options = {}) {
49
+ const displayName = options.agentDisplayName ?? ((key) => key);
50
+ // Pre-index reports by the BenchmarkRun id they belong to
51
+ // (EvaluationReport.experimentRunId === BenchmarkRun.id).
52
+ const reportsByRunDocId = new Map();
53
+ for (const report of reports) {
54
+ if (!report.experimentRunId)
55
+ continue;
56
+ const existing = reportsByRunDocId.get(report.experimentRunId);
57
+ if (existing)
58
+ existing.push(report);
59
+ else
60
+ reportsByRunDocId.set(report.experimentRunId, [report]);
61
+ }
62
+ const points = [];
63
+ for (const benchmark of benchmarks) {
64
+ if (options.benchmarkId && benchmark.id !== options.benchmarkId)
65
+ continue;
66
+ for (const run of benchmark.runs || []) {
67
+ const timestamp = new Date(run.createdAt).getTime();
68
+ if (Number.isNaN(timestamp))
69
+ continue;
70
+ if (options.sinceMs != null && timestamp < options.sinceMs)
71
+ continue;
72
+ const { passed, failed, total, accuracyPct } = getRunAccuracy(run);
73
+ if (total === 0)
74
+ continue; // nothing to plot for an empty run
75
+ const runReports = reportsByRunDocId.get(run.id) || [];
76
+ let matched = 0;
77
+ let costSum = 0;
78
+ let tokenSum = 0;
79
+ for (const report of runReports) {
80
+ const lookup = report.runId ? metricsMap.get(report.runId) : undefined;
81
+ if (!lookup)
82
+ continue;
83
+ costSum += lookup.costUsd;
84
+ tokenSum += lookup.tokens;
85
+ matched++;
86
+ }
87
+ // Only expose a run-level cost/token total when EVERY report for this
88
+ // run resolved trace metrics. A partial sum (some test cases matched,
89
+ // some didn't) would silently understate the run's true cost/tokens
90
+ // while looking like a complete total — worse than admitting "unknown".
91
+ const isComplete = runReports.length > 0 && matched === runReports.length;
92
+ points.push({
93
+ runDocId: run.id,
94
+ benchmarkId: benchmark.id,
95
+ benchmarkName: benchmark.name,
96
+ agentKey: run.agentKey || 'unknown',
97
+ agentName: displayName(run.agentKey || 'unknown'),
98
+ modelId: run.modelId || '',
99
+ createdAt: run.createdAt,
100
+ timestamp,
101
+ passed,
102
+ failed,
103
+ total,
104
+ accuracyPct,
105
+ costUsd: isComplete ? costSum : null,
106
+ tokens: isComplete ? tokenSum : null,
107
+ });
108
+ }
109
+ }
110
+ points.sort((a, b) => a.timestamp - b.timestamp);
111
+ return points;
112
+ }
113
+ /** Group already time-sorted points by agentKey, preserving relative order. */
114
+ export function groupPointsByAgent(points) {
115
+ const map = new Map();
116
+ for (const point of points) {
117
+ const existing = map.get(point.agentKey);
118
+ if (existing)
119
+ existing.push(point);
120
+ else
121
+ map.set(point.agentKey, [point]);
122
+ }
123
+ return map;
124
+ }
125
+ export function metricValue(point, metric) {
126
+ switch (metric) {
127
+ case 'accuracy':
128
+ return point.accuracyPct;
129
+ case 'cost':
130
+ return point.costUsd;
131
+ case 'tokens':
132
+ return point.tokens;
133
+ default:
134
+ return null;
135
+ }
136
+ }
137
+ export function formatMetricValue(metric, value) {
138
+ switch (metric) {
139
+ case 'accuracy':
140
+ return `${value.toFixed(1)}%`;
141
+ case 'cost':
142
+ return formatCost(value);
143
+ case 'tokens':
144
+ return formatTokens(value);
145
+ default:
146
+ return String(value);
147
+ }
148
+ }
149
+ /**
150
+ * Build one agent's gap-broken series for `metric`: points whose metric
151
+ * value is null (no evaluable accuracy, or no trace-resolved cost/tokens)
152
+ * are dropped — never plotted as a fake zero. A synthetic `{ value: null }`
153
+ * marker is inserted at the midpoint between any two consecutive PLOTTED
154
+ * points whose time gap exceeds `gapBreakMs` (default 7 days).
155
+ *
156
+ * Why: a chart line drawn between two data-array entries connects them
157
+ * with a straight (or smoothed) segment regardless of how far apart their
158
+ * x-values are — so two runs 25 days apart look, visually, like a
159
+ * continuous trend across those 25 days ("sparse runs produce misleading
160
+ * long interpolated lines" — owner feedback). Inserting a null-valued
161
+ * entry between them gives recharts' `<Line connectNulls={false}>` (the
162
+ * default) an explicit place to break the path, while the surrounding
163
+ * dead space still renders proportionally on a time-scaled x-axis.
164
+ */
165
+ export function buildGapBrokenSeries(agentPoints, // already time-sorted, single agent
166
+ metric, gapBreakMs = GAP_BREAK_MS_DEFAULT) {
167
+ const valued = agentPoints
168
+ .map(p => ({ p, v: metricValue(p, metric) }))
169
+ .filter((d) => d.v != null);
170
+ const out = [];
171
+ for (let i = 0; i < valued.length; i++) {
172
+ const { p, v } = valued[i];
173
+ if (i > 0) {
174
+ const prev = valued[i - 1].p;
175
+ if (p.timestamp - prev.timestamp > gapBreakMs) {
176
+ out.push({ timestamp: (prev.timestamp + p.timestamp) / 2, value: null, point: null });
177
+ }
178
+ }
179
+ out.push({ timestamp: p.timestamp, value: v, point: p });
180
+ }
181
+ return out;
182
+ }
183
+ export function computeLatestDelta(agentPoints, metric) {
184
+ const validValues = agentPoints
185
+ .map(p => metricValue(p, metric))
186
+ .filter((v) => v != null);
187
+ const n = validValues.length;
188
+ const latestValue = n >= 1 ? validValues[n - 1] : null;
189
+ const previousValue = n >= 2 ? validValues[n - 2] : null;
190
+ const delta = latestValue != null && previousValue != null ? latestValue - previousValue : null;
191
+ return { latestValue, previousValue, delta };
192
+ }
193
+ export function formatDelta(metric, delta) {
194
+ if (delta == null)
195
+ return 'n/a';
196
+ const sign = delta > 0 ? '+' : delta < 0 ? '-' : '±';
197
+ if (metric === 'accuracy') {
198
+ const rounded = Math.round(delta * 10) / 10;
199
+ if (Math.abs(rounded) < 0.05)
200
+ return '±0pp';
201
+ return `${rounded > 0 ? '+' : ''}${rounded}pp`;
202
+ }
203
+ return `${sign}${formatMetricValue(metric, Math.abs(delta))}`;
204
+ }
205
+ /**
206
+ * Group `points` by agent and shape each into a sparkline-table row for
207
+ * the given metric. One row per agent that has at least one point in
208
+ * scope — a single run is a perfectly valid row (a lone dot, `delta: null`
209
+ * rendered as "n/a"); there is no "not enough data" floor at the row
210
+ * level (unlike the old all-agents chart, which needed >=2 points across
211
+ * ALL agents just to avoid an empty canvas).
212
+ */
213
+ export function buildAgentTrendRows(points, metric, gapBreakMs = GAP_BREAK_MS_DEFAULT) {
214
+ const grouped = groupPointsByAgent(points);
215
+ const rows = [];
216
+ for (const [agentKey, agentPoints] of grouped) {
217
+ if (agentPoints.length === 0)
218
+ continue;
219
+ const latest = agentPoints[agentPoints.length - 1];
220
+ const { latestValue, previousValue, delta } = computeLatestDelta(agentPoints, metric);
221
+ rows.push({
222
+ agentKey,
223
+ agentName: latest.agentName,
224
+ runCount: agentPoints.length,
225
+ latestRunAt: latest.createdAt,
226
+ latestRunDocId: latest.runDocId,
227
+ latestBenchmarkId: latest.benchmarkId,
228
+ points: agentPoints,
229
+ series: buildGapBrokenSeries(agentPoints, metric, gapBreakMs),
230
+ latestValue,
231
+ previousValue,
232
+ delta,
233
+ });
234
+ }
235
+ return rows;
236
+ }
237
+ /**
238
+ * Sort rows by latest value (highest first; rows with no value for the
239
+ * current metric sort last) or by biggest drop (most negative delta
240
+ * first; rows with no delta sort last, ordered among themselves by latest
241
+ * value so a still-informative "no signal yet" row isn't buried below a
242
+ * literal `undefined`-sorts-arbitrarily bucket).
243
+ */
244
+ export function sortAgentTrendRows(rows, sortMode) {
245
+ const sorted = [...rows];
246
+ if (sortMode === 'biggestDrop') {
247
+ sorted.sort((a, b) => {
248
+ if (a.delta == null && b.delta == null)
249
+ return (b.latestValue ?? -Infinity) - (a.latestValue ?? -Infinity);
250
+ if (a.delta == null)
251
+ return 1;
252
+ if (b.delta == null)
253
+ return -1;
254
+ return a.delta - b.delta; // ascending: most negative (biggest drop) first
255
+ });
256
+ }
257
+ else {
258
+ sorted.sort((a, b) => {
259
+ if (a.latestValue == null && b.latestValue == null)
260
+ return 0;
261
+ if (a.latestValue == null)
262
+ return 1;
263
+ if (b.latestValue == null)
264
+ return -1;
265
+ return b.latestValue - a.latestValue; // descending: highest first
266
+ });
267
+ }
268
+ return sorted;
269
+ }
270
+ /** Default benchmark selector value: the benchmark with the most recent run. */
271
+ export function getMostRecentlyActiveBenchmarkId(benchmarks) {
272
+ let bestId = null;
273
+ let bestTimestamp = -Infinity;
274
+ for (const benchmark of benchmarks) {
275
+ for (const run of benchmark.runs || []) {
276
+ const ts = new Date(run.createdAt).getTime();
277
+ if (!Number.isNaN(ts) && ts > bestTimestamp) {
278
+ bestTimestamp = ts;
279
+ bestId = benchmark.id;
280
+ }
281
+ }
282
+ }
283
+ return bestId;
284
+ }
285
+ /**
286
+ * Group `points` (already scoped to one benchmark by the caller) by agent
287
+ * and split each agent's valued runs into `latest` (most recent with a
288
+ * resolved value) + `history` (earlier ones). An agent with zero valued
289
+ * runs for this metric still gets a row (`latest: null`) so it's visible
290
+ * in the list rather than silently disappearing when switching metrics.
291
+ */
292
+ export function buildBenchmarkDotPlotRows(points, metric) {
293
+ const grouped = groupPointsByAgent(points);
294
+ const rows = [];
295
+ for (const [agentKey, agentPoints] of grouped) {
296
+ const valued = agentPoints
297
+ .map(p => ({ point: p, value: metricValue(p, metric) }))
298
+ .filter((d) => d.value != null);
299
+ const agentName = agentPoints[0].agentName;
300
+ if (valued.length === 0) {
301
+ rows.push({ agentKey, agentName, latest: null, history: [], rank: -1 });
302
+ continue;
303
+ }
304
+ const latest = valued[valued.length - 1];
305
+ const history = valued.slice(0, -1);
306
+ rows.push({ agentKey, agentName, latest, history, rank: -1 });
307
+ }
308
+ return rows;
309
+ }
310
+ /**
311
+ * Rank rows by latest value, best (highest) first — rows with no value for
312
+ * the current metric sort last (still listed, just unranked among
313
+ * themselves by name for a stable order). Stamps `rank` (0 = top) on the
314
+ * returned copies; does not mutate the input.
315
+ */
316
+ export function rankDotPlotRows(rows) {
317
+ const sorted = [...rows].sort((a, b) => {
318
+ if (a.latest == null && b.latest == null)
319
+ return a.agentName.localeCompare(b.agentName);
320
+ if (a.latest == null)
321
+ return 1;
322
+ if (b.latest == null)
323
+ return -1;
324
+ return b.latest.value - a.latest.value;
325
+ });
326
+ return sorted.map((row, index) => ({ ...row, rank: index }));
327
+ }
328
+ /**
329
+ * Shared X-domain for a dot plot: accuracy is always anchored to the true
330
+ * [0, 100] scale (auto-scaling to the observed min/max would visually
331
+ * exaggerate small real-world differences, e.g. an 88-95% cluster would
332
+ * stretch to fill the whole width). Cost/tokens are zero-anchored (0 is a
333
+ * meaningful reference point for both) up to the observed max plus a
334
+ * little headroom so the top dot isn't flush against the edge. Returns
335
+ * `null` when there is nothing to plot at all.
336
+ */
337
+ export function metricDomain(rows, metric) {
338
+ const values = [];
339
+ for (const row of rows) {
340
+ if (row.latest)
341
+ values.push(row.latest.value);
342
+ for (const h of row.history)
343
+ values.push(h.value);
344
+ }
345
+ if (values.length === 0)
346
+ return null;
347
+ if (metric === 'accuracy')
348
+ return [0, 100];
349
+ const max = Math.max(...values);
350
+ return [0, max <= 0 ? 1 : max * 1.08];
351
+ }
352
+ /** Map a value into a 0-100 position within `domain`, clamped — usable directly as a CSS `left`/`width` percentage. */
353
+ export function valueToPercent(value, domain) {
354
+ const [min, max] = domain;
355
+ if (max === min)
356
+ return 50;
357
+ const pct = ((value - min) / (max - min)) * 100;
358
+ return Math.max(0, Math.min(100, pct));
359
+ }
360
+ //# sourceMappingURL=agentTrends.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agentTrends.js","sourceRoot":"","sources":["../../agentTrends.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA4BH,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAsC9D,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAE/D;;;;;;;;;;;;;GAaG;AAEH;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,GAAiB;IAM9C,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAChF,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/E,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,QAAgB,IAAI,CAAC,GAAG,EAAE;IACnF,MAAM,IAAI,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5D,OAAO,KAAK,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC5C,CAAC;AAQD;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,UAAuB,EACvB,OAA2B,EAC3B,UAAyC,EACzC,UAAsC,EAAE;IAExC,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAEvE,0DAA0D;IAC1D,0DAA0D;IAC1D,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,eAAe;YAAE,SAAS;QACtC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAC/D,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;YAC/B,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,WAAW,IAAI,SAAS,CAAC,EAAE,KAAK,OAAO,CAAC,WAAW;YAAE,SAAS;QAE1E,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;YACpD,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;gBAAE,SAAS;YACtC,IAAI,OAAO,CAAC,OAAO,IAAI,IAAI,IAAI,SAAS,GAAG,OAAO,CAAC,OAAO;gBAAE,SAAS;YAErE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACnE,IAAI,KAAK,KAAK,CAAC;gBAAE,SAAS,CAAC,mCAAmC;YAE9D,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YACvD,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACvE,IAAI,CAAC,MAAM;oBAAE,SAAS;gBACtB,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;gBAC1B,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC1B,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,oEAAoE;YACpE,wEAAwE;YACxE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,KAAK,UAAU,CAAC,MAAM,CAAC;YAE1E,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,GAAG,CAAC,EAAE;gBAChB,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,aAAa,EAAE,SAAS,CAAC,IAAI;gBAC7B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;gBACnC,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,QAAQ,IAAI,SAAS,CAAC;gBACjD,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;gBAC1B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,SAAS;gBACT,MAAM;gBACN,MAAM;gBACN,KAAK;gBACL,WAAW;gBACX,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;gBACpC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;aACrC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,kBAAkB,CAAC,MAAuB;IACxD,MAAM,GAAG,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;YAC9B,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAoB,EAAE,MAAsB;IACtE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU;YACb,OAAO,KAAK,CAAC,WAAW,CAAC;QAC3B,KAAK,MAAM;YACT,OAAO,KAAK,CAAC,OAAO,CAAC;QACvB,KAAK,QAAQ;YACX,OAAO,KAAK,CAAC,MAAM,CAAC;QACtB;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAsB,EAAE,KAAa;IACrE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU;YACb,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAChC,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7B;YACE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;AACH,CAAC;AASD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,oBAAoB,CAClC,WAA4B,EAAE,oCAAoC;AAClE,MAAsB,EACtB,aAAqB,oBAAoB;IAEzC,MAAM,MAAM,GAAG,WAAW;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;SAC5C,MAAM,CAAC,CAAC,CAAC,EAAwC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IAEpE,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC;gBAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAUD,MAAM,UAAU,kBAAkB,CAAC,WAA4B,EAAE,MAAsB;IACrF,MAAM,WAAW,GAAG,WAAW;SAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;IAC7B,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,aAAa,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,MAAM,KAAK,GAAG,WAAW,IAAI,IAAI,IAAI,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAsB,EAAE,KAAoB;IACtE,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,KAAK,CAAC;IAChC,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI;YAAE,OAAO,MAAM,CAAC;QAC5C,OAAO,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,IAAI,CAAC;IACjD,CAAC;IACD,OAAO,GAAG,IAAI,GAAG,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;AAChE,CAAC;AAmBD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAuB,EACvB,MAAsB,EACtB,aAAqB,oBAAoB;IAEzC,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,IAAI,GAAoB,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACvC,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnD,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,kBAAkB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACtF,IAAI,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,WAAW,CAAC,MAAM;YAC5B,WAAW,EAAE,MAAM,CAAC,SAAS;YAC7B,cAAc,EAAE,MAAM,CAAC,QAAQ;YAC/B,iBAAiB,EAAE,MAAM,CAAC,WAAW;YACrC,MAAM,EAAE,WAAW;YACnB,MAAM,EAAE,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC;YAC7D,WAAW;YACX,aAAa;YACb,KAAK;SACN,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAqB,EAAE,QAAuB;IAC/E,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACzB,IAAI,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACnB,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3G,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;gBAAE,OAAO,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC/B,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,gDAAgD;QAC5E,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACnB,IAAI,CAAC,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,CAAC,WAAW,IAAI,IAAI;gBAAE,OAAO,CAAC,CAAC;YAC7D,IAAI,CAAC,CAAC,WAAW,IAAI,IAAI;gBAAE,OAAO,CAAC,CAAC;YACpC,IAAI,CAAC,CAAC,WAAW,IAAI,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC;YACrC,OAAO,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,4BAA4B;QACpE,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,gCAAgC,CAAC,UAAuB;IACtE,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,aAAa,GAAG,CAAC,QAAQ,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,aAAa,EAAE,CAAC;gBAC5C,aAAa,GAAG,EAAE,CAAC;gBACnB,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AA6BD;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAAuB,EACvB,MAAsB;IAEtB,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,IAAI,GAA0B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,WAAW;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;aACvD,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,IAA2B;IACzD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrC,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI;YAAE,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACxF,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI;YAAE,OAAO,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,IAA2B,EAAE,MAAsB;IAC9E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,OAAO,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,uHAAuH;AACvH,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,MAAwB;IACpE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;IAC1B,IAAI,GAAG,KAAK,GAAG;QAAE,OAAO,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACzC,CAAC"}
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Pure helpers for the benchmark case-review matrix.
3
+ *
4
+ * The benchmark document supplies the canonical run/case relationship, while
5
+ * lightweight report summaries supply the actual pass/fail/error verdicts.
6
+ * Keeping this logic outside React makes the bucket priority and pager order
7
+ * deterministic (and cheap to test) for suites with hundreds of cases.
8
+ */
9
+ export type CaseVerdict = 'passed' | 'failed' | 'errored' | 'not-run';
10
+ export type CaseReviewBucket = 'needs-attention' | 'flaky' | 'stable' | 'no-data';
11
+ export type CaseReviewFilter = 'all' | Exclude<CaseReviewBucket, 'no-data'>;
12
+ export type SwipeDirection = 'prev' | 'next';
13
+ export interface PagerDragInput {
14
+ dx: number;
15
+ dy: number;
16
+ latched: boolean;
17
+ atStart: boolean;
18
+ atEnd: boolean;
19
+ }
20
+ export interface PagerDragResult {
21
+ offset: number;
22
+ latched: boolean;
23
+ }
24
+ /**
25
+ * Latch a horizontal pager drag after a small slop, then preserve that latch
26
+ * for the rest of the gesture. Pulling beyond either end receives resistance
27
+ * so the pane communicates the boundary without changing cases.
28
+ *
29
+ * Reduced-motion behavior intentionally stays out of this geometry helper;
30
+ * callers suppress the returned offset while retaining gesture navigation.
31
+ */
32
+ export declare function computePagerDrag({ dx, dy, latched, atStart, atEnd, }: PagerDragInput): PagerDragResult;
33
+ /**
34
+ * Resolve a completed gesture without suppressing native touch movement. A
35
+ * negative x displacement is a left swipe (next); a positive one is previous.
36
+ */
37
+ export declare function resolveSwipeDirection(dx: number, dy: number, touchCount?: number): SwipeDirection | null;
38
+ export interface VerdictReportSummary {
39
+ status?: string;
40
+ passFailStatus?: 'passed' | 'failed' | null;
41
+ metricsStatus?: string;
42
+ }
43
+ export interface VerdictRunResult {
44
+ reportId?: string;
45
+ status?: string;
46
+ /** Present on newer embedded benchmark results; reports remain authoritative. */
47
+ passFailStatus?: 'passed' | 'failed';
48
+ }
49
+ export interface VerdictRun {
50
+ id: string;
51
+ createdAt: string;
52
+ status?: string;
53
+ results?: Record<string, VerdictRunResult>;
54
+ }
55
+ export interface ReviewableCase {
56
+ id: string;
57
+ name: string;
58
+ prompt?: string;
59
+ }
60
+ export interface CaseReviewRow<T extends ReviewableCase = ReviewableCase> {
61
+ testCase: T;
62
+ verdicts: CaseVerdict[];
63
+ bucket: CaseReviewBucket;
64
+ passRate: number | null;
65
+ }
66
+ /** A legacy run with no top-level status is complete once all results are terminal. */
67
+ export declare function isCompletedVerdictRun(run: VerdictRun): boolean;
68
+ export declare function getRecentCompletedRuns<T extends VerdictRun>(runs: T[], limit?: number): T[];
69
+ /**
70
+ * Resolve one case×run cell. A real pass/fail verdict from the report is
71
+ * authoritative and outranks `metricsStatus === 'error'`: metrics/cost
72
+ * enrichment runs after the agent-vs-judge verdict is already known, so a
73
+ * later enrichment failure must not overwrite a genuine pass/fail with
74
+ * 'errored'. `metricsStatus === 'error'` only decides the cell when the
75
+ * report has no real passFailStatus yet.
76
+ */
77
+ export declare function deriveCaseVerdict(run: VerdictRun, testCaseId: string, reportsById: Record<string, VerdictReportSummary | undefined>): CaseVerdict;
78
+ export declare function deriveCaseVerdicts(runs: VerdictRun[], testCaseId: string, reportsById: Record<string, VerdictReportSummary | undefined>): CaseVerdict[];
79
+ /**
80
+ * Buckets are disjoint by priority. Evaluator errors require attention but do
81
+ * not become failed verdicts or enter the pass-rate denominator.
82
+ */
83
+ export declare function classifyCaseVerdicts(verdicts: CaseVerdict[]): CaseReviewBucket;
84
+ export declare function getCasePassRate(verdicts: CaseVerdict[]): number | null;
85
+ export declare function buildCaseReviewRows<T extends ReviewableCase>(testCases: T[], recentRuns: VerdictRun[], reportsById: Record<string, VerdictReportSummary | undefined>): CaseReviewRow<T>[];
86
+ /** Search/filter first, then pass-rate ascending; no-data sorts last. */
87
+ export declare function filterAndSortCaseRows<T extends ReviewableCase>(rows: CaseReviewRow<T>[], search: string, filter: CaseReviewFilter): CaseReviewRow<T>[];
88
+ export declare function getCasePagerPosition<T extends ReviewableCase>(rows: CaseReviewRow<T>[], selectedCaseId: string | undefined): {
89
+ index: number;
90
+ position: number;
91
+ total: number;
92
+ previousId?: string;
93
+ nextId?: string;
94
+ };
95
+ export interface VersionedRunResult {
96
+ reportId?: string;
97
+ }
98
+ export interface VersionedRun {
99
+ results?: Record<string, VersionedRunResult>;
100
+ }
101
+ export interface VersionedReportSummary {
102
+ testCaseVersion?: number;
103
+ }
104
+ /**
105
+ * The Cases tab always renders a case's *current* definition, even when the
106
+ * user arrived from a specific historical run (e.g. a heat-strip cell click).
107
+ * This detects that mismatch so callers can show a minimal, honest notice
108
+ * instead of silently presenting a definition the run never actually used.
109
+ * Deliberately does not resurrect the old per-version selector — just a
110
+ * disclosure that the definition may have changed since some of the runs
111
+ * shown here.
112
+ */
113
+ export declare function hasHistoricalVersionMismatch(currentVersion: number, runs: VersionedRun[], testCaseId: string, reportsById: Record<string, VersionedReportSummary | undefined>): boolean;
114
+ //# sourceMappingURL=benchmarkCaseReview.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmarkCaseReview.d.ts","sourceRoot":"","sources":["../../benchmarkCaseReview.ts"],"names":[],"mappings":"AAKA;;;;;;;GAOG;AAEH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;AACtE,MAAM,MAAM,gBAAgB,GAAG,iBAAiB,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAClF,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,OAAO,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;AAC5E,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,CAAC;AAO7C,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,EAC/B,EAAE,EACF,EAAE,EACF,OAAO,EACP,OAAO,EACP,KAAK,GACN,EAAE,cAAc,GAAG,eAAe,CAalC;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,UAAU,SAAI,GACb,cAAc,GAAG,IAAI,CAKvB;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAC5C,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iFAAiF;IACjF,cAAc,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,cAAc,GAAG,cAAc;IACtE,QAAQ,EAAE,CAAC,CAAC;IACZ,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,uFAAuF;AACvF,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAM9D;AAED,wBAAgB,sBAAsB,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,KAAK,SAAI,GAAG,CAAC,EAAE,CAKtF;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,UAAU,EACf,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,GAAG,SAAS,CAAC,GAC5D,WAAW,CAsBb;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,UAAU,EAAE,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,GAAG,SAAS,CAAC,GAC5D,WAAW,EAAE,CAEf;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,gBAAgB,CAmB9E;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,GAAG,IAAI,CAItE;AAED,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,cAAc,EAC1D,SAAS,EAAE,CAAC,EAAE,EACd,UAAU,EAAE,UAAU,EAAE,EACxB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,GAAG,SAAS,CAAC,GAC5D,aAAa,CAAC,CAAC,CAAC,EAAE,CAUpB;AAED,yEAAyE;AACzE,wBAAgB,qBAAqB,CAAC,CAAC,SAAS,cAAc,EAC5D,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,EACxB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,gBAAgB,GACvB,aAAa,CAAC,CAAC,CAAC,EAAE,CAgBpB;AAED,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,cAAc,EAC3D,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,EACxB,cAAc,EAAE,MAAM,GAAG,SAAS,GACjC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAS1F;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,sBAAsB;IACrC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC1C,cAAc,EAAE,MAAM,EACtB,IAAI,EAAE,YAAY,EAAE,EACpB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,GAAG,SAAS,CAAC,GAC9D,OAAO,CAMT"}