@opensearch-project/agent-health 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/cli/dist/index.js +722 -330
  2. package/dist/assets/index-D-Np_l_T.js +246 -0
  3. package/dist/assets/index-vZt9QZKf.css +1 -0
  4. package/dist/index.html +2 -2
  5. package/docs/CLI.md +138 -1
  6. package/docs/CONNECTORS.md +1 -1
  7. package/docs/SDK.md +126 -2
  8. package/docs/skills/add-connector/SKILL.md +5 -1
  9. package/lib/dist/lib/agentTrends.d.ts +210 -0
  10. package/lib/dist/lib/agentTrends.d.ts.map +1 -0
  11. package/lib/dist/lib/agentTrends.js +360 -0
  12. package/lib/dist/lib/agentTrends.js.map +1 -0
  13. package/lib/dist/lib/benchmarkCaseReview.d.ts +114 -0
  14. package/lib/dist/lib/benchmarkCaseReview.d.ts.map +1 -0
  15. package/lib/dist/lib/benchmarkCaseReview.js +177 -0
  16. package/lib/dist/lib/benchmarkCaseReview.js.map +1 -0
  17. package/lib/dist/lib/benchmarkRunsTable.d.ts +109 -0
  18. package/lib/dist/lib/benchmarkRunsTable.d.ts.map +1 -0
  19. package/lib/dist/lib/benchmarkRunsTable.js +212 -0
  20. package/lib/dist/lib/benchmarkRunsTable.js.map +1 -0
  21. package/lib/dist/lib/comparisonInsights.d.ts +49 -2
  22. package/lib/dist/lib/comparisonInsights.d.ts.map +1 -1
  23. package/lib/dist/lib/comparisonInsights.js +65 -7
  24. package/lib/dist/lib/comparisonInsights.js.map +1 -1
  25. package/lib/dist/lib/config/loader.d.ts.map +1 -1
  26. package/lib/dist/lib/config/loader.js +11 -1
  27. package/lib/dist/lib/config/loader.js.map +1 -1
  28. package/lib/dist/lib/dashboardMetrics.d.ts +11 -2
  29. package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -1
  30. package/lib/dist/lib/dashboardMetrics.js +38 -3
  31. package/lib/dist/lib/dashboardMetrics.js.map +1 -1
  32. package/lib/dist/lib/evaluationRerun.d.ts +39 -0
  33. package/lib/dist/lib/evaluationRerun.d.ts.map +1 -1
  34. package/lib/dist/lib/evaluationRerun.js +49 -0
  35. package/lib/dist/lib/evaluationRerun.js.map +1 -1
  36. package/lib/dist/lib/judgeFailureSummary.d.ts +66 -0
  37. package/lib/dist/lib/judgeFailureSummary.d.ts.map +1 -0
  38. package/lib/dist/lib/judgeFailureSummary.js +68 -0
  39. package/lib/dist/lib/judgeFailureSummary.js.map +1 -0
  40. package/lib/dist/lib/judgeStrategies.d.ts +108 -0
  41. package/lib/dist/lib/judgeStrategies.d.ts.map +1 -0
  42. package/lib/dist/lib/judgeStrategies.js +135 -0
  43. package/lib/dist/lib/judgeStrategies.js.map +1 -0
  44. package/lib/dist/lib/matchers/expect.d.ts +21 -1
  45. package/lib/dist/lib/matchers/expect.d.ts.map +1 -1
  46. package/lib/dist/lib/matchers/expect.js +51 -0
  47. package/lib/dist/lib/matchers/expect.js.map +1 -1
  48. package/lib/dist/lib/matchers/judgeAccessor.d.ts +4 -0
  49. package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -1
  50. package/lib/dist/lib/matchers/judgeAccessor.js +13 -2
  51. package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -1
  52. package/lib/dist/lib/matchers/judgeReasoningParse.d.ts +49 -0
  53. package/lib/dist/lib/matchers/judgeReasoningParse.d.ts.map +1 -0
  54. package/lib/dist/lib/matchers/judgeReasoningParse.js +128 -0
  55. package/lib/dist/lib/matchers/judgeReasoningParse.js.map +1 -0
  56. package/lib/dist/lib/matchers/types.d.ts +25 -0
  57. package/lib/dist/lib/matchers/types.d.ts.map +1 -1
  58. package/lib/dist/lib/resolveCanonicalRun.d.ts +23 -0
  59. package/lib/dist/lib/resolveCanonicalRun.d.ts.map +1 -0
  60. package/lib/dist/lib/resolveCanonicalRun.js +26 -0
  61. package/lib/dist/lib/resolveCanonicalRun.js.map +1 -0
  62. package/lib/dist/lib/runActions.d.ts +120 -0
  63. package/lib/dist/lib/runActions.d.ts.map +1 -0
  64. package/lib/dist/lib/runActions.js +130 -0
  65. package/lib/dist/lib/runActions.js.map +1 -0
  66. package/lib/dist/lib/runInsights.d.ts +86 -0
  67. package/lib/dist/lib/runInsights.d.ts.map +1 -0
  68. package/lib/dist/lib/runInsights.js +185 -0
  69. package/lib/dist/lib/runInsights.js.map +1 -0
  70. package/lib/dist/lib/runName.d.ts +29 -0
  71. package/lib/dist/lib/runName.d.ts.map +1 -0
  72. package/lib/dist/lib/runName.js +38 -0
  73. package/lib/dist/lib/runName.js.map +1 -0
  74. package/lib/dist/lib/runReportPath.d.ts +16 -0
  75. package/lib/dist/lib/runReportPath.d.ts.map +1 -0
  76. package/lib/dist/lib/runReportPath.js +22 -0
  77. package/lib/dist/lib/runReportPath.js.map +1 -0
  78. package/lib/dist/lib/runSort.d.ts +27 -0
  79. package/lib/dist/lib/runSort.d.ts.map +1 -0
  80. package/lib/dist/lib/runSort.js +31 -0
  81. package/lib/dist/lib/runSort.js.map +1 -0
  82. package/lib/dist/lib/runStats.d.ts +86 -6
  83. package/lib/dist/lib/runStats.d.ts.map +1 -1
  84. package/lib/dist/lib/runStats.js +170 -19
  85. package/lib/dist/lib/runStats.js.map +1 -1
  86. package/lib/dist/lib/testCases/define.d.ts.map +1 -1
  87. package/lib/dist/lib/testCases/define.js +93 -46
  88. package/lib/dist/lib/testCases/define.js.map +1 -1
  89. package/lib/dist/lib/testCases/judge.d.ts.map +1 -1
  90. package/lib/dist/lib/testCases/judge.js +22 -4
  91. package/lib/dist/lib/testCases/judge.js.map +1 -1
  92. package/lib/dist/lib/testCases/loader.d.ts +24 -0
  93. package/lib/dist/lib/testCases/loader.d.ts.map +1 -1
  94. package/lib/dist/lib/testCases/loader.js +253 -36
  95. package/lib/dist/lib/testCases/loader.js.map +1 -1
  96. package/lib/dist/lib/trajectoryStepDisplay.d.ts +25 -0
  97. package/lib/dist/lib/trajectoryStepDisplay.d.ts.map +1 -0
  98. package/lib/dist/lib/trajectoryStepDisplay.js +42 -0
  99. package/lib/dist/lib/trajectoryStepDisplay.js.map +1 -0
  100. package/lib/dist/lib/utils.d.ts +19 -0
  101. package/lib/dist/lib/utils.d.ts.map +1 -1
  102. package/lib/dist/lib/utils.js +27 -0
  103. package/lib/dist/lib/utils.js.map +1 -1
  104. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +68 -32
  105. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -1
  106. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +148 -125
  107. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -1
  108. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +21 -13
  109. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -1
  110. package/lib/dist/services/connectors/kiro/KiroConnector.js +22 -25
  111. package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -1
  112. package/lib/dist/services/connectors/pi/PiConnector.d.ts +49 -10
  113. package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -1
  114. package/lib/dist/services/connectors/pi/PiConnector.js +102 -86
  115. package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -1
  116. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +59 -14
  117. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -1
  118. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +87 -61
  119. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -1
  120. package/lib/dist/services/evaluation/bedrockJudge.d.ts +11 -0
  121. package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -1
  122. package/lib/dist/services/evaluation/bedrockJudge.js +2 -0
  123. package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -1
  124. package/lib/dist/services/evaluation/index.d.ts +18 -1
  125. package/lib/dist/services/evaluation/index.d.ts.map +1 -1
  126. package/lib/dist/services/evaluation/index.js +156 -19
  127. package/lib/dist/services/evaluation/index.js.map +1 -1
  128. package/lib/dist/services/metrics.d.ts +55 -0
  129. package/lib/dist/services/metrics.d.ts.map +1 -0
  130. package/lib/dist/services/metrics.js +89 -0
  131. package/lib/dist/services/metrics.js.map +1 -0
  132. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -1
  133. package/lib/dist/services/storage/asyncBenchmarkStorage.js +16 -0
  134. package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -1
  135. package/lib/dist/services/storage/asyncRunStorage.d.ts +11 -0
  136. package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -1
  137. package/lib/dist/services/storage/asyncRunStorage.js +49 -2
  138. package/lib/dist/services/storage/asyncRunStorage.js.map +1 -1
  139. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +1 -1
  140. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -1
  141. package/lib/dist/services/storage/asyncTestCaseStorage.js +2 -1
  142. package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -1
  143. package/lib/dist/services/storage/opensearchClient.d.ts +12 -0
  144. package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -1
  145. package/lib/dist/services/storage/opensearchClient.js.map +1 -1
  146. package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -1
  147. package/lib/dist/services/traces/browserRecovery.js +3 -0
  148. package/lib/dist/services/traces/browserRecovery.js.map +1 -1
  149. package/lib/dist/services/traces/index.d.ts +8 -1
  150. package/lib/dist/services/traces/index.d.ts.map +1 -1
  151. package/lib/dist/services/traces/index.js +33 -12
  152. package/lib/dist/services/traces/index.js.map +1 -1
  153. package/lib/dist/services/traces/judgeAgentsHints.d.ts +98 -3
  154. package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -1
  155. package/lib/dist/services/traces/judgeAgentsHints.js +144 -3
  156. package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -1
  157. package/lib/dist/services/traces/spansToTrajectory.js +4 -4
  158. package/lib/dist/services/traces/spansToTrajectory.js.map +1 -1
  159. package/lib/dist/services/traces/tracePoller.d.ts.map +1 -1
  160. package/lib/dist/services/traces/tracePoller.js +17 -20
  161. package/lib/dist/services/traces/tracePoller.js.map +1 -1
  162. package/lib/dist/services/traces/trajectoryMerge.d.ts +79 -0
  163. package/lib/dist/services/traces/trajectoryMerge.d.ts.map +1 -0
  164. package/lib/dist/services/traces/trajectoryMerge.js +109 -0
  165. package/lib/dist/services/traces/trajectoryMerge.js.map +1 -0
  166. package/lib/dist/types/index.d.ts +195 -2
  167. package/lib/dist/types/index.d.ts.map +1 -1
  168. package/lib/dist/types/index.js +24 -0
  169. package/lib/dist/types/index.js.map +1 -1
  170. package/package.json +6 -5
  171. package/server/dist/app.js +3398 -926
  172. package/server/dist/index.js +3401 -929
  173. package/dist/assets/index-BfxtxmKc.css +0 -1
  174. package/dist/assets/index-CrjAfDHu.js +0 -243
@@ -0,0 +1,177 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ const MIN_SWIPE_DISTANCE = 48;
6
+ const HORIZONTAL_SWIPE_RATIO = 1.5;
7
+ const PAGER_DRAG_SLOP = 10;
8
+ const PAGER_EDGE_RESISTANCE = 0.35;
9
+ /**
10
+ * Latch a horizontal pager drag after a small slop, then preserve that latch
11
+ * for the rest of the gesture. Pulling beyond either end receives resistance
12
+ * so the pane communicates the boundary without changing cases.
13
+ *
14
+ * Reduced-motion behavior intentionally stays out of this geometry helper;
15
+ * callers suppress the returned offset while retaining gesture navigation.
16
+ */
17
+ export function computePagerDrag({ dx, dy, latched, atStart, atEnd, }) {
18
+ const nextLatched = latched || (Math.abs(dx) > PAGER_DRAG_SLOP &&
19
+ Math.abs(dx) > Math.abs(dy));
20
+ if (!nextLatched)
21
+ return { offset: 0, latched: false };
22
+ const beyondStart = atStart && dx > 0;
23
+ const beyondEnd = atEnd && dx < 0;
24
+ return {
25
+ offset: (beyondStart || beyondEnd) ? dx * PAGER_EDGE_RESISTANCE : dx,
26
+ latched: true,
27
+ };
28
+ }
29
+ /**
30
+ * Resolve a completed gesture without suppressing native touch movement. A
31
+ * negative x displacement is a left swipe (next); a positive one is previous.
32
+ */
33
+ export function resolveSwipeDirection(dx, dy, touchCount = 1) {
34
+ if (touchCount !== 1)
35
+ return null;
36
+ if (Math.abs(dx) < MIN_SWIPE_DISTANCE)
37
+ return null;
38
+ if (Math.abs(dx) < HORIZONTAL_SWIPE_RATIO * Math.abs(dy))
39
+ return null;
40
+ return dx < 0 ? 'next' : 'prev';
41
+ }
42
+ /** A legacy run with no top-level status is complete once all results are terminal. */
43
+ export function isCompletedVerdictRun(run) {
44
+ if (run.status)
45
+ return run.status === 'completed';
46
+ const results = Object.values(run.results || {});
47
+ return results.length > 0 && results.every(result => result.status !== 'pending' && result.status !== 'running');
48
+ }
49
+ export function getRecentCompletedRuns(runs, limit = 5) {
50
+ return [...runs]
51
+ .filter(isCompletedVerdictRun)
52
+ .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
53
+ .slice(0, limit);
54
+ }
55
+ /**
56
+ * Resolve one case×run cell. A real pass/fail verdict from the report is
57
+ * authoritative and outranks `metricsStatus === 'error'`: metrics/cost
58
+ * enrichment runs after the agent-vs-judge verdict is already known, so a
59
+ * later enrichment failure must not overwrite a genuine pass/fail with
60
+ * 'errored'. `metricsStatus === 'error'` only decides the cell when the
61
+ * report has no real passFailStatus yet.
62
+ */
63
+ export function deriveCaseVerdict(run, testCaseId, reportsById) {
64
+ const result = run.results?.[testCaseId];
65
+ if (!result || result.status === 'pending' || result.status === 'running')
66
+ return 'not-run';
67
+ const report = result.reportId ? reportsById[result.reportId] : undefined;
68
+ if (report?.passFailStatus === 'passed')
69
+ return 'passed';
70
+ if (report?.passFailStatus === 'failed')
71
+ return 'failed';
72
+ if (report?.metricsStatus === 'error')
73
+ return 'errored';
74
+ // New benchmark documents carry this summary. It is a resilience fallback
75
+ // for deleted/legacy report documents, not the primary source of truth.
76
+ if (result.passFailStatus === 'passed')
77
+ return 'passed';
78
+ if (result.passFailStatus === 'failed')
79
+ return 'failed';
80
+ if (result.status === 'failed' || result.status === 'cancelled' || report?.status === 'failed') {
81
+ return 'failed';
82
+ }
83
+ // A completed result without a verdict means the evaluator did not produce
84
+ // a usable score. Keep it visually/semantically separate from agent failure.
85
+ if (result.status === 'completed')
86
+ return 'errored';
87
+ return 'not-run';
88
+ }
89
+ export function deriveCaseVerdicts(runs, testCaseId, reportsById) {
90
+ return runs.map(run => deriveCaseVerdict(run, testCaseId, reportsById));
91
+ }
92
+ /**
93
+ * Buckets are disjoint by priority. Evaluator errors require attention but do
94
+ * not become failed verdicts or enter the pass-rate denominator.
95
+ */
96
+ export function classifyCaseVerdicts(verdicts) {
97
+ const observed = verdicts.filter(verdict => verdict !== 'not-run');
98
+ if (observed.length === 0)
99
+ return 'no-data';
100
+ const latestThree = verdicts.slice(0, 3);
101
+ if (latestThree.some(verdict => verdict === 'failed' || verdict === 'errored')) {
102
+ return 'needs-attention';
103
+ }
104
+ const scored = verdicts.filter(verdict => verdict === 'passed' || verdict === 'failed');
105
+ const hasPass = scored.includes('passed');
106
+ const hasFail = scored.includes('failed');
107
+ if (hasPass && hasFail)
108
+ return 'flaky';
109
+ if (hasPass)
110
+ return 'stable';
111
+ // Sparse histories can put an old failure/error outside the last-three
112
+ // window. They still have data and are not passing, so attention is the only
113
+ // honest bucket.
114
+ return 'needs-attention';
115
+ }
116
+ export function getCasePassRate(verdicts) {
117
+ const passed = verdicts.filter(verdict => verdict === 'passed').length;
118
+ const failed = verdicts.filter(verdict => verdict === 'failed').length;
119
+ return passed + failed === 0 ? null : passed / (passed + failed);
120
+ }
121
+ export function buildCaseReviewRows(testCases, recentRuns, reportsById) {
122
+ return testCases.map(testCase => {
123
+ const verdicts = deriveCaseVerdicts(recentRuns, testCase.id, reportsById);
124
+ return {
125
+ testCase,
126
+ verdicts,
127
+ bucket: classifyCaseVerdicts(verdicts),
128
+ passRate: getCasePassRate(verdicts),
129
+ };
130
+ });
131
+ }
132
+ /** Search/filter first, then pass-rate ascending; no-data sorts last. */
133
+ export function filterAndSortCaseRows(rows, search, filter) {
134
+ const query = search.trim().toLocaleLowerCase();
135
+ return rows
136
+ .filter(row => filter === 'all' || row.bucket === filter)
137
+ .filter(row => !query ||
138
+ row.testCase.name.toLocaleLowerCase().includes(query) ||
139
+ (row.testCase.prompt || '').toLocaleLowerCase().includes(query))
140
+ .sort((a, b) => {
141
+ if (a.passRate === null && b.passRate !== null)
142
+ return 1;
143
+ if (a.passRate !== null && b.passRate === null)
144
+ return -1;
145
+ if (a.passRate !== null && b.passRate !== null && a.passRate !== b.passRate) {
146
+ return a.passRate - b.passRate;
147
+ }
148
+ return a.testCase.name.localeCompare(b.testCase.name);
149
+ });
150
+ }
151
+ export function getCasePagerPosition(rows, selectedCaseId) {
152
+ const index = selectedCaseId ? rows.findIndex(row => row.testCase.id === selectedCaseId) : -1;
153
+ return {
154
+ index,
155
+ position: index >= 0 ? index + 1 : 0,
156
+ total: rows.length,
157
+ previousId: index > 0 ? rows[index - 1].testCase.id : undefined,
158
+ nextId: index >= 0 && index < rows.length - 1 ? rows[index + 1].testCase.id : undefined,
159
+ };
160
+ }
161
+ /**
162
+ * The Cases tab always renders a case's *current* definition, even when the
163
+ * user arrived from a specific historical run (e.g. a heat-strip cell click).
164
+ * This detects that mismatch so callers can show a minimal, honest notice
165
+ * instead of silently presenting a definition the run never actually used.
166
+ * Deliberately does not resurrect the old per-version selector — just a
167
+ * disclosure that the definition may have changed since some of the runs
168
+ * shown here.
169
+ */
170
+ export function hasHistoricalVersionMismatch(currentVersion, runs, testCaseId, reportsById) {
171
+ return runs.some(run => {
172
+ const result = run.results?.[testCaseId];
173
+ const report = result?.reportId ? reportsById[result.reportId] : undefined;
174
+ return typeof report?.testCaseVersion === 'number' && report.testCaseVersion !== currentVersion;
175
+ });
176
+ }
177
+ //# sourceMappingURL=benchmarkCaseReview.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmarkCaseReview.js","sourceRoot":"","sources":["../../benchmarkCaseReview.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAenC;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAC/B,EAAE,EACF,EAAE,EACF,OAAO,EACP,OAAO,EACP,KAAK,GACU;IACf,MAAM,WAAW,GAAG,OAAO,IAAI,CAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe;QAC9B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAC5B,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAEvD,MAAM,WAAW,GAAG,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,KAAK,IAAI,EAAE,GAAG,CAAC,CAAC;IAClC,OAAO;QACL,MAAM,EAAE,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,CAAC,EAAE;QACpE,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,EAAU,EACV,EAAU,EACV,UAAU,GAAG,CAAC;IAEd,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAClC,CAAC;AAmCD,uFAAuF;AACvF,MAAM,UAAU,qBAAqB,CAAC,GAAe;IACnD,IAAI,GAAG,CAAC,MAAM;QAAE,OAAO,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC;IAClD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACjD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAClD,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAC3D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAuB,IAAS,EAAE,KAAK,GAAG,CAAC;IAC/E,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,qBAAqB,CAAC;SAC7B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;SACjF,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,GAAe,EACf,UAAkB,EAClB,WAA6D;IAE7D,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAE5F,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,MAAM,EAAE,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACzD,IAAI,MAAM,EAAE,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACzD,IAAI,MAAM,EAAE,aAAa,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC;IAExD,0EAA0E;IAC1E,wEAAwE;IACxE,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACxD,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAExD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/F,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,2EAA2E;IAC3E,6EAA6E;IAC7E,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;QAAE,OAAO,SAAS,CAAC;IACpD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,IAAkB,EAClB,UAAkB,EAClB,WAA6D;IAE7D,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAuB;IAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;IACnE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE5C,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,SAAS,CAAC,EAAE,CAAC;QAC/E,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,QAAQ,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,OAAO,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IACvC,IAAI,OAAO;QAAE,OAAO,QAAQ,CAAC;IAE7B,uEAAuE;IACvE,6EAA6E;IAC7E,iBAAiB;IACjB,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAuB;IACrD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvE,OAAO,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,SAAc,EACd,UAAwB,EACxB,WAA6D;IAE7D,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;QAC9B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC1E,OAAO;YACL,QAAQ;YACR,QAAQ;YACR,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC;YACtC,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC;SACpC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,qBAAqB,CACnC,IAAwB,EACxB,MAAc,EACd,MAAwB;IAExB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,iBAAiB,EAAE,CAAC;IAChD,OAAO,IAAI;SACR,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC;SACxD,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK;QACnB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;QACrD,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAChE;SACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC5E,OAAO,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QACjC,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAwB,EACxB,cAAkC;IAElC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9F,OAAO;QACL,KAAK;QACL,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;QAC/D,MAAM,EAAE,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;KACxF,CAAC;AACJ,CAAC;AAcD;;;;;;;;GAQG;AACH,MAAM,UAAU,4BAA4B,CAC1C,cAAsB,EACtB,IAAoB,EACpB,UAAkB,EAClB,WAA+D;IAE/D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,OAAO,OAAO,MAAM,EAAE,eAAe,KAAK,QAAQ,IAAI,MAAM,CAAC,eAAe,KAAK,cAAc,CAAC;IAClG,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,109 @@
1
+ /**
2
+ * Pure helpers for the benchmark Runs tab (table + pass-rate-over-time chart).
3
+ *
4
+ * Kept free of React/DOM so the click-to-filter semantics and the chart
5
+ * series derivation can be unit-tested directly. The component
6
+ * (components/evals3/BenchmarkRunsPage.tsx) is a thin renderer over these.
7
+ */
8
+ import type { BenchmarkRun } from '../types/index.js';
9
+ import { getEffectiveRunStatus } from '../lib/runStats.js';
10
+ /** The dimensions a user can click on in the table to filter by. */
11
+ export type RunFilterField = 'agent' | 'model' | 'judge' | 'evaluator' | 'status';
12
+ export interface RunFilter {
13
+ field: RunFilterField;
14
+ /** Raw value (id/key), NOT the display label — labels can collide. */
15
+ value: string;
16
+ /** Human label for the pill. */
17
+ label: string;
18
+ }
19
+ export interface RunTableRow {
20
+ run: BenchmarkRun;
21
+ agentKey: string;
22
+ agentName: string;
23
+ modelId: string;
24
+ modelName: string;
25
+ judgeModelId: string;
26
+ judgeLabel: string;
27
+ evaluatorId: string;
28
+ evaluatorLabel: string;
29
+ status: ReturnType<typeof getEffectiveRunStatus>;
30
+ passed: number;
31
+ failed: number;
32
+ errored: number;
33
+ /** Still to come — only ever non-zero while the run is live. */
34
+ pending: number;
35
+ /** Per-case entries currently executing — only ever non-zero while the run is live. */
36
+ running: number;
37
+ /**
38
+ * Planned test cases that never executed because the run reached a
39
+ * TERMINAL status first (cancelled mid-way, or the executor died). Neither
40
+ * a pass, a fail nor "pending": nothing will ever happen to them, so they
41
+ * must never render with an in-progress spinner.
42
+ */
43
+ notRun: number;
44
+ total: number;
45
+ /**
46
+ * Pass rate over the *evaluable* set (total − errored − pending − running),
47
+ * or null when nothing has been evaluated yet. Percent 0–100.
48
+ */
49
+ passRate: number | null;
50
+ /** Number of test cases in the run (snapshotted size, falling back to results). */
51
+ size: number;
52
+ }
53
+ export interface RowLabelResolvers {
54
+ agentName: (agentKey: string) => string;
55
+ modelName: (modelId: string) => string;
56
+ judgeLabel: (judgeModelId?: string | null) => string;
57
+ evaluatorLabel: (evaluatorId?: string | null) => string;
58
+ }
59
+ export declare function computePassRate(passed: number, failed: number): number | null;
60
+ export declare function buildRunTableRow(run: BenchmarkRun, resolve: RowLabelResolvers): RunTableRow;
61
+ export declare function rowFieldValue(row: RunTableRow, field: RunFilterField): string;
62
+ export declare function rowFieldLabel(row: RunTableRow, field: RunFilterField): string;
63
+ export declare function isSameFilter(a: RunFilter, b: RunFilter): boolean;
64
+ /**
65
+ * Click semantics: clicking a value that is already an active filter removes
66
+ * it (toggle); otherwise it is added. Multiple values on the same field are
67
+ * OR-ed; different fields are AND-ed (see {@link applyRunFilters}).
68
+ */
69
+ export declare function toggleRunFilter(filters: RunFilter[], next: RunFilter): RunFilter[];
70
+ export declare function removeRunFilter(filters: RunFilter[], target: RunFilter): RunFilter[];
71
+ export declare function applyRunFilters(rows: RunTableRow[], filters: RunFilter[]): RunTableRow[];
72
+ export type RunSortField = 'name' | 'agent' | 'model' | 'size' | 'passRate' | 'judge' | 'evaluator' | 'date';
73
+ export interface RunSort {
74
+ field: RunSortField;
75
+ dir: 'asc' | 'desc';
76
+ }
77
+ export declare const DEFAULT_RUN_SORT: RunSort;
78
+ export declare function toggleRunSort(current: RunSort, field: RunSortField): RunSort;
79
+ export declare function sortRunRows(rows: RunTableRow[], sort: RunSort): RunTableRow[];
80
+ export interface PassRatePoint {
81
+ /** Epoch ms of run.createdAt — numeric so the X axis is a real time axis. */
82
+ t: number;
83
+ runId: string;
84
+ runName: string;
85
+ passRate: number;
86
+ passed: number;
87
+ failed: number;
88
+ total: number;
89
+ }
90
+ export interface PassRateSeries {
91
+ /** Series key = agentKey (the sketch groups lines by agent, e.g. CC vs AIS). */
92
+ key: string;
93
+ label: string;
94
+ points: PassRatePoint[];
95
+ }
96
+ /**
97
+ * One line per agent, points ordered by time. Runs without an evaluable
98
+ * result (pass rate null — e.g. still running with nothing judged, or fully
99
+ * errored) are excluded: plotting them as 0% would read as a regression.
100
+ */
101
+ export declare function buildPassRateSeries(rows: RunTableRow[]): PassRateSeries[];
102
+ /** Latest run = max createdAt over the given runs (not "first in the array" —
103
+ * the merged list appends standalone eval-run docs after embedded runs, so
104
+ * array order is not chronological). */
105
+ export declare function latestRunId(runs: Array<Pick<BenchmarkRun, 'id' | 'createdAt'>>): string | null;
106
+ /** Deterministic, distinguishable palette for agent lines (index-based). */
107
+ export declare const SERIES_COLORS: string[];
108
+ export declare function seriesColor(index: number): string;
109
+ //# sourceMappingURL=benchmarkRunsTable.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmarkRunsTable.d.ts","sourceRoot":"","sources":["../../benchmarkRunsTable.ts"],"names":[],"mappings":"AAKA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAmB,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAIxE,oEAAoE;AACpE,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC;AAElF,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,cAAc,CAAC;IACtB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,YAAY,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,mFAAmF;IACnF,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;IACvC,UAAU,EAAE,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC;IACrD,cAAc,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC;CACzD;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAC7B,MAAM,GAAG,IAAI,CAIf;AAgBD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,GAAG,WAAW,CAsC3F;AAID,wBAAgB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,CAQ7E;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,CAQ7E;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,CAEhE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,SAAS,EAAE,CAGlF;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAEpF;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,WAAW,EAAE,CAaxF;AAID,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,GAAG,WAAW,GAAG,MAAM,CAAC;AAC7G,MAAM,WAAW,OAAO;IAAG,KAAK,EAAE,YAAY,CAAC;IAAC,GAAG,EAAE,KAAK,GAAG,MAAM,CAAC;CAAE;AAEtE,eAAO,MAAM,gBAAgB,EAAE,OAAwC,CAAC;AAExE,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAM5E;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,GAAG,WAAW,EAAE,CAuB7E;AAID,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,cAAc,EAAE,CAuBzE;AAED;;yCAEyC;AACzC,wBAAgB,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAQ9F;AAED,4EAA4E;AAC5E,eAAO,MAAM,aAAa,UASzB,CAAC;AAEF,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEjD"}
@@ -0,0 +1,212 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ import { computeRunStats, getEffectiveRunStatus } from '../lib/runStats.js';
6
+ export function computePassRate(passed, failed) {
7
+ const evaluable = passed + failed;
8
+ if (evaluable <= 0)
9
+ return null;
10
+ return Math.round((passed / evaluable) * 1000) / 10;
11
+ }
12
+ /**
13
+ * Run statuses after which no further per-case progress can happen. Only the
14
+ * EXPLICIT persisted status counts: the results-derived fallback in
15
+ * getEffectiveRunStatus reports 'completed' for any status-less legacy run
16
+ * whose observed results all happen to be settled — exactly what an in-flight
17
+ * legacy run looks like between two cases — so it must not gate bucketing
18
+ * (a status-less run therefore keeps the pre-existing pending rendering).
19
+ * Mirrors `TERMINAL_RUN_STATUSES` in the terminal-aware `lib/runStats`
20
+ * (#486); collapse onto `isTerminalRunStatus` from there once it lands.
21
+ */
22
+ function isTerminalStatus(status) {
23
+ return status === 'completed' || status === 'failed' || status === 'cancelled';
24
+ }
25
+ export function buildRunTableRow(run, resolve) {
26
+ const status = getEffectiveRunStatus(run);
27
+ const terminal = isTerminalStatus(run.status);
28
+ const { passed, failed, errored, total } = computeRunStats(run);
29
+ const settled = passed + failed + errored;
30
+ // Bug (owner report, 2026-09-08): a run cancelled at 43/62 rendered
31
+ // "11/32 /19 ⟳" — the never-started remainder (planned − observed) was
32
+ // re-derived as `pending` here regardless of the run's TERMINAL status, so
33
+ // every cancelled/failed run on the benchmark page looked in progress
34
+ // forever. Once the run is terminal nothing can start those cases (nor
35
+ // finish a per-case entry an executor left `running` when it died), so
36
+ // the remainder is `notRun`, and pending/running are zero by definition.
37
+ let running = 0;
38
+ if (!terminal) {
39
+ Object.values(run.results || {}).forEach(r => { if (r.status === 'running')
40
+ running++; });
41
+ }
42
+ const notRun = terminal ? Math.max(0, total - settled) : 0;
43
+ const pending = terminal ? 0 : Math.max(0, total - settled - running);
44
+ const size = run.testCaseSnapshots?.length || total;
45
+ const agentKey = run.agentKey || '';
46
+ const modelId = run.modelId || '';
47
+ const judgeModelId = run.judgeModelId || '';
48
+ const evaluatorId = run.evaluatorId || '';
49
+ return {
50
+ run,
51
+ agentKey,
52
+ agentName: resolve.agentName(agentKey),
53
+ modelId,
54
+ modelName: resolve.modelName(modelId),
55
+ judgeModelId,
56
+ judgeLabel: resolve.judgeLabel(judgeModelId || undefined),
57
+ evaluatorId,
58
+ evaluatorLabel: resolve.evaluatorLabel(evaluatorId || undefined),
59
+ status,
60
+ passed, failed, errored, pending, running, notRun, total,
61
+ passRate: computePassRate(passed, failed),
62
+ size,
63
+ };
64
+ }
65
+ // ─── Filters ─────────────────────────────────────────────────────────────────
66
+ export function rowFieldValue(row, field) {
67
+ switch (field) {
68
+ case 'agent': return row.agentKey;
69
+ case 'model': return row.modelId;
70
+ case 'judge': return row.judgeModelId;
71
+ case 'evaluator': return row.evaluatorId;
72
+ case 'status': return row.status;
73
+ }
74
+ }
75
+ export function rowFieldLabel(row, field) {
76
+ switch (field) {
77
+ case 'agent': return row.agentName;
78
+ case 'model': return row.modelName;
79
+ case 'judge': return row.judgeLabel;
80
+ case 'evaluator': return row.evaluatorLabel;
81
+ case 'status': return row.status;
82
+ }
83
+ }
84
+ export function isSameFilter(a, b) {
85
+ return a.field === b.field && a.value === b.value;
86
+ }
87
+ /**
88
+ * Click semantics: clicking a value that is already an active filter removes
89
+ * it (toggle); otherwise it is added. Multiple values on the same field are
90
+ * OR-ed; different fields are AND-ed (see {@link applyRunFilters}).
91
+ */
92
+ export function toggleRunFilter(filters, next) {
93
+ const exists = filters.some(f => isSameFilter(f, next));
94
+ return exists ? filters.filter(f => !isSameFilter(f, next)) : [...filters, next];
95
+ }
96
+ export function removeRunFilter(filters, target) {
97
+ return filters.filter(f => !isSameFilter(f, target));
98
+ }
99
+ export function applyRunFilters(rows, filters) {
100
+ if (filters.length === 0)
101
+ return rows;
102
+ const byField = new Map();
103
+ for (const f of filters) {
104
+ if (!byField.has(f.field))
105
+ byField.set(f.field, new Set());
106
+ byField.get(f.field).add(f.value);
107
+ }
108
+ return rows.filter(row => {
109
+ for (const [field, values] of byField) {
110
+ if (!values.has(rowFieldValue(row, field)))
111
+ return false;
112
+ }
113
+ return true;
114
+ });
115
+ }
116
+ export const DEFAULT_RUN_SORT = { field: 'date', dir: 'desc' };
117
+ export function toggleRunSort(current, field) {
118
+ if (current.field === field)
119
+ return { field, dir: current.dir === 'asc' ? 'desc' : 'asc' };
120
+ // Numeric/date columns default to descending (biggest/newest first);
121
+ // text columns to ascending.
122
+ const numeric = ['size', 'passRate', 'date'];
123
+ return { field, dir: numeric.includes(field) ? 'desc' : 'asc' };
124
+ }
125
+ export function sortRunRows(rows, sort) {
126
+ const dir = sort.dir === 'asc' ? 1 : -1;
127
+ const cmpNum = (a, b) => {
128
+ // nulls always sink to the bottom regardless of direction
129
+ if (a === null && b === null)
130
+ return 0;
131
+ if (a === null)
132
+ return 1;
133
+ if (b === null)
134
+ return -1;
135
+ return dir * (a - b);
136
+ };
137
+ return [...rows].sort((a, b) => {
138
+ switch (sort.field) {
139
+ case 'name': return dir * a.run.name.localeCompare(b.run.name);
140
+ case 'agent': return dir * a.agentName.localeCompare(b.agentName);
141
+ case 'model': return dir * a.modelName.localeCompare(b.modelName);
142
+ case 'judge': return dir * a.judgeLabel.localeCompare(b.judgeLabel);
143
+ case 'evaluator': return dir * a.evaluatorLabel.localeCompare(b.evaluatorLabel);
144
+ case 'size': return cmpNum(a.size, b.size);
145
+ case 'passRate': return cmpNum(a.passRate, b.passRate);
146
+ case 'date':
147
+ default:
148
+ return dir * (new Date(a.run.createdAt).getTime() - new Date(b.run.createdAt).getTime());
149
+ }
150
+ });
151
+ }
152
+ /**
153
+ * One line per agent, points ordered by time. Runs without an evaluable
154
+ * result (pass rate null — e.g. still running with nothing judged, or fully
155
+ * errored) are excluded: plotting them as 0% would read as a regression.
156
+ */
157
+ export function buildPassRateSeries(rows) {
158
+ const byAgent = new Map();
159
+ for (const row of rows) {
160
+ if (row.passRate === null)
161
+ continue;
162
+ const t = new Date(row.run.createdAt).getTime();
163
+ if (Number.isNaN(t))
164
+ continue;
165
+ let series = byAgent.get(row.agentKey);
166
+ if (!series) {
167
+ series = { key: row.agentKey, label: row.agentName, points: [] };
168
+ byAgent.set(row.agentKey, series);
169
+ }
170
+ series.points.push({
171
+ t, runId: row.run.id, runName: row.run.name,
172
+ passRate: row.passRate, passed: row.passed, failed: row.failed, total: row.total,
173
+ });
174
+ }
175
+ const out = [...byAgent.values()];
176
+ for (const s of out)
177
+ s.points.sort((a, b) => a.t - b.t);
178
+ // Order by label (then key) — NOT by point count. Series index drives the
179
+ // line colour, so the order must be stable while the page polls every few
180
+ // seconds and new runs land; "busiest first" would recolour agents mid-run.
181
+ out.sort((a, b) => a.label.localeCompare(b.label) || a.key.localeCompare(b.key));
182
+ return out;
183
+ }
184
+ /** Latest run = max createdAt over the given runs (not "first in the array" —
185
+ * the merged list appends standalone eval-run docs after embedded runs, so
186
+ * array order is not chronological). */
187
+ export function latestRunId(runs) {
188
+ let best = null;
189
+ for (const r of runs) {
190
+ const t = new Date(r.createdAt).getTime();
191
+ if (Number.isNaN(t))
192
+ continue;
193
+ if (!best || t > best.t)
194
+ best = { id: r.id, t };
195
+ }
196
+ return best?.id ?? null;
197
+ }
198
+ /** Deterministic, distinguishable palette for agent lines (index-based). */
199
+ export const SERIES_COLORS = [
200
+ '#3b82f6', // blue-500
201
+ '#f59e0b', // amber-500
202
+ '#10b981', // emerald-500
203
+ '#8b5cf6', // violet-500
204
+ '#ef4444', // red-500
205
+ '#06b6d4', // cyan-500
206
+ '#ec4899', // pink-500
207
+ '#84cc16', // lime-500
208
+ ];
209
+ export function seriesColor(index) {
210
+ return SERIES_COLORS[index % SERIES_COLORS.length];
211
+ }
212
+ //# sourceMappingURL=benchmarkRunsTable.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmarkRunsTable.js","sourceRoot":"","sources":["../../benchmarkRunsTable.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAyDxE,MAAM,UAAU,eAAe,CAC7B,MAAc,EAAE,MAAc;IAE9B,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAClC,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AACtD,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,gBAAgB,CAAC,MAA8B;IACtD,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,WAAW,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAiB,EAAE,OAA0B;IAC5E,MAAM,MAAM,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IAChE,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC1C,oEAAoE;IACpE,uEAAuE;IACvE,2EAA2E;IAC3E,sEAAsE;IACtE,uEAAuE;IACvE,uEAAuE;IACvE,yEAAyE;IACzE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,GAAG,CAAC,iBAAiB,EAAE,MAAM,IAAI,KAAK,CAAC;IACpD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAClC,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;IAC1C,OAAO;QACL,GAAG;QACH,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC;QACtC,OAAO;QACP,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;QACrC,YAAY;QACZ,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,YAAY,IAAI,SAAS,CAAC;QACzD,WAAW;QACX,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI,SAAS,CAAC;QAChE,MAAM;QACN,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;QACxD,QAAQ,EAAE,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC;QACzC,IAAI;KACL,CAAC;AACJ,CAAC;AAED,gFAAgF;AAEhF,MAAM,UAAU,aAAa,CAAC,GAAgB,EAAE,KAAqB;IACnE,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,QAAQ,CAAC;QAClC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC;QACjC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,YAAY,CAAC;QACtC,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,CAAC,WAAW,CAAC;QACzC,KAAK,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAgB,EAAE,KAAqB;IACnE,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC;QACnC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC;QACnC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,UAAU,CAAC;QACpC,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,CAAC,cAAc,CAAC;QAC5C,KAAK,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAY,EAAE,CAAY;IACrD,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC;AACpD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAAoB,EAAE,IAAe;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAoB,EAAE,MAAiB;IACrE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAmB,EAAE,OAAoB;IACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;QACvB,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAOD,MAAM,CAAC,MAAM,gBAAgB,GAAY,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AAExE,MAAM,UAAU,aAAa,CAAC,OAAgB,EAAE,KAAmB;IACjE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;QAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3F,qEAAqE;IACrE,6BAA6B;IAC7B,MAAM,OAAO,GAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAC7D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAmB,EAAE,IAAa;IAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,CAAgB,EAAE,CAAgB,EAAE,EAAE;QACpD,0DAA0D;QAC1D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,CAAC,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7B,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,KAAK,MAAM,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC/D,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAClE,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAClE,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACpE,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YAChF,KAAK,MAAM,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3C,KAAK,UAAU,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;YACvD,KAAK,MAAM,CAAC;YACZ;gBACE,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAsBD;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAmB;IACrD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI;YAAE,SAAS;QACpC,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QAC9B,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI;YAC3C,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK;SACjF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,GAAG;QAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,0EAA0E;IAC1E,0EAA0E;IAC1E,4EAA4E;IAC5E,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;yCAEyC;AACzC,MAAM,UAAU,WAAW,CAAC,IAAmD;IAC7E,IAAI,IAAI,GAAqC,IAAI,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QAC9B,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YAAE,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAClD,CAAC;IACD,OAAO,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,YAAY;IACvB,SAAS,EAAE,cAAc;IACzB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,UAAU;IACrB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,OAAO,aAAa,CAAC,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;AACrD,CAAC"}
@@ -54,10 +54,49 @@ export declare const OTHER_CATEGORY = "(other)";
54
54
  * `subcategory` as a dedicated concept).
55
55
  * 4. {@link UNCATEGORIZED}.
56
56
  * (`category:<x>` labels are intentionally NOT used here: imported benchmarks
57
- * stamp a single `category:RAG` on every case, which would collapse the
58
- * breakdown into one column — that's the coarse domain, not this facet.)
57
+ * commonly stamp a single `category:RAG` on every case, which would collapse
58
+ * the breakdown into one column — that's the coarse domain, not this facet.
59
+ * {@link categoryLabelIsUsableFallback} + {@link extractRowCategoryEffective}
60
+ * add `category:` back in, but ONLY for the comparisons where it's actually
61
+ * informative — see their docs.)
59
62
  */
60
63
  export declare function extractRowCategory(row: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>): string;
64
+ /**
65
+ * Whether `category:` should be used as a LAST-RESORT fallback facet for
66
+ * this set of rows. True only when, among the rows {@link extractRowCategory}
67
+ * leaves as {@link UNCATEGORIZED} (rows with a real `[bracket]`/`topic:`
68
+ * facet keep using that, whether or not `category:` also varies — one
69
+ * differently-tagged row must not disable the fallback for every OTHER row
70
+ * that has nothing better):
71
+ * 1. There's at least one such row.
72
+ * 2. The `category:` label, normalized the same way {@link
73
+ * extractRowCategoryEffective} returns it (lowercased, so
74
+ * `category:RAG` and `category:rag` count as ONE value, not two —
75
+ * inconsistent casing must not look like variance), has ≥2 distinct
76
+ * values that EACH appear on ≥2 rows. The 2-per-value floor keeps a
77
+ * single stray/typo'd label from flipping this on for what is really a
78
+ * uniform category with one dirty row — that must still resolve to
79
+ * "nothing to facet by", not a real-facet-plus-a-noise-column matrix.
80
+ * A uniformly-stamped `category:RAG` (the case the original "intentionally
81
+ * NOT used" heuristic was written for) stays excluded — falling back to it
82
+ * would add a redundant single-value column to every such comparison,
83
+ * exactly the noise that heuristic was trying to avoid. WixQA-400's
84
+ * `category:expertwritten|simulated` (200/200) is the motivating case where
85
+ * it DOES vary (both values well above the floor) and is genuinely useful.
86
+ * Callers that resolve a per-row category for a set of rows should compute
87
+ * this ONCE for the whole set and pass it to every {@link
88
+ * extractRowCategoryEffective} call, so the matrix, its shared-weakness
89
+ * callout, and the table's click-to-filter all agree on the same facet.
90
+ */
91
+ export declare function categoryLabelIsUsableFallback(rows: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>[]): boolean;
92
+ /**
93
+ * {@link extractRowCategory}, extended with the `category:` fallback when
94
+ * {@link categoryLabelIsUsableFallback} (computed once per row-set, passed
95
+ * in as `useCategoryFallback`) says it's actually informative for this
96
+ * comparison. Only reached when the base extraction is {@link
97
+ * UNCATEGORIZED} — a real bracket/topic facet always wins.
98
+ */
99
+ export declare function extractRowCategoryEffective(row: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>, useCategoryFallback: boolean): string;
61
100
  export interface CategoryCell {
62
101
  passed: number;
63
102
  total: number;
@@ -76,6 +115,14 @@ export interface CategoryBreakdown {
76
115
  * the rows its cell counted.
77
116
  */
78
117
  members: Record<string, string[]>;
118
+ /**
119
+ * Whether this breakdown fell back to the `category:` label (see {@link
120
+ * categoryLabelIsUsableFallback}). Callers that need to resolve a row's
121
+ * category OUTSIDE this function (e.g. {@link detectSharedWeakness}, or
122
+ * the compare page's click-to-filter) must pass this to {@link
123
+ * extractRowCategoryEffective} to stay consistent with what's displayed.
124
+ */
125
+ usesCategoryFallback: boolean;
79
126
  }
80
127
  /**
81
128
  * Default minimum case count for a category to get its own column; smaller
@@ -1 +1 @@
1
- {"version":3,"file":"comparisonInsights.d.ts","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAGrD,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6EAA6E;IAC7E,SAAS,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAyBD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,kBAAkB,CAgBpB;AAED,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,IAAI,CAO9F;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAC/C,wEAAwE;AACxE,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,GAAG,MAAM,CAQtG;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACrD,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACnC;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,kFAAkF;AAClF,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,GAAE,MAA2B,GACpC,iBAAiB,CAuCnB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,iBAAiB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,cAAc,GAAG,IAAI,CAuCvB"}
1
+ {"version":3,"file":"comparisonInsights.d.ts","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAGrD,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6EAA6E;IAC7E,SAAS,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAyBD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,kBAAkB,CAgBpB;AAED,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,IAAI,CAO9F;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAC/C,wEAAwE;AACxE,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,GAAG,MAAM,CAQtG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,EAAE,GAC7D,OAAO,CAUT;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,EAC3D,mBAAmB,EAAE,OAAO,GAC3B,MAAM,CAKR;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACrD,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAClC;;;;;;OAMG;IACH,oBAAoB,EAAE,OAAO,CAAC;CAC/B;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,kFAAkF;AAClF,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,GAAE,MAA2B,GACpC,iBAAiB,CAwCnB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,iBAAiB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,cAAc,GAAG,IAAI,CAuCvB"}