@opensearch-project/agent-health 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/dist/index.js +722 -330
- package/dist/assets/index-D-Np_l_T.js +246 -0
- package/dist/assets/index-vZt9QZKf.css +1 -0
- package/dist/index.html +2 -2
- package/docs/CLI.md +138 -1
- package/docs/CONNECTORS.md +1 -1
- package/docs/SDK.md +126 -2
- package/docs/skills/add-connector/SKILL.md +5 -1
- package/lib/dist/lib/agentTrends.d.ts +210 -0
- package/lib/dist/lib/agentTrends.d.ts.map +1 -0
- package/lib/dist/lib/agentTrends.js +360 -0
- package/lib/dist/lib/agentTrends.js.map +1 -0
- package/lib/dist/lib/benchmarkCaseReview.d.ts +114 -0
- package/lib/dist/lib/benchmarkCaseReview.d.ts.map +1 -0
- package/lib/dist/lib/benchmarkCaseReview.js +177 -0
- package/lib/dist/lib/benchmarkCaseReview.js.map +1 -0
- package/lib/dist/lib/benchmarkRunsTable.d.ts +109 -0
- package/lib/dist/lib/benchmarkRunsTable.d.ts.map +1 -0
- package/lib/dist/lib/benchmarkRunsTable.js +212 -0
- package/lib/dist/lib/benchmarkRunsTable.js.map +1 -0
- package/lib/dist/lib/comparisonInsights.d.ts +49 -2
- package/lib/dist/lib/comparisonInsights.d.ts.map +1 -1
- package/lib/dist/lib/comparisonInsights.js +65 -7
- package/lib/dist/lib/comparisonInsights.js.map +1 -1
- package/lib/dist/lib/config/loader.d.ts.map +1 -1
- package/lib/dist/lib/config/loader.js +11 -1
- package/lib/dist/lib/config/loader.js.map +1 -1
- package/lib/dist/lib/dashboardMetrics.d.ts +11 -2
- package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -1
- package/lib/dist/lib/dashboardMetrics.js +38 -3
- package/lib/dist/lib/dashboardMetrics.js.map +1 -1
- package/lib/dist/lib/evaluationRerun.d.ts +39 -0
- package/lib/dist/lib/evaluationRerun.d.ts.map +1 -1
- package/lib/dist/lib/evaluationRerun.js +49 -0
- package/lib/dist/lib/evaluationRerun.js.map +1 -1
- package/lib/dist/lib/judgeFailureSummary.d.ts +66 -0
- package/lib/dist/lib/judgeFailureSummary.d.ts.map +1 -0
- package/lib/dist/lib/judgeFailureSummary.js +68 -0
- package/lib/dist/lib/judgeFailureSummary.js.map +1 -0
- package/lib/dist/lib/judgeStrategies.d.ts +108 -0
- package/lib/dist/lib/judgeStrategies.d.ts.map +1 -0
- package/lib/dist/lib/judgeStrategies.js +135 -0
- package/lib/dist/lib/judgeStrategies.js.map +1 -0
- package/lib/dist/lib/matchers/expect.d.ts +21 -1
- package/lib/dist/lib/matchers/expect.d.ts.map +1 -1
- package/lib/dist/lib/matchers/expect.js +51 -0
- package/lib/dist/lib/matchers/expect.js.map +1 -1
- package/lib/dist/lib/matchers/judgeAccessor.d.ts +4 -0
- package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -1
- package/lib/dist/lib/matchers/judgeAccessor.js +13 -2
- package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -1
- package/lib/dist/lib/matchers/judgeReasoningParse.d.ts +49 -0
- package/lib/dist/lib/matchers/judgeReasoningParse.d.ts.map +1 -0
- package/lib/dist/lib/matchers/judgeReasoningParse.js +128 -0
- package/lib/dist/lib/matchers/judgeReasoningParse.js.map +1 -0
- package/lib/dist/lib/matchers/types.d.ts +25 -0
- package/lib/dist/lib/matchers/types.d.ts.map +1 -1
- package/lib/dist/lib/resolveCanonicalRun.d.ts +23 -0
- package/lib/dist/lib/resolveCanonicalRun.d.ts.map +1 -0
- package/lib/dist/lib/resolveCanonicalRun.js +26 -0
- package/lib/dist/lib/resolveCanonicalRun.js.map +1 -0
- package/lib/dist/lib/runActions.d.ts +120 -0
- package/lib/dist/lib/runActions.d.ts.map +1 -0
- package/lib/dist/lib/runActions.js +130 -0
- package/lib/dist/lib/runActions.js.map +1 -0
- package/lib/dist/lib/runInsights.d.ts +86 -0
- package/lib/dist/lib/runInsights.d.ts.map +1 -0
- package/lib/dist/lib/runInsights.js +185 -0
- package/lib/dist/lib/runInsights.js.map +1 -0
- package/lib/dist/lib/runName.d.ts +29 -0
- package/lib/dist/lib/runName.d.ts.map +1 -0
- package/lib/dist/lib/runName.js +38 -0
- package/lib/dist/lib/runName.js.map +1 -0
- package/lib/dist/lib/runReportPath.d.ts +16 -0
- package/lib/dist/lib/runReportPath.d.ts.map +1 -0
- package/lib/dist/lib/runReportPath.js +22 -0
- package/lib/dist/lib/runReportPath.js.map +1 -0
- package/lib/dist/lib/runSort.d.ts +27 -0
- package/lib/dist/lib/runSort.d.ts.map +1 -0
- package/lib/dist/lib/runSort.js +31 -0
- package/lib/dist/lib/runSort.js.map +1 -0
- package/lib/dist/lib/runStats.d.ts +86 -6
- package/lib/dist/lib/runStats.d.ts.map +1 -1
- package/lib/dist/lib/runStats.js +170 -19
- package/lib/dist/lib/runStats.js.map +1 -1
- package/lib/dist/lib/testCases/define.d.ts.map +1 -1
- package/lib/dist/lib/testCases/define.js +93 -46
- package/lib/dist/lib/testCases/define.js.map +1 -1
- package/lib/dist/lib/testCases/judge.d.ts.map +1 -1
- package/lib/dist/lib/testCases/judge.js +22 -4
- package/lib/dist/lib/testCases/judge.js.map +1 -1
- package/lib/dist/lib/testCases/loader.d.ts +24 -0
- package/lib/dist/lib/testCases/loader.d.ts.map +1 -1
- package/lib/dist/lib/testCases/loader.js +253 -36
- package/lib/dist/lib/testCases/loader.js.map +1 -1
- package/lib/dist/lib/trajectoryStepDisplay.d.ts +25 -0
- package/lib/dist/lib/trajectoryStepDisplay.d.ts.map +1 -0
- package/lib/dist/lib/trajectoryStepDisplay.js +42 -0
- package/lib/dist/lib/trajectoryStepDisplay.js.map +1 -0
- package/lib/dist/lib/utils.d.ts +19 -0
- package/lib/dist/lib/utils.d.ts.map +1 -1
- package/lib/dist/lib/utils.js +27 -0
- package/lib/dist/lib/utils.js.map +1 -1
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +68 -32
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -1
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +148 -125
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -1
- package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +21 -13
- package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -1
- package/lib/dist/services/connectors/kiro/KiroConnector.js +22 -25
- package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -1
- package/lib/dist/services/connectors/pi/PiConnector.d.ts +49 -10
- package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -1
- package/lib/dist/services/connectors/pi/PiConnector.js +102 -86
- package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -1
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +59 -14
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -1
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +87 -61
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -1
- package/lib/dist/services/evaluation/bedrockJudge.d.ts +11 -0
- package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -1
- package/lib/dist/services/evaluation/bedrockJudge.js +2 -0
- package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -1
- package/lib/dist/services/evaluation/index.d.ts +18 -1
- package/lib/dist/services/evaluation/index.d.ts.map +1 -1
- package/lib/dist/services/evaluation/index.js +156 -19
- package/lib/dist/services/evaluation/index.js.map +1 -1
- package/lib/dist/services/metrics.d.ts +55 -0
- package/lib/dist/services/metrics.d.ts.map +1 -0
- package/lib/dist/services/metrics.js +89 -0
- package/lib/dist/services/metrics.js.map +1 -0
- package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -1
- package/lib/dist/services/storage/asyncBenchmarkStorage.js +16 -0
- package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -1
- package/lib/dist/services/storage/asyncRunStorage.d.ts +11 -0
- package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -1
- package/lib/dist/services/storage/asyncRunStorage.js +49 -2
- package/lib/dist/services/storage/asyncRunStorage.js.map +1 -1
- package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +1 -1
- package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -1
- package/lib/dist/services/storage/asyncTestCaseStorage.js +2 -1
- package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -1
- package/lib/dist/services/storage/opensearchClient.d.ts +12 -0
- package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -1
- package/lib/dist/services/storage/opensearchClient.js.map +1 -1
- package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -1
- package/lib/dist/services/traces/browserRecovery.js +3 -0
- package/lib/dist/services/traces/browserRecovery.js.map +1 -1
- package/lib/dist/services/traces/index.d.ts +8 -1
- package/lib/dist/services/traces/index.d.ts.map +1 -1
- package/lib/dist/services/traces/index.js +33 -12
- package/lib/dist/services/traces/index.js.map +1 -1
- package/lib/dist/services/traces/judgeAgentsHints.d.ts +98 -3
- package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -1
- package/lib/dist/services/traces/judgeAgentsHints.js +144 -3
- package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -1
- package/lib/dist/services/traces/spansToTrajectory.js +4 -4
- package/lib/dist/services/traces/spansToTrajectory.js.map +1 -1
- package/lib/dist/services/traces/tracePoller.d.ts.map +1 -1
- package/lib/dist/services/traces/tracePoller.js +17 -20
- package/lib/dist/services/traces/tracePoller.js.map +1 -1
- package/lib/dist/services/traces/trajectoryMerge.d.ts +79 -0
- package/lib/dist/services/traces/trajectoryMerge.d.ts.map +1 -0
- package/lib/dist/services/traces/trajectoryMerge.js +109 -0
- package/lib/dist/services/traces/trajectoryMerge.js.map +1 -0
- package/lib/dist/types/index.d.ts +195 -2
- package/lib/dist/types/index.d.ts.map +1 -1
- package/lib/dist/types/index.js +24 -0
- package/lib/dist/types/index.js.map +1 -1
- package/package.json +6 -5
- package/server/dist/app.js +3398 -926
- package/server/dist/index.js +3401 -929
- package/dist/assets/index-BfxtxmKc.css +0 -1
- package/dist/assets/index-CrjAfDHu.js +0 -243
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright OpenSearch Contributors
|
|
3
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
*/
|
|
5
|
+
const MIN_SWIPE_DISTANCE = 48;
|
|
6
|
+
const HORIZONTAL_SWIPE_RATIO = 1.5;
|
|
7
|
+
const PAGER_DRAG_SLOP = 10;
|
|
8
|
+
const PAGER_EDGE_RESISTANCE = 0.35;
|
|
9
|
+
/**
|
|
10
|
+
* Latch a horizontal pager drag after a small slop, then preserve that latch
|
|
11
|
+
* for the rest of the gesture. Pulling beyond either end receives resistance
|
|
12
|
+
* so the pane communicates the boundary without changing cases.
|
|
13
|
+
*
|
|
14
|
+
* Reduced-motion behavior intentionally stays out of this geometry helper;
|
|
15
|
+
* callers suppress the returned offset while retaining gesture navigation.
|
|
16
|
+
*/
|
|
17
|
+
export function computePagerDrag({ dx, dy, latched, atStart, atEnd, }) {
|
|
18
|
+
const nextLatched = latched || (Math.abs(dx) > PAGER_DRAG_SLOP &&
|
|
19
|
+
Math.abs(dx) > Math.abs(dy));
|
|
20
|
+
if (!nextLatched)
|
|
21
|
+
return { offset: 0, latched: false };
|
|
22
|
+
const beyondStart = atStart && dx > 0;
|
|
23
|
+
const beyondEnd = atEnd && dx < 0;
|
|
24
|
+
return {
|
|
25
|
+
offset: (beyondStart || beyondEnd) ? dx * PAGER_EDGE_RESISTANCE : dx,
|
|
26
|
+
latched: true,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Resolve a completed gesture without suppressing native touch movement. A
|
|
31
|
+
* negative x displacement is a left swipe (next); a positive one is previous.
|
|
32
|
+
*/
|
|
33
|
+
export function resolveSwipeDirection(dx, dy, touchCount = 1) {
|
|
34
|
+
if (touchCount !== 1)
|
|
35
|
+
return null;
|
|
36
|
+
if (Math.abs(dx) < MIN_SWIPE_DISTANCE)
|
|
37
|
+
return null;
|
|
38
|
+
if (Math.abs(dx) < HORIZONTAL_SWIPE_RATIO * Math.abs(dy))
|
|
39
|
+
return null;
|
|
40
|
+
return dx < 0 ? 'next' : 'prev';
|
|
41
|
+
}
|
|
42
|
+
/** A legacy run with no top-level status is complete once all results are terminal. */
|
|
43
|
+
export function isCompletedVerdictRun(run) {
|
|
44
|
+
if (run.status)
|
|
45
|
+
return run.status === 'completed';
|
|
46
|
+
const results = Object.values(run.results || {});
|
|
47
|
+
return results.length > 0 && results.every(result => result.status !== 'pending' && result.status !== 'running');
|
|
48
|
+
}
|
|
49
|
+
export function getRecentCompletedRuns(runs, limit = 5) {
|
|
50
|
+
return [...runs]
|
|
51
|
+
.filter(isCompletedVerdictRun)
|
|
52
|
+
.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
|
|
53
|
+
.slice(0, limit);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Resolve one case×run cell. A real pass/fail verdict from the report is
|
|
57
|
+
* authoritative and outranks `metricsStatus === 'error'`: metrics/cost
|
|
58
|
+
* enrichment runs after the agent-vs-judge verdict is already known, so a
|
|
59
|
+
* later enrichment failure must not overwrite a genuine pass/fail with
|
|
60
|
+
* 'errored'. `metricsStatus === 'error'` only decides the cell when the
|
|
61
|
+
* report has no real passFailStatus yet.
|
|
62
|
+
*/
|
|
63
|
+
export function deriveCaseVerdict(run, testCaseId, reportsById) {
|
|
64
|
+
const result = run.results?.[testCaseId];
|
|
65
|
+
if (!result || result.status === 'pending' || result.status === 'running')
|
|
66
|
+
return 'not-run';
|
|
67
|
+
const report = result.reportId ? reportsById[result.reportId] : undefined;
|
|
68
|
+
if (report?.passFailStatus === 'passed')
|
|
69
|
+
return 'passed';
|
|
70
|
+
if (report?.passFailStatus === 'failed')
|
|
71
|
+
return 'failed';
|
|
72
|
+
if (report?.metricsStatus === 'error')
|
|
73
|
+
return 'errored';
|
|
74
|
+
// New benchmark documents carry this summary. It is a resilience fallback
|
|
75
|
+
// for deleted/legacy report documents, not the primary source of truth.
|
|
76
|
+
if (result.passFailStatus === 'passed')
|
|
77
|
+
return 'passed';
|
|
78
|
+
if (result.passFailStatus === 'failed')
|
|
79
|
+
return 'failed';
|
|
80
|
+
if (result.status === 'failed' || result.status === 'cancelled' || report?.status === 'failed') {
|
|
81
|
+
return 'failed';
|
|
82
|
+
}
|
|
83
|
+
// A completed result without a verdict means the evaluator did not produce
|
|
84
|
+
// a usable score. Keep it visually/semantically separate from agent failure.
|
|
85
|
+
if (result.status === 'completed')
|
|
86
|
+
return 'errored';
|
|
87
|
+
return 'not-run';
|
|
88
|
+
}
|
|
89
|
+
export function deriveCaseVerdicts(runs, testCaseId, reportsById) {
|
|
90
|
+
return runs.map(run => deriveCaseVerdict(run, testCaseId, reportsById));
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Buckets are disjoint by priority. Evaluator errors require attention but do
|
|
94
|
+
* not become failed verdicts or enter the pass-rate denominator.
|
|
95
|
+
*/
|
|
96
|
+
export function classifyCaseVerdicts(verdicts) {
|
|
97
|
+
const observed = verdicts.filter(verdict => verdict !== 'not-run');
|
|
98
|
+
if (observed.length === 0)
|
|
99
|
+
return 'no-data';
|
|
100
|
+
const latestThree = verdicts.slice(0, 3);
|
|
101
|
+
if (latestThree.some(verdict => verdict === 'failed' || verdict === 'errored')) {
|
|
102
|
+
return 'needs-attention';
|
|
103
|
+
}
|
|
104
|
+
const scored = verdicts.filter(verdict => verdict === 'passed' || verdict === 'failed');
|
|
105
|
+
const hasPass = scored.includes('passed');
|
|
106
|
+
const hasFail = scored.includes('failed');
|
|
107
|
+
if (hasPass && hasFail)
|
|
108
|
+
return 'flaky';
|
|
109
|
+
if (hasPass)
|
|
110
|
+
return 'stable';
|
|
111
|
+
// Sparse histories can put an old failure/error outside the last-three
|
|
112
|
+
// window. They still have data and are not passing, so attention is the only
|
|
113
|
+
// honest bucket.
|
|
114
|
+
return 'needs-attention';
|
|
115
|
+
}
|
|
116
|
+
export function getCasePassRate(verdicts) {
|
|
117
|
+
const passed = verdicts.filter(verdict => verdict === 'passed').length;
|
|
118
|
+
const failed = verdicts.filter(verdict => verdict === 'failed').length;
|
|
119
|
+
return passed + failed === 0 ? null : passed / (passed + failed);
|
|
120
|
+
}
|
|
121
|
+
export function buildCaseReviewRows(testCases, recentRuns, reportsById) {
|
|
122
|
+
return testCases.map(testCase => {
|
|
123
|
+
const verdicts = deriveCaseVerdicts(recentRuns, testCase.id, reportsById);
|
|
124
|
+
return {
|
|
125
|
+
testCase,
|
|
126
|
+
verdicts,
|
|
127
|
+
bucket: classifyCaseVerdicts(verdicts),
|
|
128
|
+
passRate: getCasePassRate(verdicts),
|
|
129
|
+
};
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
/** Search/filter first, then pass-rate ascending; no-data sorts last. */
|
|
133
|
+
export function filterAndSortCaseRows(rows, search, filter) {
|
|
134
|
+
const query = search.trim().toLocaleLowerCase();
|
|
135
|
+
return rows
|
|
136
|
+
.filter(row => filter === 'all' || row.bucket === filter)
|
|
137
|
+
.filter(row => !query ||
|
|
138
|
+
row.testCase.name.toLocaleLowerCase().includes(query) ||
|
|
139
|
+
(row.testCase.prompt || '').toLocaleLowerCase().includes(query))
|
|
140
|
+
.sort((a, b) => {
|
|
141
|
+
if (a.passRate === null && b.passRate !== null)
|
|
142
|
+
return 1;
|
|
143
|
+
if (a.passRate !== null && b.passRate === null)
|
|
144
|
+
return -1;
|
|
145
|
+
if (a.passRate !== null && b.passRate !== null && a.passRate !== b.passRate) {
|
|
146
|
+
return a.passRate - b.passRate;
|
|
147
|
+
}
|
|
148
|
+
return a.testCase.name.localeCompare(b.testCase.name);
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
export function getCasePagerPosition(rows, selectedCaseId) {
|
|
152
|
+
const index = selectedCaseId ? rows.findIndex(row => row.testCase.id === selectedCaseId) : -1;
|
|
153
|
+
return {
|
|
154
|
+
index,
|
|
155
|
+
position: index >= 0 ? index + 1 : 0,
|
|
156
|
+
total: rows.length,
|
|
157
|
+
previousId: index > 0 ? rows[index - 1].testCase.id : undefined,
|
|
158
|
+
nextId: index >= 0 && index < rows.length - 1 ? rows[index + 1].testCase.id : undefined,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The Cases tab always renders a case's *current* definition, even when the
|
|
163
|
+
* user arrived from a specific historical run (e.g. a heat-strip cell click).
|
|
164
|
+
* This detects that mismatch so callers can show a minimal, honest notice
|
|
165
|
+
* instead of silently presenting a definition the run never actually used.
|
|
166
|
+
* Deliberately does not resurrect the old per-version selector — just a
|
|
167
|
+
* disclosure that the definition may have changed since some of the runs
|
|
168
|
+
* shown here.
|
|
169
|
+
*/
|
|
170
|
+
export function hasHistoricalVersionMismatch(currentVersion, runs, testCaseId, reportsById) {
|
|
171
|
+
return runs.some(run => {
|
|
172
|
+
const result = run.results?.[testCaseId];
|
|
173
|
+
const report = result?.reportId ? reportsById[result.reportId] : undefined;
|
|
174
|
+
return typeof report?.testCaseVersion === 'number' && report.testCaseVersion !== currentVersion;
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=benchmarkCaseReview.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmarkCaseReview.js","sourceRoot":"","sources":["../../benchmarkCaseReview.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAenC;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAC/B,EAAE,EACF,EAAE,EACF,OAAO,EACP,OAAO,EACP,KAAK,GACU;IACf,MAAM,WAAW,GAAG,OAAO,IAAI,CAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe;QAC9B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAC5B,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAEvD,MAAM,WAAW,GAAG,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,KAAK,IAAI,EAAE,GAAG,CAAC,CAAC;IAClC,OAAO;QACL,MAAM,EAAE,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,CAAC,EAAE;QACpE,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,EAAU,EACV,EAAU,EACV,UAAU,GAAG,CAAC;IAEd,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAClC,CAAC;AAmCD,uFAAuF;AACvF,MAAM,UAAU,qBAAqB,CAAC,GAAe;IACnD,IAAI,GAAG,CAAC,MAAM;QAAE,OAAO,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC;IAClD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACjD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAClD,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAC3D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAuB,IAAS,EAAE,KAAK,GAAG,CAAC;IAC/E,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,qBAAqB,CAAC;SAC7B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;SACjF,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,GAAe,EACf,UAAkB,EAClB,WAA6D;IAE7D,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAE5F,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,MAAM,EAAE,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACzD,IAAI,MAAM,EAAE,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACzD,IAAI,MAAM,EAAE,aAAa,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC;IAExD,0EAA0E;IAC1E,wEAAwE;IACxE,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACxD,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAExD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/F,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,2EAA2E;IAC3E,6EAA6E;IAC7E,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;QAAE,OAAO,SAAS,CAAC;IACpD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,IAAkB,EAClB,UAAkB,EAClB,WAA6D;IAE7D,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAuB;IAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;IACnE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE5C,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,SAAS,CAAC,EAAE,CAAC;QAC/E,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,QAAQ,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,OAAO,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IACvC,IAAI,OAAO;QAAE,OAAO,QAAQ,CAAC;IAE7B,uEAAuE;IACvE,6EAA6E;IAC7E,iBAAiB;IACjB,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAuB;IACrD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvE,OAAO,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,SAAc,EACd,UAAwB,EACxB,WAA6D;IAE7D,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;QAC9B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC1E,OAAO;YACL,QAAQ;YACR,QAAQ;YACR,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC;YACtC,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC;SACpC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,qBAAqB,CACnC,IAAwB,EACxB,MAAc,EACd,MAAwB;IAExB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,iBAAiB,EAAE,CAAC;IAChD,OAAO,IAAI;SACR,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC;SACxD,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK;QACnB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;QACrD,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAChE;SACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC5E,OAAO,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QACjC,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAwB,EACxB,cAAkC;IAElC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9F,OAAO;QACL,KAAK;QACL,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;QAC/D,MAAM,EAAE,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;KACxF,CAAC;AACJ,CAAC;AAcD;;;;;;;;GAQG;AACH,MAAM,UAAU,4BAA4B,CAC1C,cAAsB,EACtB,IAAoB,EACpB,UAAkB,EAClB,WAA+D;IAE/D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,OAAO,OAAO,MAAM,EAAE,eAAe,KAAK,QAAQ,IAAI,MAAM,CAAC,eAAe,KAAK,cAAc,CAAC;IAClG,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure helpers for the benchmark Runs tab (table + pass-rate-over-time chart).
|
|
3
|
+
*
|
|
4
|
+
* Kept free of React/DOM so the click-to-filter semantics and the chart
|
|
5
|
+
* series derivation can be unit-tested directly. The component
|
|
6
|
+
* (components/evals3/BenchmarkRunsPage.tsx) is a thin renderer over these.
|
|
7
|
+
*/
|
|
8
|
+
import type { BenchmarkRun } from '../types/index.js';
|
|
9
|
+
import { getEffectiveRunStatus } from '../lib/runStats.js';
|
|
10
|
+
/** The dimensions a user can click on in the table to filter by. */
|
|
11
|
+
export type RunFilterField = 'agent' | 'model' | 'judge' | 'evaluator' | 'status';
|
|
12
|
+
export interface RunFilter {
|
|
13
|
+
field: RunFilterField;
|
|
14
|
+
/** Raw value (id/key), NOT the display label — labels can collide. */
|
|
15
|
+
value: string;
|
|
16
|
+
/** Human label for the pill. */
|
|
17
|
+
label: string;
|
|
18
|
+
}
|
|
19
|
+
export interface RunTableRow {
|
|
20
|
+
run: BenchmarkRun;
|
|
21
|
+
agentKey: string;
|
|
22
|
+
agentName: string;
|
|
23
|
+
modelId: string;
|
|
24
|
+
modelName: string;
|
|
25
|
+
judgeModelId: string;
|
|
26
|
+
judgeLabel: string;
|
|
27
|
+
evaluatorId: string;
|
|
28
|
+
evaluatorLabel: string;
|
|
29
|
+
status: ReturnType<typeof getEffectiveRunStatus>;
|
|
30
|
+
passed: number;
|
|
31
|
+
failed: number;
|
|
32
|
+
errored: number;
|
|
33
|
+
/** Still to come — only ever non-zero while the run is live. */
|
|
34
|
+
pending: number;
|
|
35
|
+
/** Per-case entries currently executing — only ever non-zero while the run is live. */
|
|
36
|
+
running: number;
|
|
37
|
+
/**
|
|
38
|
+
* Planned test cases that never executed because the run reached a
|
|
39
|
+
* TERMINAL status first (cancelled mid-way, or the executor died). Neither
|
|
40
|
+
* a pass, a fail nor "pending": nothing will ever happen to them, so they
|
|
41
|
+
* must never render with an in-progress spinner.
|
|
42
|
+
*/
|
|
43
|
+
notRun: number;
|
|
44
|
+
total: number;
|
|
45
|
+
/**
|
|
46
|
+
* Pass rate over the *evaluable* set (total − errored − pending − running),
|
|
47
|
+
* or null when nothing has been evaluated yet. Percent 0–100.
|
|
48
|
+
*/
|
|
49
|
+
passRate: number | null;
|
|
50
|
+
/** Number of test cases in the run (snapshotted size, falling back to results). */
|
|
51
|
+
size: number;
|
|
52
|
+
}
|
|
53
|
+
export interface RowLabelResolvers {
|
|
54
|
+
agentName: (agentKey: string) => string;
|
|
55
|
+
modelName: (modelId: string) => string;
|
|
56
|
+
judgeLabel: (judgeModelId?: string | null) => string;
|
|
57
|
+
evaluatorLabel: (evaluatorId?: string | null) => string;
|
|
58
|
+
}
|
|
59
|
+
export declare function computePassRate(passed: number, failed: number): number | null;
|
|
60
|
+
export declare function buildRunTableRow(run: BenchmarkRun, resolve: RowLabelResolvers): RunTableRow;
|
|
61
|
+
export declare function rowFieldValue(row: RunTableRow, field: RunFilterField): string;
|
|
62
|
+
export declare function rowFieldLabel(row: RunTableRow, field: RunFilterField): string;
|
|
63
|
+
export declare function isSameFilter(a: RunFilter, b: RunFilter): boolean;
|
|
64
|
+
/**
|
|
65
|
+
* Click semantics: clicking a value that is already an active filter removes
|
|
66
|
+
* it (toggle); otherwise it is added. Multiple values on the same field are
|
|
67
|
+
* OR-ed; different fields are AND-ed (see {@link applyRunFilters}).
|
|
68
|
+
*/
|
|
69
|
+
export declare function toggleRunFilter(filters: RunFilter[], next: RunFilter): RunFilter[];
|
|
70
|
+
export declare function removeRunFilter(filters: RunFilter[], target: RunFilter): RunFilter[];
|
|
71
|
+
export declare function applyRunFilters(rows: RunTableRow[], filters: RunFilter[]): RunTableRow[];
|
|
72
|
+
export type RunSortField = 'name' | 'agent' | 'model' | 'size' | 'passRate' | 'judge' | 'evaluator' | 'date';
|
|
73
|
+
export interface RunSort {
|
|
74
|
+
field: RunSortField;
|
|
75
|
+
dir: 'asc' | 'desc';
|
|
76
|
+
}
|
|
77
|
+
export declare const DEFAULT_RUN_SORT: RunSort;
|
|
78
|
+
export declare function toggleRunSort(current: RunSort, field: RunSortField): RunSort;
|
|
79
|
+
export declare function sortRunRows(rows: RunTableRow[], sort: RunSort): RunTableRow[];
|
|
80
|
+
export interface PassRatePoint {
|
|
81
|
+
/** Epoch ms of run.createdAt — numeric so the X axis is a real time axis. */
|
|
82
|
+
t: number;
|
|
83
|
+
runId: string;
|
|
84
|
+
runName: string;
|
|
85
|
+
passRate: number;
|
|
86
|
+
passed: number;
|
|
87
|
+
failed: number;
|
|
88
|
+
total: number;
|
|
89
|
+
}
|
|
90
|
+
export interface PassRateSeries {
|
|
91
|
+
/** Series key = agentKey (the sketch groups lines by agent, e.g. CC vs AIS). */
|
|
92
|
+
key: string;
|
|
93
|
+
label: string;
|
|
94
|
+
points: PassRatePoint[];
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* One line per agent, points ordered by time. Runs without an evaluable
|
|
98
|
+
* result (pass rate null — e.g. still running with nothing judged, or fully
|
|
99
|
+
* errored) are excluded: plotting them as 0% would read as a regression.
|
|
100
|
+
*/
|
|
101
|
+
export declare function buildPassRateSeries(rows: RunTableRow[]): PassRateSeries[];
|
|
102
|
+
/** Latest run = max createdAt over the given runs (not "first in the array" —
|
|
103
|
+
* the merged list appends standalone eval-run docs after embedded runs, so
|
|
104
|
+
* array order is not chronological). */
|
|
105
|
+
export declare function latestRunId(runs: Array<Pick<BenchmarkRun, 'id' | 'createdAt'>>): string | null;
|
|
106
|
+
/** Deterministic, distinguishable palette for agent lines (index-based). */
|
|
107
|
+
export declare const SERIES_COLORS: string[];
|
|
108
|
+
export declare function seriesColor(index: number): string;
|
|
109
|
+
//# sourceMappingURL=benchmarkRunsTable.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmarkRunsTable.d.ts","sourceRoot":"","sources":["../../benchmarkRunsTable.ts"],"names":[],"mappings":"AAKA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAmB,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAIxE,oEAAoE;AACpE,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC;AAElF,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,cAAc,CAAC;IACtB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,YAAY,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,mFAAmF;IACnF,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;IACvC,UAAU,EAAE,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC;IACrD,cAAc,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC;CACzD;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAC7B,MAAM,GAAG,IAAI,CAIf;AAgBD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,GAAG,WAAW,CAsC3F;AAID,wBAAgB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,CAQ7E;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,CAQ7E;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,CAEhE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,SAAS,EAAE,CAGlF;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAEpF;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,WAAW,EAAE,CAaxF;AAID,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,GAAG,WAAW,GAAG,MAAM,CAAC;AAC7G,MAAM,WAAW,OAAO;IAAG,KAAK,EAAE,YAAY,CAAC;IAAC,GAAG,EAAE,KAAK,GAAG,MAAM,CAAC;CAAE;AAEtE,eAAO,MAAM,gBAAgB,EAAE,OAAwC,CAAC;AAExE,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAM5E;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,GAAG,WAAW,EAAE,CAuB7E;AAID,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,cAAc,EAAE,CAuBzE;AAED;;yCAEyC;AACzC,wBAAgB,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAQ9F;AAED,4EAA4E;AAC5E,eAAO,MAAM,aAAa,UASzB,CAAC;AAEF,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEjD"}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright OpenSearch Contributors
|
|
3
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
*/
|
|
5
|
+
import { computeRunStats, getEffectiveRunStatus } from '../lib/runStats.js';
|
|
6
|
+
export function computePassRate(passed, failed) {
|
|
7
|
+
const evaluable = passed + failed;
|
|
8
|
+
if (evaluable <= 0)
|
|
9
|
+
return null;
|
|
10
|
+
return Math.round((passed / evaluable) * 1000) / 10;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Run statuses after which no further per-case progress can happen. Only the
|
|
14
|
+
* EXPLICIT persisted status counts: the results-derived fallback in
|
|
15
|
+
* getEffectiveRunStatus reports 'completed' for any status-less legacy run
|
|
16
|
+
* whose observed results all happen to be settled — exactly what an in-flight
|
|
17
|
+
* legacy run looks like between two cases — so it must not gate bucketing
|
|
18
|
+
* (a status-less run therefore keeps the pre-existing pending rendering).
|
|
19
|
+
* Mirrors `TERMINAL_RUN_STATUSES` in the terminal-aware `lib/runStats`
|
|
20
|
+
* (#486); collapse onto `isTerminalRunStatus` from there once it lands.
|
|
21
|
+
*/
|
|
22
|
+
function isTerminalStatus(status) {
|
|
23
|
+
return status === 'completed' || status === 'failed' || status === 'cancelled';
|
|
24
|
+
}
|
|
25
|
+
export function buildRunTableRow(run, resolve) {
|
|
26
|
+
const status = getEffectiveRunStatus(run);
|
|
27
|
+
const terminal = isTerminalStatus(run.status);
|
|
28
|
+
const { passed, failed, errored, total } = computeRunStats(run);
|
|
29
|
+
const settled = passed + failed + errored;
|
|
30
|
+
// Bug (owner report, 2026-09-08): a run cancelled at 43/62 rendered
|
|
31
|
+
// "11/32 /19 ⟳" — the never-started remainder (planned − observed) was
|
|
32
|
+
// re-derived as `pending` here regardless of the run's TERMINAL status, so
|
|
33
|
+
// every cancelled/failed run on the benchmark page looked in progress
|
|
34
|
+
// forever. Once the run is terminal nothing can start those cases (nor
|
|
35
|
+
// finish a per-case entry an executor left `running` when it died), so
|
|
36
|
+
// the remainder is `notRun`, and pending/running are zero by definition.
|
|
37
|
+
let running = 0;
|
|
38
|
+
if (!terminal) {
|
|
39
|
+
Object.values(run.results || {}).forEach(r => { if (r.status === 'running')
|
|
40
|
+
running++; });
|
|
41
|
+
}
|
|
42
|
+
const notRun = terminal ? Math.max(0, total - settled) : 0;
|
|
43
|
+
const pending = terminal ? 0 : Math.max(0, total - settled - running);
|
|
44
|
+
const size = run.testCaseSnapshots?.length || total;
|
|
45
|
+
const agentKey = run.agentKey || '';
|
|
46
|
+
const modelId = run.modelId || '';
|
|
47
|
+
const judgeModelId = run.judgeModelId || '';
|
|
48
|
+
const evaluatorId = run.evaluatorId || '';
|
|
49
|
+
return {
|
|
50
|
+
run,
|
|
51
|
+
agentKey,
|
|
52
|
+
agentName: resolve.agentName(agentKey),
|
|
53
|
+
modelId,
|
|
54
|
+
modelName: resolve.modelName(modelId),
|
|
55
|
+
judgeModelId,
|
|
56
|
+
judgeLabel: resolve.judgeLabel(judgeModelId || undefined),
|
|
57
|
+
evaluatorId,
|
|
58
|
+
evaluatorLabel: resolve.evaluatorLabel(evaluatorId || undefined),
|
|
59
|
+
status,
|
|
60
|
+
passed, failed, errored, pending, running, notRun, total,
|
|
61
|
+
passRate: computePassRate(passed, failed),
|
|
62
|
+
size,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
// ─── Filters ─────────────────────────────────────────────────────────────────
|
|
66
|
+
export function rowFieldValue(row, field) {
|
|
67
|
+
switch (field) {
|
|
68
|
+
case 'agent': return row.agentKey;
|
|
69
|
+
case 'model': return row.modelId;
|
|
70
|
+
case 'judge': return row.judgeModelId;
|
|
71
|
+
case 'evaluator': return row.evaluatorId;
|
|
72
|
+
case 'status': return row.status;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
export function rowFieldLabel(row, field) {
|
|
76
|
+
switch (field) {
|
|
77
|
+
case 'agent': return row.agentName;
|
|
78
|
+
case 'model': return row.modelName;
|
|
79
|
+
case 'judge': return row.judgeLabel;
|
|
80
|
+
case 'evaluator': return row.evaluatorLabel;
|
|
81
|
+
case 'status': return row.status;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
export function isSameFilter(a, b) {
|
|
85
|
+
return a.field === b.field && a.value === b.value;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Click semantics: clicking a value that is already an active filter removes
|
|
89
|
+
* it (toggle); otherwise it is added. Multiple values on the same field are
|
|
90
|
+
* OR-ed; different fields are AND-ed (see {@link applyRunFilters}).
|
|
91
|
+
*/
|
|
92
|
+
export function toggleRunFilter(filters, next) {
|
|
93
|
+
const exists = filters.some(f => isSameFilter(f, next));
|
|
94
|
+
return exists ? filters.filter(f => !isSameFilter(f, next)) : [...filters, next];
|
|
95
|
+
}
|
|
96
|
+
export function removeRunFilter(filters, target) {
|
|
97
|
+
return filters.filter(f => !isSameFilter(f, target));
|
|
98
|
+
}
|
|
99
|
+
export function applyRunFilters(rows, filters) {
|
|
100
|
+
if (filters.length === 0)
|
|
101
|
+
return rows;
|
|
102
|
+
const byField = new Map();
|
|
103
|
+
for (const f of filters) {
|
|
104
|
+
if (!byField.has(f.field))
|
|
105
|
+
byField.set(f.field, new Set());
|
|
106
|
+
byField.get(f.field).add(f.value);
|
|
107
|
+
}
|
|
108
|
+
return rows.filter(row => {
|
|
109
|
+
for (const [field, values] of byField) {
|
|
110
|
+
if (!values.has(rowFieldValue(row, field)))
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
return true;
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export const DEFAULT_RUN_SORT = { field: 'date', dir: 'desc' };
|
|
117
|
+
export function toggleRunSort(current, field) {
|
|
118
|
+
if (current.field === field)
|
|
119
|
+
return { field, dir: current.dir === 'asc' ? 'desc' : 'asc' };
|
|
120
|
+
// Numeric/date columns default to descending (biggest/newest first);
|
|
121
|
+
// text columns to ascending.
|
|
122
|
+
const numeric = ['size', 'passRate', 'date'];
|
|
123
|
+
return { field, dir: numeric.includes(field) ? 'desc' : 'asc' };
|
|
124
|
+
}
|
|
125
|
+
export function sortRunRows(rows, sort) {
|
|
126
|
+
const dir = sort.dir === 'asc' ? 1 : -1;
|
|
127
|
+
const cmpNum = (a, b) => {
|
|
128
|
+
// nulls always sink to the bottom regardless of direction
|
|
129
|
+
if (a === null && b === null)
|
|
130
|
+
return 0;
|
|
131
|
+
if (a === null)
|
|
132
|
+
return 1;
|
|
133
|
+
if (b === null)
|
|
134
|
+
return -1;
|
|
135
|
+
return dir * (a - b);
|
|
136
|
+
};
|
|
137
|
+
return [...rows].sort((a, b) => {
|
|
138
|
+
switch (sort.field) {
|
|
139
|
+
case 'name': return dir * a.run.name.localeCompare(b.run.name);
|
|
140
|
+
case 'agent': return dir * a.agentName.localeCompare(b.agentName);
|
|
141
|
+
case 'model': return dir * a.modelName.localeCompare(b.modelName);
|
|
142
|
+
case 'judge': return dir * a.judgeLabel.localeCompare(b.judgeLabel);
|
|
143
|
+
case 'evaluator': return dir * a.evaluatorLabel.localeCompare(b.evaluatorLabel);
|
|
144
|
+
case 'size': return cmpNum(a.size, b.size);
|
|
145
|
+
case 'passRate': return cmpNum(a.passRate, b.passRate);
|
|
146
|
+
case 'date':
|
|
147
|
+
default:
|
|
148
|
+
return dir * (new Date(a.run.createdAt).getTime() - new Date(b.run.createdAt).getTime());
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* One line per agent, points ordered by time. Runs without an evaluable
|
|
154
|
+
* result (pass rate null — e.g. still running with nothing judged, or fully
|
|
155
|
+
* errored) are excluded: plotting them as 0% would read as a regression.
|
|
156
|
+
*/
|
|
157
|
+
export function buildPassRateSeries(rows) {
|
|
158
|
+
const byAgent = new Map();
|
|
159
|
+
for (const row of rows) {
|
|
160
|
+
if (row.passRate === null)
|
|
161
|
+
continue;
|
|
162
|
+
const t = new Date(row.run.createdAt).getTime();
|
|
163
|
+
if (Number.isNaN(t))
|
|
164
|
+
continue;
|
|
165
|
+
let series = byAgent.get(row.agentKey);
|
|
166
|
+
if (!series) {
|
|
167
|
+
series = { key: row.agentKey, label: row.agentName, points: [] };
|
|
168
|
+
byAgent.set(row.agentKey, series);
|
|
169
|
+
}
|
|
170
|
+
series.points.push({
|
|
171
|
+
t, runId: row.run.id, runName: row.run.name,
|
|
172
|
+
passRate: row.passRate, passed: row.passed, failed: row.failed, total: row.total,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
const out = [...byAgent.values()];
|
|
176
|
+
for (const s of out)
|
|
177
|
+
s.points.sort((a, b) => a.t - b.t);
|
|
178
|
+
// Order by label (then key) — NOT by point count. Series index drives the
|
|
179
|
+
// line colour, so the order must be stable while the page polls every few
|
|
180
|
+
// seconds and new runs land; "busiest first" would recolour agents mid-run.
|
|
181
|
+
out.sort((a, b) => a.label.localeCompare(b.label) || a.key.localeCompare(b.key));
|
|
182
|
+
return out;
|
|
183
|
+
}
|
|
184
|
+
/** Latest run = max createdAt over the given runs (not "first in the array" —
|
|
185
|
+
* the merged list appends standalone eval-run docs after embedded runs, so
|
|
186
|
+
* array order is not chronological). */
|
|
187
|
+
export function latestRunId(runs) {
|
|
188
|
+
let best = null;
|
|
189
|
+
for (const r of runs) {
|
|
190
|
+
const t = new Date(r.createdAt).getTime();
|
|
191
|
+
if (Number.isNaN(t))
|
|
192
|
+
continue;
|
|
193
|
+
if (!best || t > best.t)
|
|
194
|
+
best = { id: r.id, t };
|
|
195
|
+
}
|
|
196
|
+
return best?.id ?? null;
|
|
197
|
+
}
|
|
198
|
+
/** Deterministic, distinguishable palette for agent lines (index-based). */
|
|
199
|
+
export const SERIES_COLORS = [
|
|
200
|
+
'#3b82f6', // blue-500
|
|
201
|
+
'#f59e0b', // amber-500
|
|
202
|
+
'#10b981', // emerald-500
|
|
203
|
+
'#8b5cf6', // violet-500
|
|
204
|
+
'#ef4444', // red-500
|
|
205
|
+
'#06b6d4', // cyan-500
|
|
206
|
+
'#ec4899', // pink-500
|
|
207
|
+
'#84cc16', // lime-500
|
|
208
|
+
];
|
|
209
|
+
export function seriesColor(index) {
|
|
210
|
+
return SERIES_COLORS[index % SERIES_COLORS.length];
|
|
211
|
+
}
|
|
212
|
+
//# sourceMappingURL=benchmarkRunsTable.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmarkRunsTable.js","sourceRoot":"","sources":["../../benchmarkRunsTable.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAyDxE,MAAM,UAAU,eAAe,CAC7B,MAAc,EAAE,MAAc;IAE9B,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAClC,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AACtD,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,gBAAgB,CAAC,MAA8B;IACtD,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,WAAW,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAiB,EAAE,OAA0B;IAC5E,MAAM,MAAM,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IAChE,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC1C,oEAAoE;IACpE,uEAAuE;IACvE,2EAA2E;IAC3E,sEAAsE;IACtE,uEAAuE;IACvE,uEAAuE;IACvE,yEAAyE;IACzE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,GAAG,CAAC,iBAAiB,EAAE,MAAM,IAAI,KAAK,CAAC;IACpD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAClC,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;IAC1C,OAAO;QACL,GAAG;QACH,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC;QACtC,OAAO;QACP,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;QACrC,YAAY;QACZ,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,YAAY,IAAI,SAAS,CAAC;QACzD,WAAW;QACX,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI,SAAS,CAAC;QAChE,MAAM;QACN,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;QACxD,QAAQ,EAAE,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC;QACzC,IAAI;KACL,CAAC;AACJ,CAAC;AAED,gFAAgF;AAEhF,MAAM,UAAU,aAAa,CAAC,GAAgB,EAAE,KAAqB;IACnE,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,QAAQ,CAAC;QAClC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC;QACjC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,YAAY,CAAC;QACtC,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,CAAC,WAAW,CAAC;QACzC,KAAK,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAgB,EAAE,KAAqB;IACnE,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC;QACnC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC;QACnC,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,UAAU,CAAC;QACpC,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,CAAC,cAAc,CAAC;QAC5C,KAAK,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAY,EAAE,CAAY;IACrD,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC;AACpD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAAoB,EAAE,IAAe;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAoB,EAAE,MAAiB;IACrE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAmB,EAAE,OAAoB;IACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;QACvB,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAOD,MAAM,CAAC,MAAM,gBAAgB,GAAY,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AAExE,MAAM,UAAU,aAAa,CAAC,OAAgB,EAAE,KAAmB;IACjE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;QAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3F,qEAAqE;IACrE,6BAA6B;IAC7B,MAAM,OAAO,GAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAC7D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAmB,EAAE,IAAa;IAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,CAAgB,EAAE,CAAgB,EAAE,EAAE;QACpD,0DAA0D;QAC1D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,CAAC,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7B,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,KAAK,MAAM,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC/D,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAClE,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAClE,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACpE,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YAChF,KAAK,MAAM,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3C,KAAK,UAAU,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;YACvD,KAAK,MAAM,CAAC;YACZ;gBACE,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAsBD;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAmB;IACrD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI;YAAE,SAAS;QACpC,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QAC9B,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI;YAC3C,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK;SACjF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,GAAG;QAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,0EAA0E;IAC1E,0EAA0E;IAC1E,4EAA4E;IAC5E,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;yCAEyC;AACzC,MAAM,UAAU,WAAW,CAAC,IAAmD;IAC7E,IAAI,IAAI,GAAqC,IAAI,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QAC9B,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YAAE,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAClD,CAAC;IACD,OAAO,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,YAAY;IACvB,SAAS,EAAE,cAAc;IACzB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,UAAU;IACrB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,OAAO,aAAa,CAAC,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;AACrD,CAAC"}
|
|
@@ -54,10 +54,49 @@ export declare const OTHER_CATEGORY = "(other)";
|
|
|
54
54
|
* `subcategory` as a dedicated concept).
|
|
55
55
|
* 4. {@link UNCATEGORIZED}.
|
|
56
56
|
* (`category:<x>` labels are intentionally NOT used here: imported benchmarks
|
|
57
|
-
* stamp a single `category:RAG` on every case, which would collapse
|
|
58
|
-
* breakdown into one column — that's the coarse domain, not this facet.
|
|
57
|
+
* commonly stamp a single `category:RAG` on every case, which would collapse
|
|
58
|
+
* the breakdown into one column — that's the coarse domain, not this facet.
|
|
59
|
+
* {@link categoryLabelIsUsableFallback} + {@link extractRowCategoryEffective}
|
|
60
|
+
* add `category:` back in, but ONLY for the comparisons where it's actually
|
|
61
|
+
* informative — see their docs.)
|
|
59
62
|
*/
|
|
60
63
|
export declare function extractRowCategory(row: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>): string;
|
|
64
|
+
/**
|
|
65
|
+
* Whether `category:` should be used as a LAST-RESORT fallback facet for
|
|
66
|
+
* this set of rows. True only when, among the rows {@link extractRowCategory}
|
|
67
|
+
* leaves as {@link UNCATEGORIZED} (rows with a real `[bracket]`/`topic:`
|
|
68
|
+
* facet keep using that, whether or not `category:` also varies — one
|
|
69
|
+
* differently-tagged row must not disable the fallback for every OTHER row
|
|
70
|
+
* that has nothing better):
|
|
71
|
+
* 1. There's at least one such row.
|
|
72
|
+
* 2. The `category:` label, normalized the same way {@link
|
|
73
|
+
* extractRowCategoryEffective} returns it (lowercased, so
|
|
74
|
+
* `category:RAG` and `category:rag` count as ONE value, not two —
|
|
75
|
+
* inconsistent casing must not look like variance), has ≥2 distinct
|
|
76
|
+
* values that EACH appear on ≥2 rows. The 2-per-value floor keeps a
|
|
77
|
+
* single stray/typo'd label from flipping this on for what is really a
|
|
78
|
+
* uniform category with one dirty row — that must still resolve to
|
|
79
|
+
* "nothing to facet by", not a real-facet-plus-a-noise-column matrix.
|
|
80
|
+
* A uniformly-stamped `category:RAG` (the case the original "intentionally
|
|
81
|
+
* NOT used" heuristic was written for) stays excluded — falling back to it
|
|
82
|
+
* would add a redundant single-value column to every such comparison,
|
|
83
|
+
* exactly the noise that heuristic was trying to avoid. WixQA-400's
|
|
84
|
+
* `category:expertwritten|simulated` (200/200) is the motivating case where
|
|
85
|
+
* it DOES vary (both values well above the floor) and is genuinely useful.
|
|
86
|
+
* Callers that resolve a per-row category for a set of rows should compute
|
|
87
|
+
* this ONCE for the whole set and pass it to every {@link
|
|
88
|
+
* extractRowCategoryEffective} call, so the matrix, its shared-weakness
|
|
89
|
+
* callout, and the table's click-to-filter all agree on the same facet.
|
|
90
|
+
*/
|
|
91
|
+
export declare function categoryLabelIsUsableFallback(rows: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>[]): boolean;
|
|
92
|
+
/**
|
|
93
|
+
* {@link extractRowCategory}, extended with the `category:` fallback when
|
|
94
|
+
* {@link categoryLabelIsUsableFallback} (computed once per row-set, passed
|
|
95
|
+
* in as `useCategoryFallback`) says it's actually informative for this
|
|
96
|
+
* comparison. Only reached when the base extraction is {@link
|
|
97
|
+
* UNCATEGORIZED} — a real bracket/topic facet always wins.
|
|
98
|
+
*/
|
|
99
|
+
export declare function extractRowCategoryEffective(row: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>, useCategoryFallback: boolean): string;
|
|
61
100
|
export interface CategoryCell {
|
|
62
101
|
passed: number;
|
|
63
102
|
total: number;
|
|
@@ -76,6 +115,14 @@ export interface CategoryBreakdown {
|
|
|
76
115
|
* the rows its cell counted.
|
|
77
116
|
*/
|
|
78
117
|
members: Record<string, string[]>;
|
|
118
|
+
/**
|
|
119
|
+
* Whether this breakdown fell back to the `category:` label (see {@link
|
|
120
|
+
* categoryLabelIsUsableFallback}). Callers that need to resolve a row's
|
|
121
|
+
* category OUTSIDE this function (e.g. {@link detectSharedWeakness}, or
|
|
122
|
+
* the compare page's click-to-filter) must pass this to {@link
|
|
123
|
+
* extractRowCategoryEffective} to stay consistent with what's displayed.
|
|
124
|
+
*/
|
|
125
|
+
usesCategoryFallback: boolean;
|
|
79
126
|
}
|
|
80
127
|
/**
|
|
81
128
|
* Default minimum case count for a category to get its own column; smaller
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"comparisonInsights.d.ts","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAGrD,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6EAA6E;IAC7E,SAAS,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAyBD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,kBAAkB,CAgBpB;AAED,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,IAAI,CAO9F;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAC/C,wEAAwE;AACxE,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC
|
|
1
|
+
{"version":3,"file":"comparisonInsights.d.ts","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAGrD,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6EAA6E;IAC7E,SAAS,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAyBD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,kBAAkB,CAgBpB;AAED,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,IAAI,CAO9F;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAC/C,wEAAwE;AACxE,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,GAAG,MAAM,CAQtG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,EAAE,GAC7D,OAAO,CAUT;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,EAC3D,mBAAmB,EAAE,OAAO,GAC3B,MAAM,CAKR;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACrD,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAClC;;;;;;OAMG;IACH,oBAAoB,EAAE,OAAO,CAAC;CAC/B;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,kFAAkF;AAClF,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,GAAE,MAA2B,GACpC,iBAAiB,CAwCnB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,iBAAiB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,cAAc,GAAG,IAAI,CAuCvB"}
|