@post-print/agent-test 2.0.11 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/sdk/cli.d.ts +10 -0
- package/dist/sdk/cli.d.ts.map +1 -1
- package/dist/sdk/cli.js +27 -2
- package/dist/sdk/cli.js.map +1 -1
- package/dist/sdk/execution-reporter.d.ts.map +1 -1
- package/dist/sdk/execution-reporter.js +4 -0
- package/dist/sdk/execution-reporter.js.map +1 -1
- package/dist/sdk/execution-store.d.ts +9 -0
- package/dist/sdk/execution-store.d.ts.map +1 -1
- package/dist/sdk/execution-store.js +27 -5
- package/dist/sdk/execution-store.js.map +1 -1
- package/dist/sdk/stored-value.d.ts.map +1 -1
- package/dist/sdk/stored-value.js +11 -0
- package/dist/sdk/stored-value.js.map +1 -1
- package/dist/sdk/workspace.d.ts.map +1 -1
- package/dist/sdk/workspace.js +14 -1
- package/dist/sdk/workspace.js.map +1 -1
- package/dist/viewer/client-entry.js +21 -20
- package/dist/viewer/main.js +16 -4
- package/dist/viewer/main.js.map +1 -1
- package/dist/viewer/presentation.d.ts +20 -2
- package/dist/viewer/presentation.d.ts.map +1 -1
- package/dist/viewer/presentation.js +64 -2
- package/dist/viewer/presentation.js.map +1 -1
- package/dist/viewer/run-inbox.d.ts +7 -0
- package/dist/viewer/run-inbox.d.ts.map +1 -0
- package/dist/viewer/run-inbox.js +335 -0
- package/dist/viewer/run-inbox.js.map +1 -0
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +52 -4
- package/dist/viewer/server.js.map +1 -1
- package/dist/viewer/styles.d.ts.map +1 -1
- package/dist/viewer/styles.js +5 -0
- package/dist/viewer/styles.js.map +1 -1
- package/dist/viewer/stylex.css +234 -2
- package/dist/viewer/tool-images.d.ts +13 -0
- package/dist/viewer/tool-images.d.ts.map +1 -0
- package/dist/viewer/tool-images.js +77 -0
- package/dist/viewer/tool-images.js.map +1 -0
- package/dist/viewer/v2-client.d.ts.map +1 -1
- package/dist/viewer/v2-client.js +331 -172
- package/dist/viewer/v2-client.js.map +1 -1
- package/dist/viewer/viewer-queries.d.ts +26 -0
- package/dist/viewer/viewer-queries.d.ts.map +1 -0
- package/dist/viewer/viewer-queries.js +31 -0
- package/dist/viewer/viewer-queries.js.map +1 -0
- package/package.json +2 -2
package/dist/viewer/v2-client.js
CHANGED
|
@@ -4,9 +4,11 @@ import { useMutation, useQueries, useQuery, useQueryClient } from "@tanstack/rea
|
|
|
4
4
|
import { createRootRoute, createRoute, createRouter, Link, Outlet, useNavigate, } from "@tanstack/react-router";
|
|
5
5
|
import { createContext, useCallback, useContext, useEffect, useId, useRef, useState, } from "react";
|
|
6
6
|
import Markdown from "react-markdown";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
|
|
7
|
+
import { assertionComparison, conversationPlaceholder, formatRunDate, median, preferredTestExecutionId, stripAnsi, testRunStatus, unwrapShellCommand, } from "./presentation.js";
|
|
8
|
+
import { RunInboxPage, RunPage } from "./run-inbox.js";
|
|
9
|
+
import { ToolImages, toolImages } from "./tool-images.js";
|
|
10
|
+
import { cancelExecution, fetchExecution as fetchExecutionRecord, startSuite, startTest, startTestGroup, useCatalog, useExecutionHistory, } from "./viewer-queries.js";
|
|
11
|
+
const fetchExecution = (id) => fetchExecutionRecord(id);
|
|
10
12
|
const LOCAL_PATH_LINK = /\[([^\]]+)\]\(<local-path>[^)]*\)/g;
|
|
11
13
|
const runningPulse = stylex.keyframes({
|
|
12
14
|
"0%, 60%, 100%": { opacity: 0.3, transform: "translateY(0)" },
|
|
@@ -20,7 +22,7 @@ const rootRoute = createRootRoute({ component: ViewerLayout });
|
|
|
20
22
|
const indexRoute = createRoute({
|
|
21
23
|
getParentRoute: () => rootRoute,
|
|
22
24
|
path: "/",
|
|
23
|
-
component:
|
|
25
|
+
component: RunInboxPage,
|
|
24
26
|
});
|
|
25
27
|
const testRoute = createRoute({
|
|
26
28
|
getParentRoute: () => rootRoute,
|
|
@@ -36,7 +38,7 @@ const testRoute = createRoute({
|
|
|
36
38
|
const executionRoute = createRoute({
|
|
37
39
|
getParentRoute: () => rootRoute,
|
|
38
40
|
path: "/executions/$executionId",
|
|
39
|
-
component:
|
|
41
|
+
component: ExecutionRunPage,
|
|
40
42
|
});
|
|
41
43
|
const compareRoute = createRoute({
|
|
42
44
|
getParentRoute: () => rootRoute,
|
|
@@ -374,6 +376,14 @@ const styles = stylex.create({
|
|
|
374
376
|
border: "1px solid transparent",
|
|
375
377
|
backgroundColor: { default: "transparent", ":hover": "var(--panel)" },
|
|
376
378
|
},
|
|
379
|
+
batchRunLink: {
|
|
380
|
+
justifySelf: "start",
|
|
381
|
+
marginTop: "0.75rem",
|
|
382
|
+
color: { default: "var(--muted)", ":hover": "var(--text)" },
|
|
383
|
+
fontSize: "0.78rem",
|
|
384
|
+
textDecoration: "none",
|
|
385
|
+
},
|
|
386
|
+
runsLink: { display: "flex", alignItems: "center", gap: "0.5rem", fontWeight: 650 },
|
|
377
387
|
navItemHeader: {
|
|
378
388
|
display: "grid",
|
|
379
389
|
gridTemplateColumns: "0.65rem minmax(0, 1fr)",
|
|
@@ -590,7 +600,7 @@ const styles = stylex.create({
|
|
|
590
600
|
},
|
|
591
601
|
quietHeader: {
|
|
592
602
|
display: "flex",
|
|
593
|
-
justifyContent: "flex-
|
|
603
|
+
justifyContent: "flex-start",
|
|
594
604
|
marginBottom: "0.75rem",
|
|
595
605
|
},
|
|
596
606
|
statusPanel: {
|
|
@@ -622,6 +632,47 @@ const styles = stylex.create({
|
|
|
622
632
|
gap: "0.7rem",
|
|
623
633
|
},
|
|
624
634
|
attemptTitle: { fontSize: "0.84rem", lineHeight: 1.4 },
|
|
635
|
+
verdictPanel: {
|
|
636
|
+
display: "grid",
|
|
637
|
+
gap: "0.55rem",
|
|
638
|
+
padding: "0.7rem 0.8rem",
|
|
639
|
+
borderRadius: 10,
|
|
640
|
+
borderWidth: 1,
|
|
641
|
+
borderStyle: "solid",
|
|
642
|
+
borderColor: "var(--border)",
|
|
643
|
+
backgroundColor: "var(--panel-2)",
|
|
644
|
+
},
|
|
645
|
+
verdictPass: { borderColor: "oklch(0.79 0.16 155 / 0.45)", backgroundColor: "var(--pass-soft)" },
|
|
646
|
+
verdictFail: { borderColor: "oklch(0.73 0.18 25 / 0.5)", backgroundColor: "var(--fail-soft)" },
|
|
647
|
+
verdictHeadline: { margin: 0, fontSize: "0.92rem", fontWeight: 720, color: "var(--text)" },
|
|
648
|
+
verdictMetrics: {
|
|
649
|
+
display: "grid",
|
|
650
|
+
gridTemplateColumns: "repeat(auto-fit, minmax(7.5rem, 1fr))",
|
|
651
|
+
gap: "0.4rem 1rem",
|
|
652
|
+
margin: 0,
|
|
653
|
+
},
|
|
654
|
+
verdictMetric: { display: "grid", gap: "0.1rem", minWidth: 0 },
|
|
655
|
+
verdictAttachments: {
|
|
656
|
+
display: "flex",
|
|
657
|
+
flexWrap: "wrap",
|
|
658
|
+
gap: "0.25rem 1rem",
|
|
659
|
+
margin: 0,
|
|
660
|
+
padding: 0,
|
|
661
|
+
listStyle: "none",
|
|
662
|
+
fontSize: "0.8rem",
|
|
663
|
+
overflowWrap: "anywhere",
|
|
664
|
+
},
|
|
665
|
+
verdictMetricValue: {
|
|
666
|
+
display: "grid",
|
|
667
|
+
margin: 0,
|
|
668
|
+
color: "var(--text)",
|
|
669
|
+
fontSize: "0.86rem",
|
|
670
|
+
fontWeight: 650,
|
|
671
|
+
fontVariantNumeric: "tabular-nums",
|
|
672
|
+
},
|
|
673
|
+
verdictMetricMedian: { color: "var(--subtle)", fontSize: "0.68rem", fontWeight: 500 },
|
|
674
|
+
verdictPassedChecks: { display: "grid", gap: "0.45rem" },
|
|
675
|
+
verdictCount: { color: "var(--muted)", fontWeight: 560 },
|
|
625
676
|
resultPanel: {
|
|
626
677
|
display: "grid",
|
|
627
678
|
gap: "0.55rem",
|
|
@@ -712,9 +763,52 @@ const styles = stylex.create({
|
|
|
712
763
|
},
|
|
713
764
|
conversationHeader: {
|
|
714
765
|
display: "flex",
|
|
715
|
-
|
|
766
|
+
flexWrap: "wrap",
|
|
767
|
+
alignItems: "baseline",
|
|
716
768
|
justifyContent: "flex-start",
|
|
717
|
-
gap: "0.75rem",
|
|
769
|
+
gap: "0.25rem 0.75rem",
|
|
770
|
+
},
|
|
771
|
+
sectionIntro: { margin: 0, color: "var(--subtle)", fontSize: "0.72rem", lineHeight: 1.4 },
|
|
772
|
+
ranRow: { display: "flex", flexWrap: "wrap", alignItems: "center", gap: "0.35rem" },
|
|
773
|
+
ranLabel: {
|
|
774
|
+
color: "var(--subtle)",
|
|
775
|
+
fontSize: "0.66rem",
|
|
776
|
+
fontWeight: 700,
|
|
777
|
+
letterSpacing: "0.06em",
|
|
778
|
+
textTransform: "uppercase",
|
|
779
|
+
},
|
|
780
|
+
ranChip: {
|
|
781
|
+
padding: "0.12rem 0.45rem",
|
|
782
|
+
borderRadius: 999,
|
|
783
|
+
backgroundColor: "var(--panel-3)",
|
|
784
|
+
color: "var(--muted)",
|
|
785
|
+
fontSize: "0.72rem",
|
|
786
|
+
},
|
|
787
|
+
judgeOutcomeSource: {
|
|
788
|
+
gridColumn: 2,
|
|
789
|
+
margin: 0,
|
|
790
|
+
color: "var(--subtle)",
|
|
791
|
+
fontSize: "0.68rem",
|
|
792
|
+
lineHeight: 1.4,
|
|
793
|
+
},
|
|
794
|
+
judgeReason: { margin: 0, color: "var(--muted)", fontSize: "0.74rem", lineHeight: 1.5 },
|
|
795
|
+
judgeAskedLabel: {
|
|
796
|
+
marginRight: "0.35rem",
|
|
797
|
+
color: "var(--subtle)",
|
|
798
|
+
fontSize: "0.61rem",
|
|
799
|
+
fontWeight: 750,
|
|
800
|
+
letterSpacing: "0.06em",
|
|
801
|
+
textTransform: "uppercase",
|
|
802
|
+
},
|
|
803
|
+
judgeContextDisclosure: {
|
|
804
|
+
borderTop: "1px solid var(--border)",
|
|
805
|
+
paddingTop: "0.45rem",
|
|
806
|
+
},
|
|
807
|
+
judgeContextSummary: {
|
|
808
|
+
cursor: "pointer",
|
|
809
|
+
color: { default: "var(--muted)", ":hover": "var(--text)" },
|
|
810
|
+
fontSize: "0.74rem",
|
|
811
|
+
fontWeight: 650,
|
|
718
812
|
},
|
|
719
813
|
runTabs: { display: "inline-flex", alignItems: "center", gap: "0.12rem" },
|
|
720
814
|
runTab: {
|
|
@@ -846,15 +940,11 @@ const styles = stylex.create({
|
|
|
846
940
|
display: "grid",
|
|
847
941
|
justifySelf: "stretch",
|
|
848
942
|
gap: "0.45rem",
|
|
849
|
-
padding: "0.7rem",
|
|
850
|
-
border: "1px solid oklch(0.75 0.12 175 / 0.3)",
|
|
851
|
-
borderRadius: 10,
|
|
852
|
-
backgroundColor: "var(--accent-soft)",
|
|
853
943
|
},
|
|
854
944
|
judgeQuestionText: {
|
|
855
945
|
margin: 0,
|
|
856
|
-
color: "var(--
|
|
857
|
-
fontSize: "0.
|
|
946
|
+
color: "var(--muted)",
|
|
947
|
+
fontSize: "0.76rem",
|
|
858
948
|
lineHeight: 1.5,
|
|
859
949
|
},
|
|
860
950
|
judgeContext: {
|
|
@@ -863,14 +953,6 @@ const styles = stylex.create({
|
|
|
863
953
|
paddingTop: "0.55rem",
|
|
864
954
|
borderTop: "1px solid oklch(0.75 0.12 175 / 0.22)",
|
|
865
955
|
},
|
|
866
|
-
judgeContextHeader: {
|
|
867
|
-
display: "flex",
|
|
868
|
-
alignItems: "baseline",
|
|
869
|
-
justifyContent: "space-between",
|
|
870
|
-
gap: "0.75rem",
|
|
871
|
-
color: "var(--text)",
|
|
872
|
-
fontSize: "0.7rem",
|
|
873
|
-
},
|
|
874
956
|
judgeContextNote: { margin: 0, color: "var(--muted)", fontSize: "0.68rem", lineHeight: 1.4 },
|
|
875
957
|
judgeContextGroup: { display: "grid", gap: "0.35rem" },
|
|
876
958
|
judgeContextGroupHeading: {
|
|
@@ -944,13 +1026,12 @@ const styles = stylex.create({
|
|
|
944
1026
|
},
|
|
945
1027
|
judgeResponse: {
|
|
946
1028
|
display: "grid",
|
|
947
|
-
justifySelf: "
|
|
1029
|
+
justifySelf: "stretch",
|
|
948
1030
|
gap: "0.4rem",
|
|
949
1031
|
padding: "0.65rem 0.7rem",
|
|
950
1032
|
border: "1px solid var(--border)",
|
|
951
1033
|
borderRadius: 10,
|
|
952
1034
|
backgroundColor: "var(--panel-2)",
|
|
953
|
-
maxWidth: "78%",
|
|
954
1035
|
},
|
|
955
1036
|
judgeResponseHeading: {
|
|
956
1037
|
margin: 0,
|
|
@@ -1044,7 +1125,7 @@ const styles = stylex.create({
|
|
|
1044
1125
|
},
|
|
1045
1126
|
criteriaList: { display: "grid", gap: "0.55rem", margin: 0, padding: 0, listStyle: "none" },
|
|
1046
1127
|
criterion: { display: "flex", gap: "0.6rem", color: "var(--muted)", fontSize: "0.82rem" },
|
|
1047
|
-
criterionIcon: { color: "var(--
|
|
1128
|
+
criterionIcon: { color: "var(--subtle)", fontWeight: 800 },
|
|
1048
1129
|
passReasonList: { display: "grid", gap: "0.35rem", margin: 0, padding: 0, listStyle: "none" },
|
|
1049
1130
|
passReason: { display: "flex", gap: "0.5rem", color: "var(--text)", fontSize: "0.76rem" },
|
|
1050
1131
|
passReasonIcon: { color: "var(--pass)", fontWeight: 800 },
|
|
@@ -1170,9 +1251,6 @@ function useNarrowViewer() {
|
|
|
1170
1251
|
}, []);
|
|
1171
1252
|
return narrow;
|
|
1172
1253
|
}
|
|
1173
|
-
function useCatalog() {
|
|
1174
|
-
return useQuery({ queryKey: ["test-catalog"], queryFn: fetchTestCatalog });
|
|
1175
|
-
}
|
|
1176
1254
|
function MobileViewerHeader({ buttonRef, open, onOpen, }) {
|
|
1177
1255
|
const catalog = useCatalog();
|
|
1178
1256
|
return (_jsxs("header", { ...stylex.props(styles.mobileHeader), children: [_jsx("button", { ref: buttonRef, type: "button", "aria-haspopup": "dialog", "aria-controls": "mobile-test-drawer", "aria-expanded": open, "aria-label": "Open test menu", ...stylex.props(styles.buttonReset, styles.mobileMenuButton), onClick: onOpen, children: _jsx("span", { "aria-hidden": "true", children: "\u2630" }) }), _jsxs("div", { ...stylex.props(styles.mobileHeaderIdentity), children: [_jsx("span", { ...stylex.props(styles.mobileHeaderTitle), children: "Tests" }), _jsx("span", { ...stylex.props(styles.countBadge), children: catalog.data?.tests.length ?? 0 })] })] }));
|
|
@@ -1227,7 +1305,7 @@ function SidebarHeader({ mode, open, count, headingId, navigationId, onToggle, o
|
|
|
1227
1305
|
onNavigate?.();
|
|
1228
1306
|
},
|
|
1229
1307
|
});
|
|
1230
|
-
return (_jsx("header", { ...stylex.props(styles.sidebarHeader, !open && styles.sidebarHeaderCollapsed), children: open ? (_jsxs(_Fragment, { children: [_jsxs("div", { ...stylex.props(styles.sidebarTitleRow), children: [_jsxs("div", { ...stylex.props(styles.sidebarTitleIdentity), children: [_jsx("h1", { id: headingId, ...stylex.props(styles.title, styles.sidebarTitle), children: "Tests" }), _jsx("span", { ...stylex.props(styles.countBadge), children: count })] }), _jsx(SidebarToggle, { mode: mode, open: open, navigationId: navigationId, onToggle: onToggle })] }), _jsxs("div", { ...stylex.props(styles.suiteControls), children: [_jsxs("button", { type: "button", ...stylex.props(styles.buttonReset, styles.suiteRunButton), onClick: () => run.mutate(), disabled: run.isPending || count === 0, children: [_jsx("span", { "aria-hidden": "true", children: run.isPending ? "●" : "▶" }), run.isPending ? "Starting suite…" : "Run all tests"] }), _jsx(WorkerControl, {})] }), run.isError ? _jsx("p", { role: "alert", children: run.error.message }) : null] })) : (_jsx(SidebarToggle, { mode: mode, open: open, navigationId: navigationId, onToggle: onToggle })) }));
|
|
1308
|
+
return (_jsx("header", { ...stylex.props(styles.sidebarHeader, !open && styles.sidebarHeaderCollapsed), children: open ? (_jsxs(_Fragment, { children: [_jsxs("div", { ...stylex.props(styles.sidebarTitleRow), children: [_jsxs("div", { ...stylex.props(styles.sidebarTitleIdentity), children: [_jsx("h1", { id: headingId, ...stylex.props(styles.title, styles.sidebarTitle), children: "Tests" }), _jsx("span", { ...stylex.props(styles.countBadge), children: count })] }), _jsx(SidebarToggle, { mode: mode, open: open, navigationId: navigationId, onToggle: onToggle })] }), _jsxs(Link, { to: "/", activeOptions: { exact: true }, ...stylex.props(styles.navLink, styles.runsLink), activeProps: stylex.props(styles.navLink, styles.runsLink, styles.active), onClick: onNavigate, children: [_jsx("span", { "aria-hidden": "true", children: "\u25F7" }), " Runs"] }), _jsxs("div", { ...stylex.props(styles.suiteControls), children: [_jsxs("button", { type: "button", ...stylex.props(styles.buttonReset, styles.suiteRunButton), onClick: () => run.mutate(), disabled: run.isPending || count === 0, children: [_jsx("span", { "aria-hidden": "true", children: run.isPending ? "●" : "▶" }), run.isPending ? "Starting suite…" : "Run all tests"] }), _jsx(WorkerControl, {})] }), run.isError ? _jsx("p", { role: "alert", children: run.error.message }) : null] })) : (_jsx(SidebarToggle, { mode: mode, open: open, navigationId: navigationId, onToggle: onToggle })) }));
|
|
1231
1309
|
}
|
|
1232
1310
|
function WorkerControl() {
|
|
1233
1311
|
const { workers, setWorkers } = useContext(WorkerContext);
|
|
@@ -1264,9 +1342,6 @@ function TestLink({ test, status, onNavigate, }) {
|
|
|
1264
1342
|
function TestStatus({ status }) {
|
|
1265
1343
|
return (_jsx("span", { role: "img", "aria-label": `Latest execution: ${status}`, title: `Latest execution: ${status}`, ...stylex.props(styles.navStatus), children: _jsx("span", { "aria-hidden": "true", ...stylex.props(styles.navStatusDot, status === "passed" && styles.navStatusDotPass, status === "failed" && styles.navStatusDotFail, status === "running" && styles.navStatusDotRunning) }) }));
|
|
1266
1344
|
}
|
|
1267
|
-
function TestCatalogPage() {
|
|
1268
|
-
return (_jsxs("section", { ...stylex.props(styles.hero), children: [_jsx("div", { ...stylex.props(styles.brand), children: "Test explorer" }), _jsx("h2", { ...stylex.props(styles.title), children: "Choose a test from the sidebar" }), _jsx("p", { ...stylex.props(styles.description), children: "Read what it checks, see its agents and judges, then start a run or inspect a saved execution." })] }));
|
|
1269
|
-
}
|
|
1270
1345
|
function TestPage() {
|
|
1271
1346
|
const { testId } = testRoute.useParams();
|
|
1272
1347
|
const catalog = useCatalog();
|
|
@@ -1282,7 +1357,7 @@ function TestDetail({ test }) {
|
|
|
1282
1357
|
const history = useExecutionHistory();
|
|
1283
1358
|
const navigate = useNavigate();
|
|
1284
1359
|
const runs = history.data?.filter((item) => item.testIds?.includes(test.id)) ?? [];
|
|
1285
|
-
const selected = execution ??
|
|
1360
|
+
const selected = execution ?? preferredTestExecutionId(runs, test.id);
|
|
1286
1361
|
const activeView = view ?? "current";
|
|
1287
1362
|
const chooseExecution = (id) => void navigate({
|
|
1288
1363
|
to: "/tests/$testId",
|
|
@@ -1294,7 +1369,14 @@ function TestDetail({ test }) {
|
|
|
1294
1369
|
params: { testId: test.id },
|
|
1295
1370
|
search: { execution, view: next === "current" ? undefined : next },
|
|
1296
1371
|
});
|
|
1297
|
-
return (_jsxs(_Fragment, { children: [_jsx(TestHeader, { test: test, onStarted: chooseExecution }), _jsx(TestTabs, { active: activeView, onChange: chooseView }), _jsx(TestViewPanel, { view: activeView, test: test, runs: runs, selected: selected, onChooseExecution: chooseExecution })] }));
|
|
1372
|
+
return (_jsxs(_Fragment, { children: [_jsx(TestHeader, { test: test, onStarted: chooseExecution }), _jsx(BatchRunLink, { run: runs.find((run) => run.id === selected) }), _jsx(TestTabs, { active: activeView, onChange: chooseView }), _jsx(TestViewPanel, { view: activeView, test: test, runs: runs, selected: selected, historyLoading: history.isPending, onChooseExecution: chooseExecution })] }));
|
|
1373
|
+
}
|
|
1374
|
+
/** Return path to the run inbox when this test was opened from a batch run. */
|
|
1375
|
+
function BatchRunLink({ run }) {
|
|
1376
|
+
const count = run?.testIds?.length ?? 0;
|
|
1377
|
+
if (!run || count < 2)
|
|
1378
|
+
return null;
|
|
1379
|
+
return (_jsxs(Link, { to: "/executions/$executionId", params: { executionId: run.id }, ...stylex.props(styles.batchRunLink), children: ["\u2190 Back to run \u00B7 ", formatRunDate(run.startedAt), " \u00B7 ", count, " tests"] }));
|
|
1298
1380
|
}
|
|
1299
1381
|
function TestHeader({ test, onStarted }) {
|
|
1300
1382
|
return (_jsxs("header", { ...stylex.props(styles.hero), children: [_jsxs("div", { ...stylex.props(styles.heroTop), children: [_jsxs("div", { ...stylex.props(styles.heroCopy), children: [_jsx("div", { ...stylex.props(styles.brand), children: test.title.slice(0, -1).join(" / ") }), _jsx("h2", { ...stylex.props(styles.title), children: test.title.at(-1) }), _jsx("p", { ...stylex.props(styles.description), children: test.description ?? "This test has no additional description." })] }), _jsx(RunTestButton, { test: test, onStarted: onStarted })] }), _jsxs("div", { ...stylex.props(styles.path), children: [test.project, " \u00B7 ", test.file] })] }));
|
|
@@ -1324,16 +1406,21 @@ function handleTabKeyDown(event, active, onChange) {
|
|
|
1324
1406
|
document.getElementById(`test-tab-${next.id}`)?.focus();
|
|
1325
1407
|
}
|
|
1326
1408
|
function TestViewPanel(props) {
|
|
1327
|
-
return (_jsxs("div", { id: `test-panel-${props.view}`, role: "tabpanel", "aria-labelledby": `test-tab-${props.view}`, ...stylex.props(styles.tabPanel), children: [props.view === "setup" ? _jsx(TestResources, { test: props.test }) : null, props.view === "current" ? (_jsx(CurrentRun, { test: props.test, executionId: props.selected })) : null, props.view === "history" ? (_jsx(RunHistory, { runs: props.runs, onChoose: props.onChooseExecution })) : null] }));
|
|
1409
|
+
return (_jsxs("div", { id: `test-panel-${props.view}`, role: "tabpanel", "aria-labelledby": `test-tab-${props.view}`, ...stylex.props(styles.tabPanel), children: [props.view === "setup" ? _jsx(TestResources, { test: props.test }) : null, props.view === "current" ? (_jsx(CurrentRun, { test: props.test, executionId: props.selected, loading: props.historyLoading })) : null, props.view === "history" ? (_jsx(RunHistory, { runs: props.runs, testId: props.test.id, onChoose: props.onChooseExecution })) : null] }));
|
|
1410
|
+
}
|
|
1411
|
+
function CurrentRun({ test, executionId, loading, }) {
|
|
1412
|
+
return (_jsx("section", { ...stylex.props(styles.tabSection), children: loading && !executionId ? (_jsx("p", { ...stylex.props(styles.empty), children: "Loading runs\u2026" })) : executionId ? (_jsx(TestExecution, { id: executionId, test: test })) : (_jsx("p", { ...stylex.props(styles.empty), children: "No runs yet. Start this test to see its progress here." })) }));
|
|
1328
1413
|
}
|
|
1329
|
-
function
|
|
1330
|
-
return (
|
|
1414
|
+
function RunHistory({ runs, testId, onChoose, }) {
|
|
1415
|
+
return (_jsxs("section", { ...stylex.props(styles.tabSection), children: [_jsx("div", { ...stylex.props(styles.quietHeader), children: _jsxs("span", { ...stylex.props(styles.sectionIntro), children: [runs.length, " ", pluralize(runs.length, "run"), " included this test. Each shows this test's own result."] }) }), runs.length ? (_jsx("div", { ...stylex.props(styles.historyList), children: runs.map((run) => (_jsx(HistoryItem, { run: run, testId: testId, onChoose: onChoose }, run.id))) })) : (_jsx("p", { ...stylex.props(styles.empty), children: "No saved runs for this test." }))] }));
|
|
1331
1416
|
}
|
|
1332
|
-
function
|
|
1333
|
-
|
|
1417
|
+
function HistoryItem({ run, testId, onChoose, }) {
|
|
1418
|
+
const status = testRunStatus(run, testId);
|
|
1419
|
+
const others = (run.testIds?.length ?? 1) - 1;
|
|
1420
|
+
return (_jsxs("button", { type: "button", "aria-label": `Open ${status} execution ${run.id.slice(0, 8)}`, ...stylex.props(styles.buttonReset, styles.historyButton), onClick: () => onChoose(run.id), children: [_jsxs("span", { ...stylex.props(styles.historyIdentity), children: [_jsx("span", { ...stylex.props(styles.historyDate), children: formatRunDate(run.startedAt) }), _jsxs("span", { ...stylex.props(styles.historyId), children: [run.id.slice(0, 8), others > 0 ? ` · with ${others} other ${pluralize(others, "test")}` : ""] })] }), _jsx(StatusPill, { status: status })] }));
|
|
1334
1421
|
}
|
|
1335
|
-
function
|
|
1336
|
-
return (
|
|
1422
|
+
function StatusPill({ status }) {
|
|
1423
|
+
return (_jsx("span", { ...stylex.props(styles.status, status === "passed" && styles.statusPass, status === "failed" && styles.statusFail, (status === "running" || status === "queued") && styles.statusRunning), children: status }));
|
|
1337
1424
|
}
|
|
1338
1425
|
function RunTestButton({ test, onStarted }) {
|
|
1339
1426
|
const { workers } = useContext(WorkerContext);
|
|
@@ -1355,7 +1442,7 @@ function TestResources({ test }) {
|
|
|
1355
1442
|
}
|
|
1356
1443
|
function CriteriaSetup({ test }) {
|
|
1357
1444
|
const criteria = testCriteria(test);
|
|
1358
|
-
return (_jsxs("section", { "aria-labelledby": "setup-criteria", ...stylex.props(styles.criteriaPanel), children: [_jsx("h3", { id: "setup-criteria", ...stylex.props(styles.contentHeading), children: "Pass criteria" }), criteria.length ? (_jsx("ul", { ...stylex.props(styles.criteriaList), children: criteria.map((criterion) => (_jsxs("li", { ...stylex.props(styles.criterion), children: [_jsx("span", { "aria-hidden": "true", ...stylex.props(styles.criterionIcon), children: "\
|
|
1445
|
+
return (_jsxs("section", { "aria-labelledby": "setup-criteria", ...stylex.props(styles.criteriaPanel), children: [_jsx("h3", { id: "setup-criteria", ...stylex.props(styles.contentHeading), children: "Pass criteria" }), _jsx("p", { ...stylex.props(styles.sectionIntro), children: "Every one of these must hold for the test to pass." }), criteria.length ? (_jsx("ul", { ...stylex.props(styles.criteriaList), children: criteria.map((criterion) => (_jsxs("li", { ...stylex.props(styles.criterion), children: [_jsx("span", { "aria-hidden": "true", ...stylex.props(styles.criterionIcon), children: "\u2022" }), _jsx("span", { children: criterion })] }, criterion))) })) : (_jsx("p", { ...stylex.props(styles.meta), children: "No explicit pass criteria are declared." }))] }));
|
|
1359
1446
|
}
|
|
1360
1447
|
function ResourceSection({ title, description, resources, }) {
|
|
1361
1448
|
const id = `setup-${title.toLowerCase()}`;
|
|
@@ -1366,7 +1453,7 @@ function ResourceCard({ resource }) {
|
|
|
1366
1453
|
return (_jsxs("article", { ...stylex.props(styles.card, styles.resourceCard), children: [_jsx("div", { ...stylex.props(styles.resourceHeader), children: _jsx("strong", { children: resource.name }) }), _jsx(ResourceContext, { resource: resource })] }));
|
|
1367
1454
|
}
|
|
1368
1455
|
function WorkspaceSetup({ test }) {
|
|
1369
|
-
return (_jsxs("section", { "aria-labelledby": "setup-workspace", ...stylex.props(styles.setupSection), children: [_jsxs("div", { ...stylex.props(styles.setupSectionCopy), children: [_jsx("h3", { id: "setup-workspace", ...stylex.props(styles.contentHeading), children: "Workspace" }), _jsx("p", { ...stylex.props(styles.meta), children: "
|
|
1456
|
+
return (_jsxs("section", { "aria-labelledby": "setup-workspace", ...stylex.props(styles.setupSection), children: [_jsxs("div", { ...stylex.props(styles.setupSectionCopy), children: [_jsx("h3", { id: "setup-workspace", ...stylex.props(styles.contentHeading), children: "Workspace" }), _jsx("p", { ...stylex.props(styles.meta), children: "Each agent run starts from a fresh copy of this folder." })] }), _jsx("div", { ...stylex.props(styles.workspaceRow), children: _jsx(ResourceFact, { label: "Source folder", value: test.workspace ?? "Project directory" }) })] }));
|
|
1370
1457
|
}
|
|
1371
1458
|
function testCriteria(test) {
|
|
1372
1459
|
if (test.criteria?.length)
|
|
@@ -1391,19 +1478,20 @@ function TestExecution({ id, test }) {
|
|
|
1391
1478
|
const attempts = detail.data.attempts.filter((attempt) => attempt.testId === test.id);
|
|
1392
1479
|
if (detail.data.testIds && !detail.data.testIds.includes(test.id))
|
|
1393
1480
|
return _jsx("p", { children: "This execution does not contain this test." });
|
|
1394
|
-
|
|
1481
|
+
if (attempts.length === 0 && testRunStatus(detail.data, test.id) === "skipped")
|
|
1482
|
+
return (_jsx("p", { ...stylex.props(styles.empty), children: "This test was skipped in this run. Open Run history to find a run that executed it." }));
|
|
1483
|
+
return (_jsx(AgentDescriptions.Provider, { value: agentDescriptions(test), children: _jsx(ExecutionDetailView, { execution: { ...detail.data, attempts }, criteria: testCriteria(test) }) }));
|
|
1484
|
+
}
|
|
1485
|
+
/** What each agent in the selected test is for, from the test's own `agent({ description })`. */
|
|
1486
|
+
const AgentDescriptions = createContext(new Map());
|
|
1487
|
+
function agentDescriptions(test) {
|
|
1488
|
+
return new Map((test.resources ?? []).flatMap((resource) => resource.kind === "agent" && resource.description
|
|
1489
|
+
? [[resource.name, resource.description]]
|
|
1490
|
+
: []));
|
|
1395
1491
|
}
|
|
1396
|
-
function
|
|
1492
|
+
function ExecutionRunPage() {
|
|
1397
1493
|
const { executionId } = executionRoute.useParams();
|
|
1398
|
-
|
|
1399
|
-
queryKey: ["execution", executionId],
|
|
1400
|
-
queryFn: () => fetchExecution(executionId),
|
|
1401
|
-
});
|
|
1402
|
-
if (detail.isLoading)
|
|
1403
|
-
return _jsx("p", { ...stylex.props(styles.empty), children: "Loading execution\u2026" });
|
|
1404
|
-
if (detail.isError || !detail.data)
|
|
1405
|
-
return _jsx("p", { ...stylex.props(styles.empty), children: "Execution is unavailable." });
|
|
1406
|
-
return _jsx(ExecutionDetailView, { execution: detail.data });
|
|
1494
|
+
return _jsx(RunPage, { executionId: executionId }, executionId);
|
|
1407
1495
|
}
|
|
1408
1496
|
function ExecutionDetailView({ execution, criteria = [], }) {
|
|
1409
1497
|
const client = useQueryClient();
|
|
@@ -1413,7 +1501,7 @@ function ExecutionDetailView({ execution, criteria = [], }) {
|
|
|
1413
1501
|
});
|
|
1414
1502
|
const stopping = execution.status === "running" && (cancel.isPending || cancel.isSuccess);
|
|
1415
1503
|
const stopLabel = (execution.testIds?.length ?? 0) > 1 ? "Stop suite run" : "Stop this run";
|
|
1416
|
-
return (_jsxs("div", { ...stylex.props(styles.statusPanel), children: [execution.status === "running" && execution.cancellable ? (_jsx("div", { ...stylex.props(styles.runControls), children: _jsx("button", { type: "button", ...stylex.props(styles.buttonReset, styles.button, styles.secondaryButton), onClick: () => cancel.mutate(), disabled: stopping, children: stopping ? "Stopping…" : stopLabel }) })) : null, cancel.isError ? _jsx("p", { role: "alert", children: "The run could not be stopped." }) : null, _jsxs("section", { ...stylex.props(styles.list), children: [execution.attempts.map((attempt) => (_jsx(AttemptCard, { attempt: attempt, criteria: criteria }, attempt.id))), execution.attempts.length === 0 ? (_jsx("p", { ...stylex.props(styles.empty), children: emptyExecutionMessage(execution, stopping) })) : null] })] }));
|
|
1504
|
+
return (_jsxs("div", { ...stylex.props(styles.statusPanel), children: [execution.status === "running" && execution.cancellable ? (_jsx("div", { ...stylex.props(styles.runControls), children: _jsx("button", { type: "button", ...stylex.props(styles.buttonReset, styles.button, styles.secondaryButton), onClick: () => cancel.mutate(), disabled: stopping, children: stopping ? "Stopping…" : stopLabel }) })) : null, cancel.isError ? _jsx("p", { role: "alert", children: "The run could not be stopped." }) : null, _jsxs("section", { ...stylex.props(styles.list), children: [execution.attempts.map((attempt) => (_jsx(AttemptCard, { attempt: attempt, criteria: criteria, showTitle: false }, attempt.id))), execution.attempts.length === 0 ? (_jsx("p", { ...stylex.props(styles.empty), children: emptyExecutionMessage(execution, stopping) })) : null] })] }));
|
|
1417
1505
|
}
|
|
1418
1506
|
function emptyExecutionMessage(execution, stopping) {
|
|
1419
1507
|
if (stopping)
|
|
@@ -1444,19 +1532,113 @@ function ComparePage() {
|
|
|
1444
1532
|
function ComparisonColumn({ execution, label }) {
|
|
1445
1533
|
return (_jsxs("article", { ...stylex.props(styles.card), children: [_jsxs("strong", { children: [label, ": ", execution.status] }), _jsx("p", { ...stylex.props(styles.meta), children: execution.id.slice(0, 8) }), execution.attempts.map((attempt) => (_jsx(AttemptCard, { attempt: attempt }, attempt.id)))] }));
|
|
1446
1534
|
}
|
|
1447
|
-
|
|
1535
|
+
/** `showTitle` is off on a test page, whose header already names the test. */
|
|
1536
|
+
function AttemptCard({ attempt, criteria = [], showTitle = true, }) {
|
|
1448
1537
|
const { operations, output, title } = attemptPresentation(attempt, criteria);
|
|
1449
1538
|
const agents = operations.filter((operation) => operation.kind !== "evaluation");
|
|
1450
1539
|
const judges = operations.filter((operation) => operation.kind === "evaluation");
|
|
1451
|
-
return (_jsxs("article", { ...stylex.props(styles.attemptCard), children: [_jsxs("div", { ...stylex.props(styles.row), children: [_jsxs("div", { children: [_jsx("strong", { ...stylex.props(styles.attemptTitle), children: title }), _jsxs("p", { ...stylex.props(styles.meta), children: [attempt.project || "default", " \u00B7 ", attempt.operations.length, " named", " ", pluralize(attempt.operations.length, "operation")] })] }), _jsxs("div", { ...stylex.props(styles.attemptActions), children: [attempt.status === "failed" ? (_jsx(CopyFailureButton, { attempt: attempt, criteria: criteria })) : null, _jsx("span", { ...stylex.props(styles.status, attempt.status === "passed" && styles.statusPass, attempt.status === "failed" && styles.statusFail, attempt.status === "running" && styles.statusRunning), children: attempt.status })] })] }), _jsx(OperationList, { operations: operations }), _jsx(ConversationSection, { label: "
|
|
1540
|
+
return (_jsxs("article", { ...stylex.props(styles.attemptCard), children: [_jsxs("div", { ...stylex.props(styles.row), children: [_jsxs("div", { children: [showTitle ? _jsx("strong", { ...stylex.props(styles.attemptTitle), children: title }) : null, _jsxs("p", { ...stylex.props(styles.meta), children: [attempt.project || "default", " \u00B7 ", attempt.operations.length, " named", " ", pluralize(attempt.operations.length, "operation")] })] }), _jsxs("div", { ...stylex.props(styles.attemptActions), children: [attempt.status === "failed" ? (_jsx(CopyFailureButton, { attempt: attempt, criteria: criteria })) : null, _jsx("span", { ...stylex.props(styles.status, attempt.status === "passed" && styles.statusPass, attempt.status === "failed" && styles.statusFail, attempt.status === "running" && styles.statusRunning), children: attempt.status })] })] }), _jsx(TestVerdict, { attempt: attempt, criteria: criteria, operations: operations }), _jsx(OperationList, { operations: operations }), _jsx(ConversationSection, { label: "Judges", sectionOperations: judges, allOperations: operations, attempt: attempt }), _jsx(ConversationSection, { label: "Agents", sectionOperations: agents, allOperations: operations, attempt: attempt }), _jsx(TechnicalDetails, { operations: operations, output: output })] }));
|
|
1541
|
+
}
|
|
1542
|
+
/** Test-level outcome first: every check with its result, including judge-decided checks. */
|
|
1543
|
+
function TestVerdict({ attempt, criteria, operations, }) {
|
|
1544
|
+
const presentations = criterionPresentations({
|
|
1545
|
+
criteria,
|
|
1546
|
+
explanations: judgeCriterionExplanations(criteria, operations),
|
|
1547
|
+
results: attempt.criterionResults,
|
|
1548
|
+
status: attempt.status,
|
|
1549
|
+
errors: attempt.errors,
|
|
1550
|
+
});
|
|
1551
|
+
return (_jsxs("section", { "aria-label": "Test verdict", ...stylex.props(styles.verdictPanel, attempt.status === "passed" && styles.verdictPass, attempt.status === "failed" && styles.verdictFail), children: [_jsxs("p", { ...stylex.props(styles.verdictHeadline), children: [capitalize(attempt.status), _jsx("span", { ...stylex.props(styles.verdictCount), children: checkSummary(presentations, attempt.status) })] }), attempt.status === "running" ? (_jsx("p", { ...stylex.props(styles.meta), children: "The test is still running." })) : null, _jsx(VerdictMetrics, { operations: operations, reported: attempt.metrics }), _jsx(VerdictChecks, { presentations: presentations }), _jsx(UnassignedFailures, { errors: attempt.errors, presentations: presentations }), _jsx(VerdictAttachments, { attachments: attempt.attachments })] }));
|
|
1552
|
+
}
|
|
1553
|
+
const SHORT_CHECK_LIST = 3;
|
|
1554
|
+
/** Checks needing attention always show; passing checks fold away once the list is long. */
|
|
1555
|
+
function VerdictChecks({ presentations }) {
|
|
1556
|
+
const checks = presentations;
|
|
1557
|
+
if (checks.length <= SHORT_CHECK_LIST)
|
|
1558
|
+
return _jsx(CriterionResults, { presentations: checks });
|
|
1559
|
+
const attention = checks.filter((check) => check.outcome !== "passed");
|
|
1560
|
+
const passed = checks.filter((check) => check.outcome === "passed");
|
|
1561
|
+
return (_jsxs(_Fragment, { children: [_jsx(CriterionResults, { presentations: attention }), passed.length ? (_jsxs("details", { "data-disclosure": true, ...stylex.props(styles.verdictPassedChecks), children: [_jsxs("summary", { ...stylex.props(styles.judgeContextSummary), children: [passed.length, " passed ", pluralize(passed.length, "check")] }), _jsx(CriterionResults, { presentations: passed, label: "Passed checks" })] })) : null] }));
|
|
1562
|
+
}
|
|
1563
|
+
/** Failures no check claimed, such as a crash before the first assertion. */
|
|
1564
|
+
function UnassignedFailures({ errors, presentations, }) {
|
|
1565
|
+
const assigned = new Set(presentations.map((item) => item.failure).filter(Boolean));
|
|
1566
|
+
return errors
|
|
1567
|
+
.filter((error) => !assigned.has(error.message))
|
|
1568
|
+
.map((error, index) => _jsx(FailureDetails, { message: error.message ?? "Test failed" }, index));
|
|
1569
|
+
}
|
|
1570
|
+
const VERDICT_METRICS = [
|
|
1571
|
+
{ key: "durationMs", label: "Duration", format: formatDuration },
|
|
1572
|
+
{ key: "totalTokens", label: "Tokens", format: (value) => Math.round(value).toLocaleString() },
|
|
1573
|
+
{ key: "toolCallCount", label: "Tool calls", format: (value) => String(Math.round(value)) },
|
|
1574
|
+
{
|
|
1575
|
+
key: "changedPathCount",
|
|
1576
|
+
label: "Files changed",
|
|
1577
|
+
format: (value) => String(Math.round(value)),
|
|
1578
|
+
},
|
|
1579
|
+
];
|
|
1580
|
+
/** `p95LatencyMs` → "P95 latency"; numbers keyed `…Ms` read as durations. */
|
|
1581
|
+
function reportedMetricRows(reported) {
|
|
1582
|
+
return Object.entries(reported ?? {}).map(([key, value]) => {
|
|
1583
|
+
const duration = key.endsWith("Ms");
|
|
1584
|
+
const words = (duration ? key.slice(0, -2) : key).replace(/([a-z0-9])([A-Z])/g, "$1 $2");
|
|
1585
|
+
return {
|
|
1586
|
+
key: `reported:${key}`,
|
|
1587
|
+
label: capitalize(words.toLowerCase()),
|
|
1588
|
+
value: duration
|
|
1589
|
+
? formatDuration(Math.round(value))
|
|
1590
|
+
: value.toLocaleString(undefined, { maximumFractionDigits: 2 }),
|
|
1591
|
+
};
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
/** Agent totals for the attempt; a median per run when the test made several agent runs. */
|
|
1595
|
+
function VerdictMetrics({ operations, reported, }) {
|
|
1596
|
+
const runs = operations
|
|
1597
|
+
.filter((operation) => operation.kind !== "evaluation")
|
|
1598
|
+
.map((operation) => operationResult([operation]));
|
|
1599
|
+
const rows = VERDICT_METRICS.flatMap((metric) => {
|
|
1600
|
+
const values = runs.flatMap((run) => run[metric.key] === undefined ? [] : [run[metric.key] ?? 0]);
|
|
1601
|
+
if (!values.length)
|
|
1602
|
+
return [];
|
|
1603
|
+
const total = values.reduce((sum, value) => sum + value, 0);
|
|
1604
|
+
return [{ ...metric, total, median: median(values) ?? total }];
|
|
1605
|
+
});
|
|
1606
|
+
const reportedRows = reportedMetricRows(reported);
|
|
1607
|
+
if (!rows.length && !reportedRows.length)
|
|
1608
|
+
return null;
|
|
1609
|
+
const repeated = runs.length > 1;
|
|
1610
|
+
return (_jsxs("dl", { "aria-label": "Run metrics", ...stylex.props(styles.verdictMetrics), children: [rows.map((row) => (_jsxs("div", { ...stylex.props(styles.verdictMetric), children: [_jsx("dt", { ...stylex.props(styles.resourceFactLabel), children: row.label }), _jsxs("dd", { ...stylex.props(styles.verdictMetricValue), children: [row.format(row.total), repeated ? (_jsxs("span", { ...stylex.props(styles.verdictMetricMedian), children: ["median ", row.format(row.median), " per run"] })) : null] })] }, row.key))), reportedRows.map((row) => (_jsxs("div", { ...stylex.props(styles.verdictMetric), children: [_jsx("dt", { ...stylex.props(styles.resourceFactLabel), children: row.label }), _jsx("dd", { ...stylex.props(styles.verdictMetricValue), children: row.value })] }, row.key)))] }));
|
|
1611
|
+
}
|
|
1612
|
+
/** Files the test attached with a path, opened from the viewer server. */
|
|
1613
|
+
function VerdictAttachments({ attachments }) {
|
|
1614
|
+
if (!attachments?.length)
|
|
1615
|
+
return null;
|
|
1616
|
+
return (_jsx("ul", { "aria-label": "Attachments", ...stylex.props(styles.verdictAttachments), children: attachments.map((attachment) => (_jsxs("li", { children: [_jsx("a", { href: attachment.url, target: "_blank", rel: "noreferrer", children: attachment.name }), _jsxs("span", { ...stylex.props(styles.verdictMetricMedian), children: [" ", attachment.contentType] })] }, attachment.url))) }));
|
|
1617
|
+
}
|
|
1618
|
+
function checkSummary(presentations, status) {
|
|
1619
|
+
const total = presentations.length;
|
|
1620
|
+
if (!total)
|
|
1621
|
+
return "";
|
|
1622
|
+
const failed = presentations.filter((item) => item.outcome === "failed").length;
|
|
1623
|
+
const passed = presentations.filter((item) => item.outcome === "passed").length;
|
|
1624
|
+
const checks = pluralize(total, "check");
|
|
1625
|
+
if (failed)
|
|
1626
|
+
return ` · ${failed} of ${total} ${checks} failed`;
|
|
1627
|
+
return ` · ${passed} of ${total} ${checks} passed${status === "running" ? " so far" : ""}`;
|
|
1628
|
+
}
|
|
1629
|
+
function judgeCriterionExplanations(criteria, operations) {
|
|
1630
|
+
const judges = operations.filter((operation) => operation.kind === "evaluation");
|
|
1631
|
+
return criteria.map((_, index) => judges
|
|
1632
|
+
.map((judge) => operationCriterionExplanations(criteria, judge)[index])
|
|
1633
|
+
.find((explanation) => explanation !== undefined));
|
|
1452
1634
|
}
|
|
1453
|
-
function ConversationSection({ label, sectionOperations, allOperations, attempt,
|
|
1635
|
+
function ConversationSection({ label, sectionOperations, allOperations, attempt, }) {
|
|
1454
1636
|
const [selectedId, setSelectedId] = useState(sectionOperations[0]?.id);
|
|
1455
1637
|
if (!sectionOperations.length && (allOperations.length > 0 || label === "Judges"))
|
|
1456
1638
|
return null;
|
|
1457
1639
|
const selected = sectionOperations.find((operation) => operation.id === selectedId) ?? sectionOperations[0];
|
|
1458
1640
|
const status = attempt.status === "running" ? (selected?.status ?? attempt.status) : attempt.status;
|
|
1459
|
-
return (_jsx(Conversation, { label: label, operation: selected, operations: sectionOperations, allOperations: allOperations, select: setSelectedId, status: status, attemptStatus: attempt.status, result: _jsx(AttemptResult, {
|
|
1641
|
+
return (_jsx(Conversation, { label: label, operation: selected, operations: sectionOperations, allOperations: allOperations, select: setSelectedId, status: status, attemptStatus: attempt.status, result: _jsx(AttemptResult, { result: selectedOperationResult(selected) }) }));
|
|
1460
1642
|
}
|
|
1461
1643
|
function attemptPresentation(attempt, criteria) {
|
|
1462
1644
|
return {
|
|
@@ -1500,21 +1682,9 @@ function operationInvocationIndex(operation) {
|
|
|
1500
1682
|
return [value.invocationIndex];
|
|
1501
1683
|
})[0];
|
|
1502
1684
|
}
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
const indexes = selectedCriterionIndexes({ criteria, operation, operations, includeShared });
|
|
1507
|
-
return (_jsx(RunResult, { result: result, status: attempt.status, criteria: selectIndexes(criteria, indexes), explanations: selectOptionalIndexes(operationCriterionExplanations(criteria, operation), indexes), errors: attempt.errors, criterionResults: selectIndexes(attempt.criterionResults, indexes) }));
|
|
1508
|
-
}
|
|
1509
|
-
function selectedCriterionIndexes(input) {
|
|
1510
|
-
const { criteria, operation, operations, includeShared } = input;
|
|
1511
|
-
const recorded = operationCriterionIndexes(criteria, operation);
|
|
1512
|
-
const allRecorded = new Set(operations.flatMap((item) => operationCriterionIndexes(criteria, item)));
|
|
1513
|
-
if (allRecorded.size)
|
|
1514
|
-
return criteria.flatMap((_, index) => recorded.includes(index) || (includeShared && !allRecorded.has(index)) ? [index] : []);
|
|
1515
|
-
if (!includeShared)
|
|
1516
|
-
return [];
|
|
1517
|
-
return inferredRepeatedRunCriterionIndexes(criteria, operation, operations);
|
|
1685
|
+
/** Per-run metrics for comparing runs. Totals and checks live in the test verdict. */
|
|
1686
|
+
function AttemptResult({ result }) {
|
|
1687
|
+
return _jsx(RunResult, { result: result });
|
|
1518
1688
|
}
|
|
1519
1689
|
function operationCriterionIndexes(criteria, operation) {
|
|
1520
1690
|
return [
|
|
@@ -1534,19 +1704,6 @@ function criterionRecordIndexes(criteria, value) {
|
|
|
1534
1704
|
}
|
|
1535
1705
|
const RUN_ORDINALS = ["first", "second", "third", "fourth", "fifth"];
|
|
1536
1706
|
const CRITERION_WORD_SEPARATOR = /[^a-z0-9]+/;
|
|
1537
|
-
function inferredRepeatedRunCriterionIndexes(criteria, operation, operations) {
|
|
1538
|
-
if (!operation)
|
|
1539
|
-
return undefined;
|
|
1540
|
-
const runs = operations.filter((item) => operationIdentity(item) === operationIdentity(operation));
|
|
1541
|
-
if (runs.length < 2)
|
|
1542
|
-
return undefined;
|
|
1543
|
-
const selected = inferredOperationOrdinal(criteria, operation) ??
|
|
1544
|
-
runs.findIndex((run) => run.id === operation.id);
|
|
1545
|
-
const scopes = criteria.map(criterionRunOrdinal);
|
|
1546
|
-
if (selected < 0 || !scopes.some((scope) => scope !== undefined))
|
|
1547
|
-
return undefined;
|
|
1548
|
-
return scopes.flatMap((scope, index) => scope === undefined || scope === selected ? [index] : []);
|
|
1549
|
-
}
|
|
1550
1707
|
function inferredOperationOrdinal(criteria, operation) {
|
|
1551
1708
|
const output = operationResult([operation]).output?.trim().toLowerCase();
|
|
1552
1709
|
if (!output)
|
|
@@ -1571,14 +1728,6 @@ function criterionRunOrdinal(criterion) {
|
|
|
1571
1728
|
const index = RUN_ORDINALS.findIndex((ordinal) => normalized.includes(`${ordinal} `));
|
|
1572
1729
|
return index < 0 ? undefined : index;
|
|
1573
1730
|
}
|
|
1574
|
-
function selectIndexes(values, indexes) {
|
|
1575
|
-
return indexes
|
|
1576
|
-
? indexes.flatMap((index) => (values[index] === undefined ? [] : [values[index]]))
|
|
1577
|
-
: values;
|
|
1578
|
-
}
|
|
1579
|
-
function selectOptionalIndexes(values, indexes) {
|
|
1580
|
-
return indexes ? indexes.map((index) => values[index]) : values;
|
|
1581
|
-
}
|
|
1582
1731
|
function CopyFailureButton({ attempt, criteria }) {
|
|
1583
1732
|
const [state, setState] = useState("idle");
|
|
1584
1733
|
const copy = async () => {
|
|
@@ -1678,6 +1827,7 @@ function toolCallsFromOperation(value) {
|
|
|
1678
1827
|
name: call.name,
|
|
1679
1828
|
args: isRecord(call.args) ? call.args : undefined,
|
|
1680
1829
|
result: optionalString(call.result),
|
|
1830
|
+
images: toolImages(call.images),
|
|
1681
1831
|
succeeded: typeof call.succeeded === "boolean" ? call.succeeded : undefined,
|
|
1682
1832
|
exitCode: optionalNumber(call.exitCode),
|
|
1683
1833
|
seq: optionalNumber(call.seq),
|
|
@@ -1747,6 +1897,7 @@ function liveToolEvent(record, seq) {
|
|
|
1747
1897
|
name: record.name,
|
|
1748
1898
|
args: isRecord(record.args) ? record.args : undefined,
|
|
1749
1899
|
result: optionalString(record.result),
|
|
1900
|
+
images: toolImages(record.images),
|
|
1750
1901
|
exitCode: optionalNumber(record.exitCode),
|
|
1751
1902
|
succeeded: typeof record.succeeded === "boolean" ? record.succeeded : undefined,
|
|
1752
1903
|
seq,
|
|
@@ -1816,18 +1967,8 @@ function nestedNumber(value, parent, child) {
|
|
|
1816
1967
|
function nestedArrayLength(value, key) {
|
|
1817
1968
|
return isRecord(value) ? optionalArrayLength(value[key]) : undefined;
|
|
1818
1969
|
}
|
|
1819
|
-
function RunResult({ result
|
|
1820
|
-
|
|
1821
|
-
criteria,
|
|
1822
|
-
explanations,
|
|
1823
|
-
results: criterionResults,
|
|
1824
|
-
status,
|
|
1825
|
-
errors,
|
|
1826
|
-
});
|
|
1827
|
-
const assignedFailures = new Set(presentations.map((item) => item.failure).filter(Boolean));
|
|
1828
|
-
return (_jsxs("section", { "aria-label": "Result", ...stylex.props(styles.resultPanel), children: [_jsx("h3", { ...stylex.props(styles.contentHeading), children: "Result" }), _jsx(ResultVerdict, { status: status }), _jsx(CriterionResults, { presentations: presentations }), errors
|
|
1829
|
-
.filter((error) => !assignedFailures.has(error.message))
|
|
1830
|
-
.map((error, index) => (_jsx(FailureDetails, { message: error.message ?? "Test failed" }, index))), _jsx(ResultMetrics, { result: result })] }));
|
|
1970
|
+
function RunResult({ result }) {
|
|
1971
|
+
return (_jsxs("section", { "aria-label": "Result", ...stylex.props(styles.resultPanel), children: [_jsx("h3", { ...stylex.props(styles.contentHeading), children: "Result" }), _jsx(ResultMetrics, { result: result })] }));
|
|
1831
1972
|
}
|
|
1832
1973
|
function FailureDetails({ message }) {
|
|
1833
1974
|
const comparison = assertionComparison(message);
|
|
@@ -1838,9 +1979,6 @@ function FailureDetails({ message }) {
|
|
|
1838
1979
|
function AssertionValue({ label, value }) {
|
|
1839
1980
|
return (_jsxs("div", { ...stylex.props(styles.assertionValue), children: [_jsx("dt", { ...stylex.props(styles.assertionLabel), children: label }), _jsx("dd", { ...stylex.props(styles.assertionCode), children: value || "(empty)" })] }));
|
|
1840
1981
|
}
|
|
1841
|
-
function ResultVerdict({ status }) {
|
|
1842
|
-
return status === "running" ? (_jsx("p", { ...stylex.props(styles.resultOutput), children: "The test is still running." })) : null;
|
|
1843
|
-
}
|
|
1844
1982
|
function criterionPresentations({ criteria, explanations, results, status, errors, }) {
|
|
1845
1983
|
const presentations = criteria.map((criterion, index) => ({
|
|
1846
1984
|
criterion,
|
|
@@ -1857,10 +1995,10 @@ function criterionPresentations({ criteria, explanations, results, status, error
|
|
|
1857
1995
|
};
|
|
1858
1996
|
});
|
|
1859
1997
|
}
|
|
1860
|
-
function CriterionResults({ presentations }) {
|
|
1998
|
+
function CriterionResults({ presentations, label = "Criterion results", }) {
|
|
1861
1999
|
if (!presentations.length)
|
|
1862
2000
|
return null;
|
|
1863
|
-
return (_jsx("section", { "aria-label":
|
|
2001
|
+
return (_jsx("section", { "aria-label": label, ...stylex.props(styles.criterionResults), children: _jsx("ul", { ...stylex.props(styles.criterionResultList), children: presentations.map((presentation) => (_jsx(CriterionResultRow, { criterion: presentation.criterion, outcome: presentation.outcome, explanation: presentation.explanation, failure: presentation.failure }, presentation.criterion))) }) }));
|
|
1864
2002
|
}
|
|
1865
2003
|
function criterionDisplayOutcome(input) {
|
|
1866
2004
|
const outcome = criterionOutcome(input);
|
|
@@ -1918,9 +2056,13 @@ function ResultMetrics({ result }) {
|
|
|
1918
2056
|
];
|
|
1919
2057
|
return (_jsx("div", { ...stylex.props(styles.metricGrid), children: metrics.map(([label, value]) => (_jsxs("div", { ...stylex.props(styles.metric), children: [_jsx("span", { ...stylex.props(styles.resourceFactLabel), children: label }), _jsx("span", { ...stylex.props(styles.metricValue), children: value })] }, label))) }));
|
|
1920
2058
|
}
|
|
2059
|
+
const SECTION_INTROS = {
|
|
2060
|
+
Agents: "The task each agent received and how it answered.",
|
|
2061
|
+
Judges: "A separate reviewer graded the agent answers below. Its findings feed the checks above.",
|
|
2062
|
+
};
|
|
1921
2063
|
function Conversation({ label, operation, operations, allOperations, select, status, attemptStatus, result, }) {
|
|
1922
2064
|
const { timeline, judge } = conversationContent(operation, operations);
|
|
1923
|
-
return (_jsx("section", { "aria-label": label, ...stylex.props(styles.conversation), children: _jsxs("div", { ...stylex.props(styles.conversationSurface), children: [
|
|
2065
|
+
return (_jsx("section", { "aria-label": label, ...stylex.props(styles.conversation), children: _jsxs("div", { ...stylex.props(styles.conversationSurface), children: [_jsxs("header", { ...stylex.props(styles.conversationHeader), children: [_jsx("h3", { ...stylex.props(styles.contentHeading), children: label }), _jsx("p", { ...stylex.props(styles.sectionIntro), children: SECTION_INTROS[label] })] }), _jsx(ConversationTabs, { label: label, operations: operations, selected: operation, select: select, attemptStatus: attemptStatus }), _jsx(ConversationRunTabs, { operation: operation, operations: operations, select: select }), operation?.kind === "evaluation" || operations.length < 2 ? null : result, _jsx("section", { "aria-label": `${label} messages`, ...stylex.props(styles.conversationThread), children: judge ? (_jsxs(_Fragment, { children: [_jsx(JudgeAsked, { review: judge }), _jsx(JudgeResponse, { review: judge, operations: allOperations }), _jsx(JudgeInputs, { review: judge, operations: allOperations })] })) : (_jsx(ConversationEvents, { timeline: timeline })) }), timeline.length === 0 && status !== "running" ? (_jsx("p", { ...stylex.props(styles.meta), children: conversationPlaceholder(status) })) : null, status === "running" ? _jsx(RunningIndicator, { kind: operation?.kind }) : null] }) }));
|
|
1924
2066
|
}
|
|
1925
2067
|
function conversationContent(operation, operations) {
|
|
1926
2068
|
if (operation?.kind === "evaluation") {
|
|
@@ -1952,17 +2094,26 @@ function recordedTimeline(result) {
|
|
|
1952
2094
|
function ConversationEvents({ timeline }) {
|
|
1953
2095
|
return timeline.map((event, index) => event.kind === "tool" ? (_jsx(ToolActivity, { tool: event.tool }, `tool:${event.tool.seq ?? index}:${event.tool.name}`)) : (_jsx(MessageBubble, { message: event.message }, `message:${event.message.seq ?? index}:${event.message.role}`)));
|
|
1954
2096
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
2097
|
+
/** The judge's instruction comes first: it defines what each finding below means. */
|
|
2098
|
+
function JudgeAsked({ review }) {
|
|
2099
|
+
if (!review.question)
|
|
1957
2100
|
return null;
|
|
1958
|
-
return (
|
|
2101
|
+
return (_jsx("section", { "aria-label": "Judge question", ...stylex.props(styles.judgeQuestion), children: _jsxs("p", { ...stylex.props(styles.judgeQuestionText), children: [_jsx("span", { ...stylex.props(styles.judgeAskedLabel), children: "Asked" }), _jsx("span", { children: review.question })] }) }));
|
|
2102
|
+
}
|
|
2103
|
+
/** Everything the test sent the judge, folded away: it mostly repeats the agent answers. */
|
|
2104
|
+
function JudgeInputs({ review, operations, }) {
|
|
2105
|
+
if (!review.context.length)
|
|
2106
|
+
return null;
|
|
2107
|
+
// Inputs matter most before findings exist and when a finding failed.
|
|
2108
|
+
const inputsFirst = review.outcomes.length === 0 || review.outcomes.some((outcome) => !outcome.passed);
|
|
2109
|
+
return (_jsx("section", { "aria-label": "Context included", children: _jsxs("details", { "data-disclosure": true, open: inputsFirst, ...stylex.props(styles.judgeContextDisclosure), children: [_jsxs("summary", { ...stylex.props(styles.judgeContextSummary), children: ["What the judge saw \u00B7 ", review.context.length, " selected", " ", pluralize(review.context.length, "field")] }), _jsx(JudgeContext, { context: review.context, operations: operations })] }) }));
|
|
1959
2110
|
}
|
|
1960
2111
|
function JudgeContext({ context, operations, }) {
|
|
1961
2112
|
const { answers, supplied } = judgeContextGroups(context, operations);
|
|
1962
2113
|
const [selectedId, setSelectedId] = useState(answers[0]?.id);
|
|
1963
2114
|
const selected = answers.find((item) => item.id === selectedId) ?? answers[0];
|
|
1964
2115
|
const panelId = useId();
|
|
1965
|
-
return (_jsxs("
|
|
2116
|
+
return (_jsxs("div", { ...stylex.props(styles.judgeContext), children: [_jsx("p", { ...stylex.props(styles.judgeContextNote), children: "Only these fields, chosen by the test, were sent to the judge." }), answers.length ? (_jsxs("section", { "aria-label": "Agent answers", ...stylex.props(styles.judgeContextGroup), children: [_jsx("h5", { ...stylex.props(styles.judgeContextGroupHeading), children: "Agent answers" }), _jsx("div", { role: "tablist", "aria-label": "Agent answers included", ...stylex.props(styles.judgeContextTabs), children: answers.map((answer, index) => (_jsx("button", { type: "button", role: "tab", "aria-selected": answer.id === selected?.id, "aria-controls": panelId, ...stylex.props(styles.buttonReset, styles.judgeContextTab, answer.id === selected?.id && styles.judgeContextTabActive), onClick: () => setSelectedId(answer.id), onKeyDown: (event) => selectContextWithKeyboard({ event, answers, index, select: setSelectedId }), children: answer.tabLabel }, answer.id))) }), selected ? (_jsx("div", { id: panelId, role: "tabpanel", "aria-label": `Agent response from ${selected.tabLabel}`, ...stylex.props(styles.judgeContextAnswer), children: _jsx("div", { ...stylex.props(styles.judgeContextValue), children: _jsx("div", { className: "message-markdown", children: _jsx(Markdown, { children: normalizeTranscriptMarkdown(selected.value) }) }) }) })) : null] })) : null, supplied.length ? (_jsxs("section", { "aria-label": "Additional context", ...stylex.props(styles.judgeContextGroup), children: [_jsx("h5", { ...stylex.props(styles.judgeContextGroupHeading), children: "Additional context" }), _jsx("dl", { ...stylex.props(styles.judgeContextList), children: supplied.map((item) => (_jsxs("div", { ...stylex.props(styles.judgeContextItem), children: [_jsx("dt", { ...stylex.props(styles.judgeContextLabel), children: item.label }), _jsx("dd", { ...stylex.props(styles.judgeContextValue), children: item.value })] }, item.label))) })] })) : null] }));
|
|
1966
2117
|
}
|
|
1967
2118
|
function judgeContextGroups(context, operations) {
|
|
1968
2119
|
const candidates = operations
|
|
@@ -1982,6 +2133,7 @@ function judgeContextGroups(context, operations) {
|
|
|
1982
2133
|
...item,
|
|
1983
2134
|
id: `${match.operation.id}:${index}`,
|
|
1984
2135
|
tabLabel: agentRunLabel(match.operation, operations),
|
|
2136
|
+
agentName: match.operation.name,
|
|
1985
2137
|
});
|
|
1986
2138
|
}
|
|
1987
2139
|
return { answers, supplied };
|
|
@@ -2026,15 +2178,50 @@ function selectContextWithKeyboard(input) {
|
|
|
2026
2178
|
function JudgeResponse({ review, operations, }) {
|
|
2027
2179
|
if (!review || (!review.outcomes.length && !review.reason))
|
|
2028
2180
|
return null;
|
|
2029
|
-
const { answers } = judgeContextGroups(review.context, operations);
|
|
2030
|
-
return _jsx(JudgeFindings, { outcomes: review.outcomes, reason: review.reason, answers: answers });
|
|
2031
|
-
}
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2181
|
+
const { answers, supplied } = judgeContextGroups(review.context, operations);
|
|
2182
|
+
return (_jsx(JudgeFindings, { outcomes: review.outcomes, reason: review.reason, answers: answers, references: supplied.filter((item) => item.value.length <= SHORT_REFERENCE) }));
|
|
2183
|
+
}
|
|
2184
|
+
const SHORT_REFERENCE = 120;
|
|
2185
|
+
const ANSWER_QUOTE = 160;
|
|
2186
|
+
/**
|
|
2187
|
+
* One row per judged field, quoting the answer it judged, then the short reference values the
|
|
2188
|
+
* test supplied and the judge's shared reason. Together they show why each row passed or failed.
|
|
2189
|
+
*/
|
|
2190
|
+
function JudgeFindings({ outcomes, reason, answers = [], references = [], }) {
|
|
2191
|
+
return (_jsxs("section", { "aria-label": "Judge response", ...stylex.props(styles.judgeResponse), children: [_jsx("h4", { ...stylex.props(styles.judgeResponseHeading), children: "Findings" }), _jsx("div", { ...stylex.props(styles.judgeOutcomeList), children: outcomes.map((outcome) => (_jsx(JudgeFindingRow, { outcome: outcome, answers: answers }, outcome.label))) }), outcomes.length && references.length ? (_jsxs("p", { ...stylex.props(styles.judgeReason), children: [_jsx("span", { ...stylex.props(styles.judgeAskedLabel), children: "Compared with" }), references.map((item) => `${item.label}: ${item.value}`).join(" · ")] })) : null, reason ? (_jsxs("p", { ...stylex.props(styles.judgeReason), children: [outcomes.length ? _jsx("span", { ...stylex.props(styles.judgeAskedLabel), children: "Reason" }) : null, _jsx("span", { children: reason })] })) : null] }));
|
|
2192
|
+
}
|
|
2193
|
+
function JudgeFindingRow({ outcome, answers, }) {
|
|
2194
|
+
const answer = judgedAnswer(outcome, answers);
|
|
2195
|
+
return (_jsxs("div", { ...stylex.props(styles.judgeOutcome), children: [_jsx(CriterionOutcomeIcon, { outcome: outcome.passed ? "passed" : "failed", label: outcome.passed ? "Passed" : "Failed" }), _jsx("span", { ...stylex.props(styles.judgeOutcomeLabel), children: judgeOutcomeLabel(outcome, answer) }), outcome.explanation ? (_jsx("p", { ...stylex.props(styles.judgeOutcomeExplanation), children: outcome.explanation })) : null, answer ? _jsx(JudgedAnswerSource, { answer: answer }) : null, answer ? (_jsxs("p", { ...stylex.props(styles.judgeOutcomeExplanation), children: ["\u201C", answerQuote(answer.value), "\u201D"] })) : null] }));
|
|
2196
|
+
}
|
|
2197
|
+
/** Which agent run produced the judged answer, and what that agent is for when the test says. */
|
|
2198
|
+
function JudgedAnswerSource({ answer }) {
|
|
2199
|
+
const descriptions = useContext(AgentDescriptions);
|
|
2200
|
+
const description = answer.agentName ? descriptions.get(answer.agentName) : undefined;
|
|
2201
|
+
return (_jsxs("p", { ...stylex.props(styles.judgeOutcomeSource), children: ["From ", answer.tabLabel, description ? ` — ${description}` : ""] }));
|
|
2202
|
+
}
|
|
2203
|
+
/** The answer a field is named after: "fileRoundOneCorrect" judges the "fileRoundOne" input. */
|
|
2204
|
+
function judgedAnswer(outcome, answers) {
|
|
2205
|
+
return answers.find((item) => outcome.label === item.label || outcome.label.startsWith(`${item.label} `));
|
|
2206
|
+
}
|
|
2207
|
+
/**
|
|
2208
|
+
* A finding is named by what the judge looked at, in the test's own words: the input label
|
|
2209
|
+
* ("File round one") when the field judges an answer, otherwise the field ("Explains risk").
|
|
2210
|
+
* The pass/fail icon carries the verdict, so the label never repeats it.
|
|
2211
|
+
*/
|
|
2212
|
+
function judgeOutcomeLabel(outcome, answer) {
|
|
2213
|
+
return sentenceCase(answer?.label ?? outcome.label);
|
|
2214
|
+
}
|
|
2215
|
+
function sentenceCase(value) {
|
|
2216
|
+
const lower = value.toLowerCase();
|
|
2217
|
+
return lower.charAt(0).toUpperCase() + lower.slice(1);
|
|
2218
|
+
}
|
|
2219
|
+
function answerQuote(value) {
|
|
2220
|
+
const plain = value
|
|
2221
|
+
.replace(/[*_`#>]/g, "")
|
|
2222
|
+
.replace(/\s+/g, " ")
|
|
2223
|
+
.trim();
|
|
2224
|
+
return plain.length > ANSWER_QUOTE ? `${plain.slice(0, ANSWER_QUOTE - 1)}…` : plain;
|
|
2038
2225
|
}
|
|
2039
2226
|
function judgeReviewData(operation) {
|
|
2040
2227
|
const records = operation.data.filter(isRecord);
|
|
@@ -2044,7 +2231,7 @@ function judgeReviewData(operation) {
|
|
|
2044
2231
|
return {
|
|
2045
2232
|
question: judgeQuestion(record),
|
|
2046
2233
|
context: judgeContext(record?.input),
|
|
2047
|
-
outcomes: judgeOutcomes(output
|
|
2234
|
+
outcomes: judgeOutcomes(output),
|
|
2048
2235
|
reason,
|
|
2049
2236
|
};
|
|
2050
2237
|
}
|
|
@@ -2072,23 +2259,15 @@ function judgeQuestion(record) {
|
|
|
2072
2259
|
const evaluation = isRecord(record?.evaluation) ? record.evaluation : undefined;
|
|
2073
2260
|
return optionalString(evaluation?.prompt);
|
|
2074
2261
|
}
|
|
2075
|
-
|
|
2262
|
+
/** Only an explanation the judge gave for that field attaches to it; a shared reason never does. */
|
|
2263
|
+
function judgeOutcomes(output) {
|
|
2076
2264
|
const entries = Object.entries(output).filter((entry) => typeof entry[1] === "boolean");
|
|
2077
|
-
|
|
2078
|
-
return entries.map(([key, passed], index) => ({
|
|
2265
|
+
return entries.map(([key, passed]) => ({
|
|
2079
2266
|
label: humanizeKey(key),
|
|
2080
2267
|
passed,
|
|
2081
|
-
explanation: judgeOutcomeExplanation(output, key)
|
|
2268
|
+
explanation: judgeOutcomeExplanation(output, key),
|
|
2082
2269
|
}));
|
|
2083
2270
|
}
|
|
2084
|
-
function splitJudgeExplanation(value, count) {
|
|
2085
|
-
if (!value || count < 1)
|
|
2086
|
-
return [];
|
|
2087
|
-
const sentences = value.split(SENTENCE_BOUNDARY).filter(Boolean);
|
|
2088
|
-
if (sentences.length < count)
|
|
2089
|
-
return Array.from({ length: count }, () => value);
|
|
2090
|
-
return Array.from({ length: count }, (_, index) => index === count - 1 ? sentences.slice(index).join(" ") : (sentences[index] ?? value));
|
|
2091
|
-
}
|
|
2092
2271
|
function judgeOutcomeExplanation(output, key) {
|
|
2093
2272
|
const explanations = isRecord(output.explanations) ? output.explanations : {};
|
|
2094
2273
|
return (optionalString(explanations[key]) ??
|
|
@@ -2205,7 +2384,7 @@ function RunningIndicator({ kind }) {
|
|
|
2205
2384
|
function ToolActivity({ tool }) {
|
|
2206
2385
|
const presentation = toolPresentation(tool);
|
|
2207
2386
|
const [outputOpen, setOutputOpen] = useState(false);
|
|
2208
|
-
return (_jsxs("article", { ...stylex.props(styles.toolEvent), children: [_jsx("span", { "aria-hidden": "true", ...stylex.props(styles.toolIcon), children: presentation.icon }), _jsxs("div", { ...stylex.props(styles.toolCopy), children: [_jsx("strong", { ...stylex.props(styles.toolTitle), children: presentation.title }), presentation.detail ? (_jsx("span", { ...stylex.props(styles.toolDetail), children: presentation.detail })) : null] }), _jsxs("div", { role: "group", "aria-label": "Tool controls", ...stylex.props(styles.toolActions), children: [_jsx("span", { ...stylex.props(styles.toolOutcome, presentation.failed && styles.toolOutcomeFail), children: presentation.outcome }), tool.result ? (_jsx("button", { type: "button", ...stylex.props(styles.buttonReset, styles.toolResultSummary), onClick: () => setOutputOpen((open) => !open), children: outputOpen ? "Hide output" : "View output" })) : null] }), tool.result && outputOpen ? (_jsx("pre", { ...stylex.props(styles.toolResultPre), children: tool.result })) : null] }));
|
|
2387
|
+
return (_jsxs("article", { ...stylex.props(styles.toolEvent), children: [_jsx("span", { "aria-hidden": "true", ...stylex.props(styles.toolIcon), children: presentation.icon }), _jsxs("div", { ...stylex.props(styles.toolCopy), children: [_jsx("strong", { ...stylex.props(styles.toolTitle), children: presentation.title }), presentation.detail ? (_jsx("span", { title: presentation.detail, ...stylex.props(styles.toolDetail), children: presentation.detail })) : null] }), _jsxs("div", { role: "group", "aria-label": "Tool controls", ...stylex.props(styles.toolActions), children: [_jsx("span", { ...stylex.props(styles.toolOutcome, presentation.failed && styles.toolOutcomeFail), children: presentation.outcome }), tool.result ? (_jsx("button", { type: "button", ...stylex.props(styles.buttonReset, styles.toolResultSummary), onClick: () => setOutputOpen((open) => !open), children: outputOpen ? "Hide output" : "View output" })) : null] }), tool.result && outputOpen ? (_jsx("pre", { ...stylex.props(styles.toolResultPre), children: tool.result })) : null, tool.images ? _jsx(ToolImages, { images: tool.images, toolName: presentation.title }) : null] }));
|
|
2209
2388
|
}
|
|
2210
2389
|
function toolPresentation(tool) {
|
|
2211
2390
|
const name = tool.name.toLowerCase();
|
|
@@ -2214,7 +2393,13 @@ function toolPresentation(tool) {
|
|
|
2214
2393
|
if (name === "read" || name.includes("read_file"))
|
|
2215
2394
|
return { icon: "↗", title: "Read file", detail: argument(tool, "path"), outcome, failed };
|
|
2216
2395
|
if (["shell", "bash", "exec_command"].includes(name))
|
|
2217
|
-
return {
|
|
2396
|
+
return {
|
|
2397
|
+
icon: ">",
|
|
2398
|
+
title: "Run command",
|
|
2399
|
+
detail: unwrapShellCommand(argument(tool, "command")),
|
|
2400
|
+
outcome,
|
|
2401
|
+
failed,
|
|
2402
|
+
};
|
|
2218
2403
|
if (name.includes("edit") || name.includes("write"))
|
|
2219
2404
|
return { icon: "✎", title: "Change file", detail: argument(tool, "path"), outcome, failed };
|
|
2220
2405
|
return { icon: "◆", title: tool.name, detail: summarizeArgs(tool.args), outcome, failed };
|
|
@@ -2260,7 +2445,7 @@ function OperationList({ operations }) {
|
|
|
2260
2445
|
if (operations.length === 0)
|
|
2261
2446
|
return null;
|
|
2262
2447
|
const summaries = summarizeOperations(operations);
|
|
2263
|
-
return (
|
|
2448
|
+
return (_jsxs("div", { ...stylex.props(styles.ranRow), children: [_jsx("span", { ...stylex.props(styles.ranLabel), children: "Ran" }), summaries.map((summary) => (_jsxs("span", { ...stylex.props(styles.ranChip), children: [summary.name, summary.count > 1 ? ` ×${summary.count}` : ""] }, summary.key)))] }));
|
|
2264
2449
|
}
|
|
2265
2450
|
function summarizeOperations(operations) {
|
|
2266
2451
|
const summaries = new Map();
|
|
@@ -2275,40 +2460,14 @@ function summarizeOperations(operations) {
|
|
|
2275
2460
|
}
|
|
2276
2461
|
return [...summaries.values()];
|
|
2277
2462
|
}
|
|
2278
|
-
function useExecutionHistory() {
|
|
2279
|
-
return useQuery({
|
|
2280
|
-
queryKey: ["execution-history"],
|
|
2281
|
-
queryFn: fetchExecutionHistory,
|
|
2282
|
-
});
|
|
2283
|
-
}
|
|
2284
|
-
async function fetchTestCatalog() {
|
|
2285
|
-
return viewerApi.getTestCatalog();
|
|
2286
|
-
}
|
|
2287
|
-
async function fetchExecutionHistory() {
|
|
2288
|
-
return (await viewerApi.listExecutions()).executions;
|
|
2289
|
-
}
|
|
2290
|
-
async function fetchExecution(id) {
|
|
2291
|
-
return (await viewerApi.getExecution(id)).execution;
|
|
2292
|
-
}
|
|
2293
|
-
async function startTest(testId, workers) {
|
|
2294
|
-
return viewerApi.startExecution(testId, workers);
|
|
2295
|
-
}
|
|
2296
|
-
async function startSuite(workers) {
|
|
2297
|
-
return viewerApi.startSuiteExecution(workers);
|
|
2298
|
-
}
|
|
2299
|
-
async function startTestGroup(testIds, workers) {
|
|
2300
|
-
return viewerApi.startTestGroup(testIds, workers);
|
|
2301
|
-
}
|
|
2302
|
-
async function cancelExecution(id) {
|
|
2303
|
-
await viewerApi.cancelExecution(id);
|
|
2304
|
-
}
|
|
2305
2463
|
function latestTestStatuses(executions) {
|
|
2306
2464
|
const statuses = new Map();
|
|
2307
2465
|
for (const execution of executions) {
|
|
2308
2466
|
for (const testId of execution.testIds ?? []) {
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2467
|
+
const status = testRunStatus(execution, testId);
|
|
2468
|
+
const known = statuses.get(testId);
|
|
2469
|
+
if (known === undefined || (known === "skipped" && status !== "skipped"))
|
|
2470
|
+
statuses.set(testId, status);
|
|
2312
2471
|
}
|
|
2313
2472
|
}
|
|
2314
2473
|
return statuses;
|