niceeval 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/zh-CN.d.ts +3 -3
- package/dist/report/aggregate.d.ts +32 -26
- package/dist/report/aggregate.js +157 -76
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +91 -164
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +86 -73
- package/dist/report/compute.js +592 -432
- package/dist/report/flag.d.ts +28 -17
- package/dist/report/flag.js +86 -16
- package/dist/report/format.d.ts +11 -11
- package/dist/report/format.js +17 -15
- package/dist/report/index.d.ts +16 -17
- package/dist/report/index.js +20 -22
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +49 -34
- package/dist/report/locale.js +106 -58
- package/dist/report/metrics.d.ts +10 -3
- package/dist/report/metrics.js +46 -12
- package/dist/report/primitives.d.ts +42 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +10 -8
- package/dist/report/react/AttemptList.js +18 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +3 -3
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +4 -2
- package/dist/report/react/ExperimentComparison.js +5 -4
- package/dist/report/react/ExperimentList.d.ts +3 -3
- package/dist/report/react/ExperimentList.js +16 -15
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +54 -17
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +105 -149
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -20
- package/dist/report/report.js +261 -34
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +225 -157
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +2 -2
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +245 -300
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +38 -14
- package/dist/shared/aggregate.d.ts +3 -2
- package/dist/shared/aggregate.js +5 -4
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
- package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
- package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
- package/docs-site/zh/index.mdx +23 -25
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +6 -6
- package/docs-site/zh/reference/cli.mdx +9 -7
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
- package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
- package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
- package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
- package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +6 -2
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +3 -3
- package/src/cli.ts +42 -23
- package/src/context/context.ts +1 -1
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/i18n/en.ts +18 -16
- package/src/i18n/zh-CN.ts +16 -15
- package/src/report/aggregate.ts +175 -87
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -285
- package/src/report/compute.ts +717 -515
- package/src/report/dual-render.test.tsx +738 -1148
- package/src/report/flag.ts +97 -33
- package/src/report/format.ts +18 -22
- package/src/report/index.ts +113 -58
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +120 -69
- package/src/report/metrics.ts +42 -12
- package/src/report/primitives.tsx +190 -45
- package/src/report/react/AttemptList.tsx +32 -20
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +12 -7
- package/src/report/react/ExperimentList.tsx +38 -26
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +74 -20
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/enhance.js +57 -5
- package/src/report/react/fixtures.ts +109 -156
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +139 -104
- package/src/report/react/styles.css +181 -91
- package/src/report/report.test.ts +761 -1031
- package/src/report/report.ts +425 -47
- package/src/report/text/faces.ts +257 -164
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +2 -2
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +257 -287
- package/src/report/web.ts +63 -20
- package/src/results/attempt-evidence.test.ts +4 -4
- package/src/results/attempt-evidence.ts +5 -5
- package/src/results/copy.ts +6 -6
- package/src/results/host-equivalence.test.ts +26 -14
- package/src/results/index.ts +10 -4
- package/src/results/open.ts +8 -4
- package/src/results/results.test.ts +4 -3
- package/src/results/select.ts +104 -34
- package/src/results/types.ts +36 -14
- package/src/runner/feedback/human.test.ts +1 -1
- package/src/runner/run.ts +1 -1
- package/src/sandbox/cli-commands.ts +2 -2
- package/src/scoring/judge.test.ts +1 -1
- package/src/shared/aggregate.ts +5 -4
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +107 -56
- package/src/show/render.ts +43 -27
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +86 -36
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +324 -63
- package/src/view/app/components/CodeView.tsx +10 -4
- package/src/view/app/i18n.ts +31 -17
- package/src/view/app/lib/artifact-url.ts +14 -3
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +21 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +6 -6
- package/src/view/data.test.ts +9 -3
- package/src/view/data.ts +145 -49
- package/src/view/index.ts +48 -44
- package/src/view/server.ts +35 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +224 -0
- package/src/view/view-report.test.ts +161 -57
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
- package/dist/report/built-ins/experiment-comparison.js +0 -119
- package/dist/report/built-ins/index.d.ts +0 -2
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/src/report/built-in-user-parity.test.tsx +0 -597
- package/src/report/built-ins/experiment-comparison.tsx +0 -179
- package/src/report/built-ins/index.ts +0 -7
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
// Scoreboard:考试成绩单——总分 + 分科小计。
|
|
2
|
-
// 固定分母的口径不藏:没跑的题挣 0
|
|
3
|
-
//
|
|
2
|
+
// 固定分母的口径不藏:没跑的题挣 0 分但留在分母里(notRun),跑了但测不了的题同样按 0 分
|
|
3
|
+
// (unscorable),两个计数分开显示,连同 refs 一起,不合并成一个笼统的缺失数;
|
|
4
|
+
// weights 是「实际生效的权重表」,渲染出来让成绩单可审计;题集之外被忽略的 eval 数在注脚。
|
|
4
5
|
|
|
5
6
|
import type { ReactElement } from "react";
|
|
6
7
|
import type { ScoreboardData } from "../types.ts";
|
|
7
|
-
import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../locale.ts";
|
|
8
|
+
import { DEFAULT_REPORT_LOCALE, countText, localeText, resolveLocalizedText, type ReportLocale } from "../locale.ts";
|
|
8
9
|
import { colorClassForKey } from "./colors.ts";
|
|
9
10
|
import { cx } from "./format.ts";
|
|
10
11
|
|
|
@@ -31,7 +32,7 @@ export function Scoreboard({
|
|
|
31
32
|
<thead>
|
|
32
33
|
<tr>
|
|
33
34
|
<th scope="col" className="nre-dimension">
|
|
34
|
-
{data.
|
|
35
|
+
{data.rowDimension}
|
|
35
36
|
</th>
|
|
36
37
|
<th scope="col" className="nre-total-col">
|
|
37
38
|
{localeText(locale, "scoreboard.total")}
|
|
@@ -51,7 +52,16 @@ export function Scoreboard({
|
|
|
51
52
|
<th scope="row" className={cx("nre-row-key", "nre-key", colorClassForKey(row.key))}>
|
|
52
53
|
{row.key}
|
|
53
54
|
</th>
|
|
54
|
-
<td className="nre-total">
|
|
55
|
+
<td className="nre-total">
|
|
56
|
+
{resolveLocalizedText(row.total.display, locale)}
|
|
57
|
+
{/* 两种 0 分分开注脚:没去考(notRun)与考了判不了(unscorable) */}
|
|
58
|
+
{row.total.notRun > 0 && (
|
|
59
|
+
<span className="nre-total-not-run">{countText(locale, "scoreboard.notRun", row.total.notRun)}</span>
|
|
60
|
+
)}
|
|
61
|
+
{row.total.unscorable > 0 && (
|
|
62
|
+
<span className="nre-total-unscorable">{countText(locale, "scoreboard.unscorable", row.total.unscorable)}</span>
|
|
63
|
+
)}
|
|
64
|
+
</td>
|
|
55
65
|
{subjectKeys.map((key) => {
|
|
56
66
|
const subject = row.subjects.find((s) => s.key === key);
|
|
57
67
|
if (!subject) return <td key={key} className="nre-td-empty" />;
|
|
@@ -60,16 +70,20 @@ export function Scoreboard({
|
|
|
60
70
|
<span
|
|
61
71
|
className="nre-subject-score"
|
|
62
72
|
title={localeText(locale, "scoreboard.subjectTitle", {
|
|
63
|
-
|
|
73
|
+
questions: subject.questions,
|
|
64
74
|
earned: subject.earned,
|
|
65
75
|
possible: subject.possible,
|
|
66
76
|
})}
|
|
67
77
|
>
|
|
68
|
-
{subject.
|
|
78
|
+
{resolveLocalizedText(subject.display, locale)}
|
|
69
79
|
</span>
|
|
70
|
-
{
|
|
71
|
-
|
|
72
|
-
|
|
80
|
+
{subject.notRun > 0 && (
|
|
81
|
+
<span className="nre-subject-not-run">{localeText(locale, "scoreboard.notRunText", { n: subject.notRun })}</span>
|
|
82
|
+
)}
|
|
83
|
+
{subject.unscorable > 0 && (
|
|
84
|
+
<span className="nre-subject-unscorable">
|
|
85
|
+
{localeText(locale, "scoreboard.unscorableText", { n: subject.unscorable })}
|
|
86
|
+
</span>
|
|
73
87
|
)}
|
|
74
88
|
</td>
|
|
75
89
|
);
|
|
@@ -90,6 +104,10 @@ export function Scoreboard({
|
|
|
90
104
|
))}
|
|
91
105
|
{data.weights.length > 0 && <span className="nre-weight-rest">{localeText(locale, "scoreboard.othersWeight")}</span>}
|
|
92
106
|
</p>
|
|
107
|
+
{/* 题集之外的 eval:忽略但如实报数 */}
|
|
108
|
+
{data.ignoredEvals > 0 && (
|
|
109
|
+
<p className="nre-scoreboard-ignored">{countText(locale, "scoreboard.ignored", data.ignoredEvals)}</p>
|
|
110
|
+
)}
|
|
93
111
|
</section>
|
|
94
112
|
);
|
|
95
113
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import type { ReactElement } from "react";
|
|
6
6
|
import type { MetricCell } from "../types.ts";
|
|
7
7
|
import type { AttemptLocator } from "../../results/locator.ts";
|
|
8
|
-
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
|
|
8
|
+
import { DEFAULT_REPORT_LOCALE, localeText, resolveLocalizedText, type ReportLocale } from "../locale.ts";
|
|
9
9
|
|
|
10
10
|
export function MetricCellView({
|
|
11
11
|
cell,
|
|
@@ -32,7 +32,7 @@ export function MetricCellView({
|
|
|
32
32
|
className="nre-value"
|
|
33
33
|
title={localeText(locale, "cell.measuredTitle", { samples: cell.samples, total: cell.total })}
|
|
34
34
|
>
|
|
35
|
-
{cell.display}
|
|
35
|
+
{resolveLocalizedText(cell.display, locale)}
|
|
36
36
|
</span>
|
|
37
37
|
{/* samples < total:有 attempt 测不了这个指标,覆盖率角标如实标出 */}
|
|
38
38
|
{cell.samples < cell.total && (
|
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
// niceeval 报告的渐进增强 runtime:纯 vanilla JS、零依赖、IIFE、幂等。
|
|
2
|
-
// 只作用于 .nre DOM 与 data-nre-*
|
|
3
|
-
//
|
|
4
|
-
// <title
|
|
5
|
-
//
|
|
6
|
-
//
|
|
2
|
+
// 只作用于 .nre DOM 与 data-nre-* 属性;六个行为——实验组切换、Tabs 单选切换、表格排序、
|
|
3
|
+
// 行过滤、SVG 点 tooltip、警告命令复制。全部只改浏览状态,不改数据、指标口径或初始 HTML 数值。
|
|
4
|
+
// 静态 HTML 无 JS 时内容完整可读是硬约束:排序有数据侧预排、tooltip 退化为原生 <title>、
|
|
5
|
+
// 过滤输入框静默无功能、Tabs 退化为原生 <details> 手风琴、命令块退化为点击全选。
|
|
6
|
+
// 全部经 document 级事件委托绑定,重复注入本文件只在首次生效(window.__nreEnhanced 守卫),
|
|
7
|
+
// DOM 被搬动(如 view 把 <template> 内容摆进报告槽)也无需重新绑定。
|
|
7
8
|
|
|
8
9
|
(function () {
|
|
9
10
|
"use strict";
|
|
10
11
|
if (typeof window === "undefined" || window.__nreEnhanced) return;
|
|
11
12
|
window.__nreEnhanced = true;
|
|
12
13
|
|
|
14
|
+
// 根类 nre-js:styles.css 用它把仅增强态的布局(单选 tab 条、复制指针)限定在 JS 在场时。
|
|
15
|
+
// 挂在 documentElement 上与报告块位置无关,块之后被搬进槽位也不需要补标记。
|
|
16
|
+
document.documentElement.classList.add("nre-js");
|
|
17
|
+
|
|
13
18
|
function closest(target, selector) {
|
|
14
19
|
return target && target.closest ? target.closest(selector) : null;
|
|
15
20
|
}
|
|
@@ -46,6 +51,53 @@
|
|
|
46
51
|
selectExperimentGroup(control);
|
|
47
52
|
});
|
|
48
53
|
|
|
54
|
+
// ───────────────────────── Tabs:[data-nre-tabs] 单选切换 ─────────────────────────
|
|
55
|
+
// 静态 HTML 每 tab 一个 <details> 且仅首个 open;点击 summary 时接管原生 toggle:
|
|
56
|
+
// 打开所点 tab、收起同组其余,点已开的 tab 保持打开(单选语义,恒有一个面板可见)。
|
|
57
|
+
// 只切换 open 状态,不触碰 tab 内任何数据;键盘 Enter/Space 走 summary 的原生激活(即 click)。
|
|
58
|
+
|
|
59
|
+
document.addEventListener("click", function (e) {
|
|
60
|
+
var title = closest(e.target, "[data-nre-tabs] > details > summary");
|
|
61
|
+
if (!title) return;
|
|
62
|
+
e.preventDefault();
|
|
63
|
+
var tab = title.parentNode;
|
|
64
|
+
var group = tab.parentNode;
|
|
65
|
+
for (var i = 0; i < group.children.length; i++) {
|
|
66
|
+
var child = group.children[i];
|
|
67
|
+
if (child.tagName === "DETAILS") child.open = child === tab;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// ───────────────────────── 复制:[data-nre-copy](宿主警告块的命令) ─────────────────────────
|
|
72
|
+
// 点击把 data-nre-copy 携带的完整命令写进剪贴板,成功后短暂打上 data-nre-copied
|
|
73
|
+
// (styles.css 显示 ✓);剪贴板不可用时退化为全选该块文本,用户手动复制。
|
|
74
|
+
// 块内文本与属性值恒不变,复制的是数据侧已写好的命令原文。
|
|
75
|
+
|
|
76
|
+
document.addEventListener("click", function (e) {
|
|
77
|
+
var block = closest(e.target, "[data-nre-copy]");
|
|
78
|
+
if (!block) return;
|
|
79
|
+
var command = block.getAttribute("data-nre-copy") || "";
|
|
80
|
+
function mark() {
|
|
81
|
+
block.setAttribute("data-nre-copied", "");
|
|
82
|
+
setTimeout(function () {
|
|
83
|
+
block.removeAttribute("data-nre-copied");
|
|
84
|
+
}, 1500);
|
|
85
|
+
}
|
|
86
|
+
function selectFallback() {
|
|
87
|
+
var selection = window.getSelection();
|
|
88
|
+
if (!selection) return;
|
|
89
|
+
var range = document.createRange();
|
|
90
|
+
range.selectNodeContents(block);
|
|
91
|
+
selection.removeAllRanges();
|
|
92
|
+
selection.addRange(range);
|
|
93
|
+
}
|
|
94
|
+
if (navigator.clipboard && navigator.clipboard.writeText) {
|
|
95
|
+
navigator.clipboard.writeText(command).then(mark, selectFallback);
|
|
96
|
+
} else {
|
|
97
|
+
selectFallback();
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
|
|
49
101
|
// ───────────────────────── 排序:th[data-nre-sort] ─────────────────────────
|
|
50
102
|
// 点击按该列排序 tbody 行(td/th 的 data-sort-value,数值优先、退回字符串;
|
|
51
103
|
// 空值恒沉底),再点反向;方向指示由 th 上的 nre-sort-asc / nre-sort-desc 类驱动。
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// 手工 fixture:模拟计算函数(
|
|
1
|
+
// 手工 fixture:模拟计算函数(metricTableData / metricMatrixData / …)的产物。
|
|
2
2
|
// 仅供渲染测试与 scripts/report-react-demo.tsx 使用,不从入口导出。
|
|
3
3
|
// 数字刻意覆盖诚实细节:coverage 角标(samples<total)、全 null 格子、
|
|
4
4
|
// 稀疏矩阵、缺数据的散点、delta 的 null 不硬算、truncated 计数。
|
|
@@ -8,12 +8,11 @@ import type {
|
|
|
8
8
|
DeltaData,
|
|
9
9
|
EvalListItem,
|
|
10
10
|
ExperimentListItem,
|
|
11
|
-
GroupSummaryData,
|
|
12
11
|
LineData,
|
|
13
12
|
MatrixData,
|
|
14
13
|
MetricColumn,
|
|
15
|
-
OverviewData,
|
|
16
14
|
ScatterData,
|
|
15
|
+
ScopeSummaryData,
|
|
17
16
|
ScoreboardData,
|
|
18
17
|
TableData,
|
|
19
18
|
} from "../types.ts";
|
|
@@ -25,56 +24,21 @@ export const passRateColumn: MetricColumn = { key: "task-pass-rate", label: "pas
|
|
|
25
24
|
export const codeLinesColumn: MetricColumn = { key: "code-lines", label: "code lines", unit: "lines", better: "lower" };
|
|
26
25
|
export const costColumn: MetricColumn = { key: "cost", label: "cost", unit: "$", better: "lower" };
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
{ experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T11:30:00Z" },
|
|
32
|
-
],
|
|
33
|
-
totals: {
|
|
34
|
-
evals: 12,
|
|
35
|
-
attempts: 48,
|
|
36
|
-
passed: 36,
|
|
37
|
-
failed: 8,
|
|
38
|
-
errored: 2,
|
|
39
|
-
skipped: 2,
|
|
40
|
-
// 两级聚合口径(computeCell)刻意不等于 36/(36+8+2)≈78% 的 attempt 原始占比:
|
|
41
|
-
// 组件必须原样渲染这个字段,不得从上面四个 verdict 计票现场重算。
|
|
42
|
-
passRate: { value: 0.7, display: "70%", samples: 46, total: 48, refs: [] },
|
|
43
|
-
costUSD: null, // 全部 attempt 都没报成本:null,组件必须显示缺数据而不是 $0
|
|
44
|
-
durationMs: 261_000,
|
|
45
|
-
},
|
|
46
|
-
warnings: [
|
|
47
|
-
{
|
|
48
|
-
kind: "partial-coverage",
|
|
49
|
-
experimentId: "compare/bub",
|
|
50
|
-
covered: 9,
|
|
51
|
-
total: 12,
|
|
52
|
-
message: "snapshot covers 9 of 12 evals seen in history; re-run `niceeval exp compare/bub` for a full snapshot",
|
|
53
|
-
},
|
|
54
|
-
],
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
export const overviewWithCost: OverviewData = {
|
|
58
|
-
...overviewData,
|
|
59
|
-
totals: { ...overviewData.totals, costUSD: 1.234 },
|
|
60
|
-
warnings: [],
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
/** GroupSummary.data 的产物形态:eval 级折叠计票 + 旧 GroupSelector 口径的通过率(包成 MetricCell)。 */
|
|
64
|
-
export const groupSummaryData: GroupSummaryData = {
|
|
27
|
+
/** scopeSummaryData 的产物形态:两级计票恒随行,成功率与总成本是官方 MetricCell。 */
|
|
28
|
+
export const scopeSummaryData: ScopeSummaryData = {
|
|
29
|
+
range: { earliestStartedAt: "2026-07-01T10:00:00Z", latestStartedAt: "2026-07-01T11:30:00Z" },
|
|
65
30
|
experiments: 2,
|
|
66
|
-
// evals = 全部 verdicts 之和(3+1+1+1=6);passRate 分母只数非 skipped 的 5 道
|
|
67
31
|
evals: 6,
|
|
68
32
|
attempts: 9,
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
33
|
+
evalVerdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
|
|
34
|
+
attemptVerdicts: { passed: 4, failed: 3, errored: 1, skipped: 1 },
|
|
35
|
+
// 两级聚合口径,刻意不等于任一计票的比例:组件必须原样渲染,不重算
|
|
36
|
+
endToEndPassRate: { value: 0.6, display: "60%", samples: 8, total: 9, refs: [] },
|
|
37
|
+
totalCostUSD: { value: 1.5, display: "$1.50", samples: 8, total: 9, refs: [] },
|
|
74
38
|
};
|
|
75
39
|
|
|
76
|
-
export const tableData: TableData
|
|
77
|
-
|
|
40
|
+
export const tableData: TableData = {
|
|
41
|
+
rowDimension: "agent",
|
|
78
42
|
// 行顺序故意不按 passRate 排:组件必须按传入顺序渲染,不重排
|
|
79
43
|
columns: [passRateColumn, codeLinesColumn],
|
|
80
44
|
rows: [
|
|
@@ -94,7 +58,7 @@ export const tableData: TableData<"task-pass-rate" | "code-lines"> = {
|
|
|
94
58
|
display: "87%",
|
|
95
59
|
samples: 6,
|
|
96
60
|
total: 6,
|
|
97
|
-
refs: [locator("@
|
|
61
|
+
refs: [locator("@1a0a0a0a")],
|
|
98
62
|
},
|
|
99
63
|
// samples < total:有 attempt 测不了 → 覆盖率角标 5/6
|
|
100
64
|
"code-lines": { value: 120, display: "120 lines", samples: 5, total: 6, refs: [] },
|
|
@@ -103,34 +67,9 @@ export const tableData: TableData<"task-pass-rate" | "code-lines"> = {
|
|
|
103
67
|
],
|
|
104
68
|
};
|
|
105
69
|
|
|
106
|
-
/** rows: "experiment" 的榜单形态:行携带 agent/model 元信息与 eval 级折叠计票。 */
|
|
107
|
-
export const tableDataWithMeta: TableData<"task-pass-rate"> = {
|
|
108
|
-
dimension: "experiment",
|
|
109
|
-
columns: [passRateColumn],
|
|
110
|
-
rows: [
|
|
111
|
-
{
|
|
112
|
-
key: "compare/bub",
|
|
113
|
-
cells: { "task-pass-rate": { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] } },
|
|
114
|
-
meta: {
|
|
115
|
-
agent: "bub",
|
|
116
|
-
model: "gpt-5.4",
|
|
117
|
-
verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
|
|
118
|
-
},
|
|
119
|
-
},
|
|
120
|
-
{
|
|
121
|
-
key: "compare/codex",
|
|
122
|
-
cells: { "task-pass-rate": { value: 1, display: "100%", samples: 2, total: 2, refs: [] } },
|
|
123
|
-
meta: {
|
|
124
|
-
agent: "codex",
|
|
125
|
-
verdicts: { passed: 2, failed: 0, errored: 0, skipped: 0 },
|
|
126
|
-
},
|
|
127
|
-
},
|
|
128
|
-
],
|
|
129
|
-
};
|
|
130
|
-
|
|
131
70
|
export const matrixData: MatrixData = {
|
|
132
|
-
|
|
133
|
-
|
|
71
|
+
rowDimension: "eval",
|
|
72
|
+
columnDimension: "agent",
|
|
134
73
|
metric: passRateColumn,
|
|
135
74
|
// 稀疏:geometry/angles × codex 没有样本,数据里不出现 → 格子空着
|
|
136
75
|
cells: [
|
|
@@ -142,7 +81,7 @@ export const matrixData: MatrixData = {
|
|
|
142
81
|
display: "100%",
|
|
143
82
|
samples: 2,
|
|
144
83
|
total: 2,
|
|
145
|
-
refs: [locator("@
|
|
84
|
+
refs: [locator("@1b3b3b3b"), locator("@1b7b7b7b")],
|
|
146
85
|
},
|
|
147
86
|
},
|
|
148
87
|
{
|
|
@@ -159,34 +98,41 @@ export const matrixData: MatrixData = {
|
|
|
159
98
|
};
|
|
160
99
|
|
|
161
100
|
export const scoreboardData: ScoreboardData = {
|
|
162
|
-
|
|
101
|
+
rowDimension: "agent",
|
|
102
|
+
questions: [
|
|
103
|
+
"algebra/quadratic",
|
|
104
|
+
"algebra/simple",
|
|
105
|
+
"geometry/angles",
|
|
106
|
+
"geometry/area",
|
|
107
|
+
],
|
|
163
108
|
fullMarks: 100,
|
|
164
109
|
weights: [{ prefix: "algebra/", weight: 2 }],
|
|
110
|
+
ignoredEvals: 1,
|
|
165
111
|
rows: [
|
|
166
112
|
{
|
|
167
113
|
key: "bub",
|
|
168
|
-
|
|
114
|
+
// notRun 1:一题没跑、按 0 计;unscorable 0 —— 两种 0 分分开计数
|
|
115
|
+
total: { value: 78.5, display: "78.5", notRun: 1, unscorable: 0, refs: [locator("@1a0a0a0a")] },
|
|
169
116
|
subjects: [
|
|
170
|
-
|
|
171
|
-
{ key: "
|
|
172
|
-
{ key: "geometry", earned: 3, possible: 4, evals: 4, missing: 0 },
|
|
117
|
+
{ key: "algebra", earned: 14, possible: 16, questions: 8, notRun: 1, unscorable: 0, display: "14/16 (87.5%)", refs: [] },
|
|
118
|
+
{ key: "geometry", earned: 3, possible: 4, questions: 4, notRun: 0, unscorable: 0, display: "3/4 (75%)", refs: [] },
|
|
173
119
|
],
|
|
174
120
|
},
|
|
175
121
|
{
|
|
176
122
|
key: "codex",
|
|
177
|
-
total: { value: 52, display: "52" },
|
|
123
|
+
total: { value: 52, display: "52", notRun: 0, unscorable: 2, refs: [] },
|
|
178
124
|
subjects: [
|
|
179
|
-
{ key: "algebra", earned: 9, possible: 16,
|
|
180
|
-
{ key: "geometry", earned: 1.4, possible: 4,
|
|
125
|
+
{ key: "algebra", earned: 9, possible: 16, questions: 8, notRun: 0, unscorable: 0, display: "9/16 (56.3%)", refs: [] },
|
|
126
|
+
{ key: "geometry", earned: 1.4, possible: 4, questions: 4, notRun: 0, unscorable: 2, display: "1.4/4 (35%)", refs: [] },
|
|
181
127
|
],
|
|
182
128
|
},
|
|
183
129
|
],
|
|
184
130
|
};
|
|
185
131
|
|
|
186
132
|
export const scatterData: ScatterData = {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
x: costColumn, // better: "lower" →
|
|
133
|
+
pointDimension: "experiment",
|
|
134
|
+
seriesDimension: "agent",
|
|
135
|
+
x: costColumn, // better: "lower" → 轴反向(贵在左、便宜在右);两轴都有 better → 提示「越靠右上越好」
|
|
190
136
|
y: passRateColumn,
|
|
191
137
|
rows: [
|
|
192
138
|
{
|
|
@@ -219,82 +165,88 @@ export const scatterData: ScatterData = {
|
|
|
219
165
|
|
|
220
166
|
export const lineData: LineData = {
|
|
221
167
|
x: { key: "latencyMs", label: "Simulated latency", unit: "ms" },
|
|
222
|
-
|
|
168
|
+
seriesDimension: "agents",
|
|
223
169
|
y: passRateColumn,
|
|
224
170
|
rows: [
|
|
225
171
|
{
|
|
226
|
-
key: "
|
|
172
|
+
key: "100",
|
|
227
173
|
series: "1 agents",
|
|
228
174
|
x: 100,
|
|
229
175
|
xDisplay: "100ms",
|
|
230
176
|
y: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
|
|
231
177
|
},
|
|
232
178
|
{
|
|
233
|
-
key: "
|
|
179
|
+
key: "300",
|
|
234
180
|
series: "1 agents",
|
|
235
181
|
x: 300,
|
|
236
182
|
xDisplay: "300ms",
|
|
237
183
|
y: { value: 0.3, display: "30%", samples: 6, total: 6, refs: [] },
|
|
238
184
|
},
|
|
239
185
|
{
|
|
240
|
-
key: "
|
|
186
|
+
key: "100",
|
|
241
187
|
series: "16 agents",
|
|
242
188
|
x: 100,
|
|
243
189
|
xDisplay: "100ms",
|
|
244
190
|
y: { value: 0.8, display: "80%", samples: 6, total: 6, refs: [] },
|
|
245
191
|
},
|
|
246
192
|
{
|
|
247
|
-
key: "
|
|
193
|
+
key: "300",
|
|
248
194
|
series: "16 agents",
|
|
249
195
|
x: 300,
|
|
250
196
|
xDisplay: "300ms",
|
|
251
197
|
y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
|
|
252
198
|
},
|
|
253
199
|
{
|
|
254
|
-
//
|
|
255
|
-
key: "
|
|
200
|
+
// 未声明数值 flag 的 attempt:不伪造 x 值,注脚报数
|
|
201
|
+
key: "null",
|
|
256
202
|
series: "1 agents",
|
|
257
203
|
x: null,
|
|
258
|
-
xDisplay: "",
|
|
204
|
+
xDisplay: "—",
|
|
259
205
|
y: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
|
|
260
206
|
},
|
|
261
207
|
],
|
|
262
208
|
};
|
|
263
209
|
|
|
264
|
-
export const deltaData: DeltaData
|
|
210
|
+
export const deltaData: DeltaData = {
|
|
211
|
+
byDimension: "experiment",
|
|
265
212
|
columns: [passRateColumn, costColumn],
|
|
266
213
|
rows: [
|
|
267
214
|
{
|
|
268
|
-
key: "bub",
|
|
269
|
-
|
|
270
|
-
|
|
215
|
+
key: "compare/bub → compare/bub--agents-md",
|
|
216
|
+
label: "bub",
|
|
217
|
+
a: { key: "compare/bub" },
|
|
218
|
+
b: { key: "compare/bub--agents-md" },
|
|
271
219
|
cells: {
|
|
272
|
-
// 通过率 +12pp:better higher →
|
|
220
|
+
// 通过率 +12pp:better higher → improved(绿)
|
|
273
221
|
"task-pass-rate": {
|
|
274
222
|
a: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
|
|
275
223
|
b: { value: 0.62, display: "62%", samples: 6, total: 6, refs: [] },
|
|
276
224
|
delta: 0.12,
|
|
277
225
|
display: "+12pp",
|
|
226
|
+
outcome: "improved",
|
|
278
227
|
},
|
|
279
|
-
// 成本 +$0.15:better lower →
|
|
228
|
+
// 成本 +$0.15:better lower → regressed(红)
|
|
280
229
|
cost: {
|
|
281
230
|
a: { value: 0.2, display: "$0.20", samples: 6, total: 6, refs: [] },
|
|
282
231
|
b: { value: 0.35, display: "$0.35", samples: 6, total: 6, refs: [] },
|
|
283
232
|
delta: 0.15,
|
|
284
233
|
display: "+$0.15",
|
|
234
|
+
outcome: "regressed",
|
|
285
235
|
},
|
|
286
236
|
},
|
|
287
237
|
},
|
|
288
238
|
{
|
|
289
|
-
key: "codex",
|
|
290
|
-
|
|
291
|
-
|
|
239
|
+
key: "compare/codex → compare/codex--agents-md",
|
|
240
|
+
label: "codex",
|
|
241
|
+
a: { key: "compare/codex" },
|
|
242
|
+
b: { key: "compare/codex--agents-md" },
|
|
292
243
|
cells: {
|
|
293
244
|
"task-pass-rate": {
|
|
294
245
|
a: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
|
|
295
246
|
b: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
|
|
296
247
|
delta: 0,
|
|
297
248
|
display: "±0",
|
|
249
|
+
outcome: "unchanged",
|
|
298
250
|
},
|
|
299
251
|
// A 侧缺数据 → delta null:显示缺,不硬算
|
|
300
252
|
cost: {
|
|
@@ -302,6 +254,7 @@ export const deltaData: DeltaData<"task-pass-rate" | "cost"> = {
|
|
|
302
254
|
b: { value: 0.3, display: "$0.30", samples: 6, total: 6, refs: [] },
|
|
303
255
|
delta: null,
|
|
304
256
|
display: "—",
|
|
257
|
+
outcome: "unavailable",
|
|
305
258
|
},
|
|
306
259
|
},
|
|
307
260
|
},
|
|
@@ -312,75 +265,75 @@ export const deltaData: DeltaData<"task-pass-rate" | "cost"> = {
|
|
|
312
265
|
|
|
313
266
|
/** algebra/quadratic 在 compare/bub 上失败的那次 attempt——两条子失败夹具共用同一条。 */
|
|
314
267
|
const failedAttempt: AttemptListItem = {
|
|
315
|
-
evalId: "algebra/quadratic",
|
|
316
268
|
experimentId: "compare/bub",
|
|
269
|
+
evalId: "algebra/quadratic",
|
|
317
270
|
attempt: 3,
|
|
318
271
|
agent: "bub",
|
|
319
272
|
verdict: "failed",
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
score: 0,
|
|
325
|
-
outcome: "failed" as const,
|
|
326
|
-
expected: "x=2",
|
|
327
|
-
received: "x=3",
|
|
328
|
-
evidence: "judge: sign flipped when substituting into the quadratic formula",
|
|
329
|
-
},
|
|
330
|
-
],
|
|
273
|
+
// 已按 Scoring display 契约折好的单行摘要;渲染面只做宽度截断,不重算
|
|
274
|
+
failureSummary: "gate: roots-correct · expected x=2 · received x=3",
|
|
275
|
+
moreFailures: 1,
|
|
276
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1a4a4a4a")] },
|
|
331
277
|
durationMs: 32_000,
|
|
332
278
|
costUSD: 0.12,
|
|
333
|
-
locator: locator("@
|
|
279
|
+
locator: locator("@1a4a4a4a"),
|
|
334
280
|
};
|
|
335
281
|
|
|
336
282
|
const erroredAttempt: AttemptListItem = {
|
|
337
|
-
evalId: "geometry/angles",
|
|
338
283
|
experimentId: "compare/codex",
|
|
284
|
+
evalId: "geometry/angles",
|
|
339
285
|
attempt: 0,
|
|
340
286
|
agent: "codex",
|
|
341
287
|
verdict: "errored",
|
|
342
|
-
//
|
|
343
|
-
error:
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
phase: "eval.run",
|
|
347
|
-
stack: "TypeError: cannot read properties of undefined (reading 'foo')\n at run (adapter.ts:42:7)",
|
|
348
|
-
cause: { name: "TypeError", message: "cannot read properties of undefined (reading 'foo')" },
|
|
349
|
-
},
|
|
350
|
-
diagnostics: [
|
|
351
|
-
{ code: "sandbox-teardown-failed", level: "warning", message: "sandbox teardown timed out", phase: "sandbox.teardown" },
|
|
352
|
-
],
|
|
353
|
-
assertions: [],
|
|
288
|
+
// errored:结构化 error 的一层摘要(phase · code · message)
|
|
289
|
+
failureSummary: "eval.run · unexpected-error · TypeError: cannot read properties of undefined (reading 'foo')",
|
|
290
|
+
moreFailures: 0,
|
|
291
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1c1c1c1c")] },
|
|
354
292
|
durationMs: 4_500,
|
|
355
|
-
|
|
293
|
+
costUSD: null,
|
|
294
|
+
locator: locator("@1c1c1c1c"),
|
|
356
295
|
};
|
|
357
296
|
|
|
358
297
|
const failedRetryAttempt: AttemptListItem = {
|
|
359
298
|
...failedAttempt,
|
|
360
299
|
attempt: 4,
|
|
361
300
|
durationMs: 35_000,
|
|
362
|
-
locator: locator("@
|
|
301
|
+
locator: locator("@1b5b5b5b"),
|
|
302
|
+
};
|
|
303
|
+
|
|
304
|
+
const passedAttempt: AttemptListItem = {
|
|
305
|
+
experimentId: "compare/bub",
|
|
306
|
+
evalId: "algebra/simple",
|
|
307
|
+
attempt: 0,
|
|
308
|
+
agent: "bub",
|
|
309
|
+
verdict: "passed",
|
|
310
|
+
failureSummary: null,
|
|
311
|
+
moreFailures: 0,
|
|
312
|
+
examScore: { value: 1, display: "100%", samples: 1, total: 1, refs: [locator("@1d2d2d2d")] },
|
|
313
|
+
durationMs: 5_000,
|
|
314
|
+
costUSD: 0.02,
|
|
315
|
+
locator: locator("@1d2d2d2d"),
|
|
363
316
|
};
|
|
364
317
|
|
|
365
318
|
export const attemptListItems: AttemptListItem[] = [failedAttempt, erroredAttempt];
|
|
366
319
|
|
|
367
320
|
export const evalListItems: EvalListItem[] = [
|
|
368
321
|
{
|
|
369
|
-
evalId: "algebra/quadratic",
|
|
370
322
|
experimentId: "compare/bub",
|
|
323
|
+
evalId: "algebra/quadratic",
|
|
371
324
|
verdict: "failed",
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
325
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
326
|
+
durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
327
|
+
costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
375
328
|
attempts: [failedAttempt],
|
|
376
329
|
},
|
|
377
330
|
{
|
|
378
|
-
evalId: "geometry/angles",
|
|
379
331
|
experimentId: "compare/codex",
|
|
332
|
+
evalId: "geometry/angles",
|
|
380
333
|
verdict: "errored",
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
334
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
|
|
335
|
+
durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
|
|
336
|
+
costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
384
337
|
attempts: [erroredAttempt],
|
|
385
338
|
},
|
|
386
339
|
];
|
|
@@ -391,10 +344,10 @@ export const experimentListItems: ExperimentListItem[] = [
|
|
|
391
344
|
agent: "bub",
|
|
392
345
|
model: "gpt-5.4",
|
|
393
346
|
flags: { memory: true },
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
347
|
+
evalVerdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
|
|
348
|
+
endToEndPassRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
|
|
349
|
+
costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
|
|
350
|
+
durationMs: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
|
|
398
351
|
tokens: { value: null, display: "—", samples: 0, total: 2, refs: [] },
|
|
399
352
|
evals: 2,
|
|
400
353
|
attempts: 3,
|
|
@@ -403,26 +356,26 @@ export const experimentListItems: ExperimentListItem[] = [
|
|
|
403
356
|
{
|
|
404
357
|
evalId: "algebra/quadratic",
|
|
405
358
|
verdict: "failed",
|
|
406
|
-
|
|
407
|
-
|
|
359
|
+
durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
360
|
+
costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
408
361
|
attempts: [failedAttempt, failedRetryAttempt],
|
|
409
362
|
},
|
|
410
363
|
{
|
|
411
364
|
evalId: "algebra/simple",
|
|
412
365
|
verdict: "passed",
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
attempts: [],
|
|
366
|
+
durationMs: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
|
|
367
|
+
costUSD: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
|
|
368
|
+
attempts: [passedAttempt],
|
|
416
369
|
},
|
|
417
370
|
],
|
|
418
371
|
},
|
|
419
372
|
{
|
|
420
373
|
experimentId: "compare/codex",
|
|
421
374
|
agent: "codex",
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
375
|
+
evalVerdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
|
|
376
|
+
endToEndPassRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
|
|
377
|
+
costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
378
|
+
durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
|
|
426
379
|
tokens: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
427
380
|
evals: 1,
|
|
428
381
|
attempts: 1,
|
|
@@ -431,8 +384,8 @@ export const experimentListItems: ExperimentListItem[] = [
|
|
|
431
384
|
{
|
|
432
385
|
evalId: "geometry/angles",
|
|
433
386
|
verdict: "errored",
|
|
434
|
-
|
|
435
|
-
|
|
387
|
+
durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
|
|
388
|
+
costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
436
389
|
attempts: [erroredAttempt],
|
|
437
390
|
},
|
|
438
391
|
],
|