niceeval 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/zh-CN.d.ts +3 -3
- package/dist/report/aggregate.d.ts +32 -26
- package/dist/report/aggregate.js +157 -76
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +91 -164
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +86 -73
- package/dist/report/compute.js +592 -432
- package/dist/report/flag.d.ts +28 -17
- package/dist/report/flag.js +86 -16
- package/dist/report/format.d.ts +11 -11
- package/dist/report/format.js +17 -15
- package/dist/report/index.d.ts +16 -17
- package/dist/report/index.js +20 -22
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +49 -34
- package/dist/report/locale.js +106 -58
- package/dist/report/metrics.d.ts +10 -3
- package/dist/report/metrics.js +46 -12
- package/dist/report/primitives.d.ts +42 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +10 -8
- package/dist/report/react/AttemptList.js +18 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +3 -3
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +4 -2
- package/dist/report/react/ExperimentComparison.js +5 -4
- package/dist/report/react/ExperimentList.d.ts +3 -3
- package/dist/report/react/ExperimentList.js +16 -15
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +54 -17
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +105 -149
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -20
- package/dist/report/report.js +261 -34
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +225 -157
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +2 -2
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +245 -300
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +38 -14
- package/dist/shared/aggregate.d.ts +3 -2
- package/dist/shared/aggregate.js +5 -4
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
- package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
- package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
- package/docs-site/zh/index.mdx +23 -25
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +6 -6
- package/docs-site/zh/reference/cli.mdx +9 -7
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
- package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
- package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
- package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
- package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +6 -2
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +3 -3
- package/src/cli.ts +42 -23
- package/src/context/context.ts +1 -1
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/i18n/en.ts +18 -16
- package/src/i18n/zh-CN.ts +16 -15
- package/src/report/aggregate.ts +175 -87
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -285
- package/src/report/compute.ts +717 -515
- package/src/report/dual-render.test.tsx +738 -1148
- package/src/report/flag.ts +97 -33
- package/src/report/format.ts +18 -22
- package/src/report/index.ts +113 -58
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +120 -69
- package/src/report/metrics.ts +42 -12
- package/src/report/primitives.tsx +190 -45
- package/src/report/react/AttemptList.tsx +32 -20
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +12 -7
- package/src/report/react/ExperimentList.tsx +38 -26
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +74 -20
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/enhance.js +57 -5
- package/src/report/react/fixtures.ts +109 -156
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +139 -104
- package/src/report/react/styles.css +181 -91
- package/src/report/report.test.ts +761 -1031
- package/src/report/report.ts +425 -47
- package/src/report/text/faces.ts +257 -164
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +2 -2
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +257 -287
- package/src/report/web.ts +63 -20
- package/src/results/attempt-evidence.test.ts +4 -4
- package/src/results/attempt-evidence.ts +5 -5
- package/src/results/copy.ts +6 -6
- package/src/results/host-equivalence.test.ts +26 -14
- package/src/results/index.ts +10 -4
- package/src/results/open.ts +8 -4
- package/src/results/results.test.ts +4 -3
- package/src/results/select.ts +104 -34
- package/src/results/types.ts +36 -14
- package/src/runner/feedback/human.test.ts +1 -1
- package/src/runner/run.ts +1 -1
- package/src/sandbox/cli-commands.ts +2 -2
- package/src/scoring/judge.test.ts +1 -1
- package/src/shared/aggregate.ts +5 -4
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +107 -56
- package/src/show/render.ts +43 -27
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +86 -36
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +324 -63
- package/src/view/app/components/CodeView.tsx +10 -4
- package/src/view/app/i18n.ts +31 -17
- package/src/view/app/lib/artifact-url.ts +14 -3
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +21 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +6 -6
- package/src/view/data.test.ts +9 -3
- package/src/view/data.ts +145 -49
- package/src/view/index.ts +48 -44
- package/src/view/server.ts +35 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +224 -0
- package/src/view/view-report.test.ts +161 -57
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
- package/dist/report/built-ins/experiment-comparison.js +0 -119
- package/dist/report/built-ins/index.d.ts +0 -2
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/src/report/built-in-user-parity.test.tsx +0 -597
- package/src/report/built-ins/experiment-comparison.tsx +0 -179
- package/src/report/built-ins/index.ts +0 -7
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
package/src/results/select.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
// 快照
|
|
1
|
+
// 快照 Scope 与 attempt 去重(定稿见 docs/feature/results/library.md「选择快照」「官方现刻水位」「身份键与去重」)。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
// 选择器必须诚实:残缺、落后、未收尾都被算出来,以结构化 warnings 随
|
|
5
|
-
// 渲染与否在消费方(message
|
|
3
|
+
// 选择器长在集合上(results.latest() / results.current()),不是 DSL,只是最常用的两种口径。
|
|
4
|
+
// 选择器必须诚实:残缺、落后、未收尾都被算出来,以结构化 warnings 随 Scope 走 ——
|
|
5
|
+
// 渲染与否在消费方(message 是渲染好的英文句子,以下一步收尾),但缺口不静默。
|
|
6
6
|
|
|
7
7
|
import type {
|
|
8
8
|
AttemptHandle,
|
|
@@ -10,22 +10,23 @@ import type {
|
|
|
10
10
|
Eval,
|
|
11
11
|
Experiment,
|
|
12
12
|
Results,
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
Scope,
|
|
14
|
+
ScopeWarning,
|
|
15
15
|
Snapshot,
|
|
16
16
|
} from "./types.ts";
|
|
17
|
+
import type { ExperimentRunInfo, JsonValue } from "../types.ts";
|
|
17
18
|
import { evalPrefixPredicate } from "../shared/aggregate.ts";
|
|
18
19
|
|
|
19
20
|
/** Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。 */
|
|
20
21
|
export function selectLatest(
|
|
21
22
|
experiments: Experiment[],
|
|
22
23
|
opts?: { experiments?: string | string[] },
|
|
23
|
-
):
|
|
24
|
+
): Scope {
|
|
24
25
|
const selected = filterExperiments(experiments, opts?.experiments);
|
|
25
26
|
const snapshots = selected.map((exp) => exp.latest);
|
|
26
|
-
const warnings:
|
|
27
|
+
const warnings: ScopeWarning[] = [];
|
|
27
28
|
|
|
28
|
-
// stale 的基准:
|
|
29
|
+
// stale 的基准:Scope 中最新的落盘(无阈值,如实触发;要阈值消费方按字段自比)。
|
|
29
30
|
let latestStartedAt = "";
|
|
30
31
|
for (const snapshot of snapshots) {
|
|
31
32
|
if (snapshot.startedAt > latestStartedAt) latestStartedAt = snapshot.startedAt;
|
|
@@ -44,6 +45,7 @@ export function selectLatest(
|
|
|
44
45
|
covered,
|
|
45
46
|
total,
|
|
46
47
|
message: `snapshot covers ${covered} of ${total} evals seen in history; re-run \`niceeval exp ${exp.id}\` for a full snapshot`,
|
|
48
|
+
command: `niceeval exp ${exp.id}`,
|
|
47
49
|
});
|
|
48
50
|
}
|
|
49
51
|
if (snapshot.startedAt < latestStartedAt) {
|
|
@@ -52,7 +54,8 @@ export function selectLatest(
|
|
|
52
54
|
experimentId: exp.id,
|
|
53
55
|
startedAt: snapshot.startedAt,
|
|
54
56
|
latestStartedAt,
|
|
55
|
-
message: `snapshot "${exp.id}" (${snapshot.startedAt}) predates the latest run in this
|
|
57
|
+
message: `snapshot "${exp.id}" (${snapshot.startedAt}) predates the latest run in this scope by ${humanizeGap(snapshot.startedAt, latestStartedAt)}; re-run \`niceeval exp ${exp.id}\` to align, or ignore if evals, agent and model are unchanged between the runs`,
|
|
58
|
+
command: `niceeval exp ${exp.id}`,
|
|
56
59
|
});
|
|
57
60
|
}
|
|
58
61
|
if (!snapshot.completedAt) {
|
|
@@ -61,46 +64,101 @@ export function selectLatest(
|
|
|
61
64
|
experimentId: exp.id,
|
|
62
65
|
startedAt: snapshot.startedAt,
|
|
63
66
|
dir: snapshot.dir,
|
|
64
|
-
message: `snapshot "${exp.id}" (${snapshot.startedAt}) has no completedAt — the run was interrupted;
|
|
67
|
+
message: `snapshot "${exp.id}" (${snapshot.startedAt}) has no completedAt — the run was interrupted; re-run \`niceeval exp ${exp.id}\` for a complete snapshot`,
|
|
68
|
+
command: `niceeval exp ${exp.id}`,
|
|
65
69
|
});
|
|
66
70
|
}
|
|
67
71
|
}
|
|
68
|
-
return
|
|
72
|
+
return makeScope("latest-snapshots", snapshots, warnings);
|
|
69
73
|
}
|
|
70
74
|
|
|
71
75
|
/** selectCurrentResults 的范围输入:experiment id 前缀与 eval id 前缀,都可缺省。 */
|
|
72
76
|
export interface ResultScope {
|
|
73
77
|
/** experiment id 前缀(--experiment),分段匹配语义同 filterExperiments。 */
|
|
74
|
-
experiment?: string;
|
|
75
|
-
/** eval id 前缀(位置参数),收窄
|
|
78
|
+
experiment?: string | string[];
|
|
79
|
+
/** eval id 前缀(位置参数),收窄 Scope 覆盖的 eval;覆盖警告分母同步收窄到范围内。 */
|
|
76
80
|
patterns?: string[];
|
|
77
81
|
}
|
|
78
82
|
|
|
83
|
+
// ───────────────────────── 可比性配置 ─────────────────────────
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* current() 跨快照拼接的可比性前提所比较的字段集(docs/feature/results/library.md
|
|
87
|
+
* 「官方现刻水位」):会改变单题被测行为或判定的字段。runs / earlyExit / maxConcurrency /
|
|
88
|
+
* selectedEvalIds / evalFilterFingerprint / description 是编排与选题字段,不参与比较。
|
|
89
|
+
*/
|
|
90
|
+
export interface ComparabilityConfig {
|
|
91
|
+
agent: string;
|
|
92
|
+
model?: string;
|
|
93
|
+
reasoningEffort?: string;
|
|
94
|
+
flags?: Record<string, JsonValue>;
|
|
95
|
+
budget?: number;
|
|
96
|
+
timeoutMs?: number;
|
|
97
|
+
sandbox?: ExperimentRunInfo["sandbox"];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** 一个快照的可比性配置投影;pairsByFlag 与 experimentListData 复用同一字段集。 */
|
|
101
|
+
export function comparabilityConfigOf(snapshot: Snapshot): ComparabilityConfig {
|
|
102
|
+
const info = snapshot.experiment;
|
|
103
|
+
return {
|
|
104
|
+
agent: snapshot.agent,
|
|
105
|
+
...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
|
|
106
|
+
...(info?.reasoningEffort !== undefined ? { reasoningEffort: info.reasoningEffort } : {}),
|
|
107
|
+
...(info?.flags !== undefined ? { flags: info.flags } : {}),
|
|
108
|
+
...(info?.budget !== undefined ? { budget: info.budget } : {}),
|
|
109
|
+
...(info?.timeoutMs !== undefined ? { timeoutMs: info.timeoutMs } : {}),
|
|
110
|
+
...(info?.sandbox !== undefined ? { sandbox: info.sandbox } : {}),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** 可序列化值的深相等(对象键序无关;undefined 字段与缺席字段等价)。 */
|
|
115
|
+
export function deepEqualJson(a: unknown, b: unknown): boolean {
|
|
116
|
+
if (a === b) return true;
|
|
117
|
+
if (Array.isArray(a) || Array.isArray(b)) {
|
|
118
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
|
|
119
|
+
return a.every((item, i) => deepEqualJson(item, b[i]));
|
|
120
|
+
}
|
|
121
|
+
if (typeof a === "object" && typeof b === "object" && a !== null && b !== null) {
|
|
122
|
+
const keysA = Object.keys(a).filter((k) => (a as Record<string, unknown>)[k] !== undefined);
|
|
123
|
+
const keysB = Object.keys(b).filter((k) => (b as Record<string, unknown>)[k] !== undefined);
|
|
124
|
+
if (keysA.length !== keysB.length) return false;
|
|
125
|
+
return keysA.every((k) =>
|
|
126
|
+
deepEqualJson((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k]),
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
|
|
79
132
|
/**
|
|
80
|
-
* 两个宿主(show / view)共用的现刻水位选择器:每个 experiment × eval
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
* 向更早的 run 补齐,再把合成好的
|
|
84
|
-
*
|
|
133
|
+
* 两个宿主(show / view)共用的现刻水位选择器:每个 experiment × eval 取「包含该 eval 的
|
|
134
|
+
* 最新快照」里的全部 attempt,跨 run 合成。results.latest() 只挑「每实验最新快照」,带 eval
|
|
135
|
+
* 前缀的局部重跑会产出残缺快照;现刻水位承诺「不会因为一次局部重跑变残缺」,所以在实验的
|
|
136
|
+
* 历史快照上逐 eval 向更早的 run 补齐,再把合成好的 Scope 交给宿主注入报告槽。
|
|
137
|
+
*
|
|
138
|
+
* **可比性前提**:每个 experiment 以最新快照的可比性配置(agent / model / reasoningEffort /
|
|
139
|
+
* flags / budget / timeoutMs / sandbox)为基准,只有配置与基准深相等的历史快照才参与补齐;
|
|
140
|
+
* 改过配置后只补跑部分 eval 时,旧配置快照覆盖的其余题不冒充新配置的水位,按 partial-coverage
|
|
141
|
+
* 如实告警。这保证 current() 产出的每个 experiment 只对应一套配置。
|
|
85
142
|
*
|
|
86
143
|
* 同一 eval 的全部 attempts 必须整批取自包含它的最新快照,不把历史快照的 attempts 平铺后
|
|
87
144
|
* 按 eval 聚合——否则会把不同运行的重试混成一次虚构运行。合成快照的 dir/元数据只服务报告
|
|
88
145
|
* 分组与来源展示,证据身份一律来自 attempt 自己的 ref。
|
|
89
|
-
* 警告随 Selection 重算:partial-coverage 的分母 = 已知并集 ∩ 范围(范围收窄时分母同步收窄,
|
|
90
|
-
* 不让范围外的缺口刷屏);stale / unfinished 与 results.latest() 同口径。
|
|
91
146
|
*/
|
|
92
|
-
export function selectCurrentResults(results: Results, scope: ResultScope = {}):
|
|
147
|
+
export function selectCurrentResults(results: Results, scope: ResultScope = {}): Scope {
|
|
93
148
|
const match =
|
|
94
149
|
scope.patterns && scope.patterns.length > 0 ? evalPrefixPredicate(scope.patterns) : () => true;
|
|
95
150
|
const experiments = filterExperiments(results.experiments, scope.experiment);
|
|
96
151
|
|
|
97
152
|
const snapshots: Snapshot[] = [];
|
|
98
|
-
const warnings:
|
|
153
|
+
const warnings: ScopeWarning[] = [];
|
|
99
154
|
|
|
100
155
|
for (const exp of experiments) {
|
|
156
|
+
// 可比性基准 = 该实验最新快照的可比性配置;不一致的旧快照整份跳过,不贡献 attempt。
|
|
157
|
+
const baseline = comparabilityConfigOf(exp.latest);
|
|
101
158
|
// 逐题取最新:快照按最新在前,首个出现即最新判定
|
|
102
159
|
const taken = new Map<string, { ev: Eval; snapshot: Snapshot }>();
|
|
103
160
|
for (const snapshot of exp.snapshots) {
|
|
161
|
+
if (!deepEqualJson(comparabilityConfigOf(snapshot), baseline)) continue;
|
|
104
162
|
for (const ev of snapshot.evals) {
|
|
105
163
|
if (!match(ev.id) || taken.has(ev.id)) continue;
|
|
106
164
|
taken.set(ev.id, { ev, snapshot });
|
|
@@ -124,6 +182,8 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
|
|
|
124
182
|
startedAt,
|
|
125
183
|
agent: base.agent,
|
|
126
184
|
...(base.model !== undefined ? { model: base.model } : {}),
|
|
185
|
+
...(base.experiment !== undefined ? { experiment: base.experiment } : {}),
|
|
186
|
+
...(base.name !== undefined ? { name: base.name } : {}),
|
|
127
187
|
producer: base.producer,
|
|
128
188
|
schemaVersion: base.schemaVersion,
|
|
129
189
|
evals,
|
|
@@ -133,8 +193,8 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
|
|
|
133
193
|
...(base.knownEvalIds ? { knownEvalIds: [...base.knownEvalIds] } : {}),
|
|
134
194
|
});
|
|
135
195
|
|
|
136
|
-
//
|
|
137
|
-
//
|
|
196
|
+
// 残缺检测:跨快照补齐后仍缺,来自「历史上见过却从未在可比配置的可读落盘里出现」的题
|
|
197
|
+
// (含改配置后未补跑的题)—— 分母收窄到范围内,不让范围外的缺口刷屏。
|
|
138
198
|
const total = exp.evalIds.filter(match).length;
|
|
139
199
|
if (evals.length < total) {
|
|
140
200
|
warnings.push({
|
|
@@ -143,6 +203,7 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
|
|
|
143
203
|
covered: evals.length,
|
|
144
204
|
total,
|
|
145
205
|
message: `verdicts cover ${evals.length} of ${total} evals seen in history; re-run \`niceeval exp ${exp.id}\` for a full snapshot`,
|
|
206
|
+
command: `niceeval exp ${exp.id}`,
|
|
146
207
|
});
|
|
147
208
|
}
|
|
148
209
|
}
|
|
@@ -158,7 +219,8 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
|
|
|
158
219
|
experimentId: snapshot.experimentId,
|
|
159
220
|
startedAt: snapshot.startedAt,
|
|
160
221
|
latestStartedAt,
|
|
161
|
-
message: `verdicts for "${snapshot.experimentId}" were produced at ${snapshot.startedAt}, before the latest run in this
|
|
222
|
+
message: `verdicts for "${snapshot.experimentId}" were produced at ${snapshot.startedAt}, ${humanizeGap(snapshot.startedAt, latestStartedAt)} before the latest run in this scope; re-run \`niceeval exp ${snapshot.experimentId}\` to align, or ignore if evals, agent and model are unchanged between the runs`,
|
|
223
|
+
command: `niceeval exp ${snapshot.experimentId}`,
|
|
162
224
|
});
|
|
163
225
|
}
|
|
164
226
|
if (snapshot.completedAt === undefined) {
|
|
@@ -167,37 +229,45 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
|
|
|
167
229
|
experimentId: snapshot.experimentId,
|
|
168
230
|
startedAt: snapshot.startedAt,
|
|
169
231
|
dir: snapshot.dir,
|
|
170
|
-
message: `snapshot "${snapshot.experimentId}" (${snapshot.startedAt}) is unfinished (the process was interrupted); completed attempts are read as-is, but the set may be incomplete`,
|
|
232
|
+
message: `snapshot "${snapshot.experimentId}" (${snapshot.startedAt}) is unfinished (the process was interrupted); completed attempts are read as-is, but the set may be incomplete — re-run \`niceeval exp ${snapshot.experimentId}\` for a complete snapshot`,
|
|
233
|
+
command: `niceeval exp ${snapshot.experimentId}`,
|
|
171
234
|
});
|
|
172
235
|
}
|
|
173
236
|
}
|
|
174
237
|
|
|
175
|
-
return
|
|
238
|
+
return makeScope("current-evals", snapshots, warnings);
|
|
176
239
|
}
|
|
177
240
|
|
|
178
241
|
/**
|
|
179
|
-
*
|
|
180
|
-
*
|
|
242
|
+
* Scope 构造:attempts 按口径物化(快照 attempts 的平铺);filter 只删不换 —— 快照删减,
|
|
243
|
+
* attempts 随之同步修剪,warnings 修剪规则是「experimentId 不在幸存快照中的丢弃,
|
|
244
|
+
* 非实验作用域的保留」(为将来非 per-experiment 的 kind 留位置)。
|
|
181
245
|
*/
|
|
182
|
-
export function
|
|
246
|
+
export function makeScope(
|
|
247
|
+
mode: Scope["mode"],
|
|
248
|
+
snapshots: Snapshot[],
|
|
249
|
+
warnings: ScopeWarning[],
|
|
250
|
+
): Scope {
|
|
183
251
|
return {
|
|
252
|
+
mode,
|
|
184
253
|
snapshots,
|
|
254
|
+
attempts: snapshots.flatMap((s) => s.attempts),
|
|
185
255
|
warnings,
|
|
186
|
-
filter(predicate: (snapshot: Snapshot) => boolean):
|
|
256
|
+
filter(predicate: (snapshot: Snapshot) => boolean): Scope {
|
|
187
257
|
const kept = snapshots.filter(predicate);
|
|
188
258
|
const survivors = new Set(kept.map((s) => s.experimentId));
|
|
189
259
|
const keptWarnings = warnings.filter((w) => {
|
|
190
260
|
const scope = (w as { experimentId?: unknown }).experimentId;
|
|
191
261
|
return typeof scope !== "string" || survivors.has(scope);
|
|
192
262
|
});
|
|
193
|
-
return
|
|
263
|
+
return makeScope(mode, kept, keptWarnings);
|
|
194
264
|
},
|
|
195
265
|
};
|
|
196
266
|
}
|
|
197
267
|
|
|
198
268
|
/**
|
|
199
269
|
* 跨快照聚合前的身份键去重:(experimentId, evalId, attempt, startedAt)。
|
|
200
|
-
*
|
|
270
|
+
* 携带合入会把上一轮已通过的结果原样合入新快照,同一 attempt 因此存在于多份落盘;
|
|
201
271
|
* 重复时保留最新快照里的那份(内容相同,取新快照的副本让 ref 落在最新落盘上;
|
|
202
272
|
* 位置取首次出现处,顺序稳定)。startedAt 缺失时宁可不去重也不误删,记入 warnings。
|
|
203
273
|
*/
|
package/src/results/types.ts
CHANGED
|
@@ -171,33 +171,49 @@ export interface Results {
|
|
|
171
171
|
experiments: Experiment[];
|
|
172
172
|
skipped: SkippedDir[];
|
|
173
173
|
/**
|
|
174
|
-
* 每个实验取最新一次快照,返回
|
|
174
|
+
* 每个实验取最新一次快照,返回 Scope(快照与挑选警告绑在一起走)。
|
|
175
175
|
* `experiments` 是 experiment id 前缀过滤(string | string[]),同 CLI 位置参数语义。
|
|
176
176
|
*/
|
|
177
|
-
latest(opts?: { experiments?: string | string[] }):
|
|
177
|
+
latest(opts?: { experiments?: string | string[] }): Scope;
|
|
178
|
+
/**
|
|
179
|
+
* 官方现刻水位:每个 experiment × eval 取「包含该 eval 的最新快照」里的全部 attempt,
|
|
180
|
+
* 跨历史拼出当前判定水位。可比性前提:每个 experiment 以最新快照的可比性配置
|
|
181
|
+
* (agent / model / reasoningEffort / flags / budget / timeoutMs / sandbox)为基准,
|
|
182
|
+
* 配置不一致的旧快照不贡献 attempt,缺口走 partial-coverage
|
|
183
|
+
* (见 docs/feature/results/library.md「官方现刻水位」)。
|
|
184
|
+
*/
|
|
185
|
+
current(opts?: { experiments?: string | string[] }): Scope;
|
|
178
186
|
}
|
|
179
187
|
|
|
180
188
|
/**
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
*
|
|
189
|
+
* Scope(范围):选出的快照 + 口径 + 已按口径物化的 attempt 全集 + 挑选警告。
|
|
190
|
+
* 渲染与否在消费方,但缺口永远被算出来。下游(Reports 计算函数、copySnapshots)收
|
|
191
|
+
* `Scope | readonly Snapshot[]`;手工挑的裸数组没有挑选过程,自然没有 warnings 可带,也如实。
|
|
184
192
|
*/
|
|
185
|
-
export interface
|
|
193
|
+
export interface Scope {
|
|
194
|
+
/** 这份 Scope 的口径,字面写在数据上。 */
|
|
195
|
+
mode: "latest-snapshots" | "current-evals";
|
|
186
196
|
snapshots: Snapshot[];
|
|
187
|
-
warnings: SelectionWarning[];
|
|
188
197
|
/**
|
|
189
|
-
*
|
|
190
|
-
*
|
|
198
|
+
* 按口径物化的 attempt 全集:消费 attempts 就自动正确,不需要自己 flatten snapshots,
|
|
199
|
+
* 也就不可能算错口径。官方计算函数同样只消费它。
|
|
200
|
+
*/
|
|
201
|
+
attempts: AttemptHandle[];
|
|
202
|
+
warnings: ScopeWarning[];
|
|
203
|
+
/**
|
|
204
|
+
* 只删不换:返回新 Scope,快照删减,attempts 与 warnings 随之同步修剪 ——
|
|
205
|
+
* experimentId 不在幸存快照中的警告丢弃,非实验作用域的警告保留。
|
|
191
206
|
* 「换成上一个完整快照」这类替换式重挑不给方法,回 exp.snapshots 自己挑。
|
|
192
207
|
*/
|
|
193
|
-
filter(predicate: (snapshot: Snapshot) => boolean):
|
|
208
|
+
filter(predicate: (snapshot: Snapshot) => boolean): Scope;
|
|
194
209
|
}
|
|
195
210
|
|
|
196
211
|
/**
|
|
197
|
-
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message
|
|
198
|
-
* kind
|
|
212
|
+
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message;能用一条命令直接推进的
|
|
213
|
+
* kind 同时带 `command`(已替换真实 id,复制即跑)。kind 是契约的一部分,全集与触发条件见
|
|
214
|
+
* docs/feature/results/library.md「警告 kind 全集」。
|
|
199
215
|
*/
|
|
200
|
-
export type
|
|
216
|
+
export type ScopeWarning =
|
|
201
217
|
| {
|
|
202
218
|
/** 选中快照的覆盖 < 该实验已知 eval 并集(本地历史 ∪ knownEvalIds)。 */
|
|
203
219
|
kind: "partial-coverage";
|
|
@@ -205,14 +221,18 @@ export type SelectionWarning =
|
|
|
205
221
|
covered: number;
|
|
206
222
|
total: number;
|
|
207
223
|
message: string;
|
|
224
|
+
/** 一条可复制即跑的推进命令:`niceeval exp <experimentId>`。 */
|
|
225
|
+
command: string;
|
|
208
226
|
}
|
|
209
227
|
| {
|
|
210
|
-
/** 该实验选中的快照早于
|
|
228
|
+
/** 该实验选中的快照早于 Scope 中最新的落盘;无阈值,如实触发,要阈值消费方按字段自比。 */
|
|
211
229
|
kind: "stale-snapshot";
|
|
212
230
|
experimentId: string;
|
|
213
231
|
startedAt: string;
|
|
214
232
|
latestStartedAt: string;
|
|
215
233
|
message: string;
|
|
234
|
+
/** 一条可复制即跑的推进命令:`niceeval exp <experimentId>`。 */
|
|
235
|
+
command: string;
|
|
216
236
|
}
|
|
217
237
|
| {
|
|
218
238
|
/** 选中快照缺 completedAt(进程中断,未收尾);已落盘 attempt 照常读出,警告提示集合可能不完整。 */
|
|
@@ -222,6 +242,8 @@ export type SelectionWarning =
|
|
|
222
242
|
/** 该快照目录的绝对路径。 */
|
|
223
243
|
dir: string;
|
|
224
244
|
message: string;
|
|
245
|
+
/** 一条可复制即跑的推进命令:`niceeval exp <experimentId>`。 */
|
|
246
|
+
command: string;
|
|
225
247
|
};
|
|
226
248
|
|
|
227
249
|
/** dedupeAttempts 的警告:身份键缺 startedAt,宁可不去重也不误删。 */
|
|
@@ -353,7 +353,7 @@ describe("完成页:失败优先摘要 + locator + show/view 下一步 + 快照
|
|
|
353
353
|
expect(text).toContain("gate: Issue 15193: selected proposal matches the accepted proposal");
|
|
354
354
|
expect(text).toContain("equals(4) · expected 4 · received 3");
|
|
355
355
|
expect(text).toContain("Inspect: niceeval show @17m2k9p");
|
|
356
|
-
expect(text).toContain("Eval: niceeval show @17m2k9p --
|
|
356
|
+
expect(text).toContain("Eval: niceeval show @17m2k9p --source");
|
|
357
357
|
expect(text).toContain("Trace: niceeval show @17m2k9p --execution");
|
|
358
358
|
expect(text).toContain("Diff: niceeval show @17m2k9p --diff");
|
|
359
359
|
expect(text).toContain("Compare: niceeval view compare");
|
package/src/runner/run.ts
CHANGED
|
@@ -265,7 +265,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
|
|
|
265
265
|
// budget 护栏:只按「已完成 attempt 的实测花费」判断,不做预测性节流。之前的实现会按
|
|
266
266
|
// 「平均成本 × 在飞数」预扣,快到顶就让还没起飞的 attempt 排队等——这在探测阶段(还没有任何
|
|
267
267
|
// 成本样本时)等价于把同一 budgetKey 的并发摁到一个很小的数,且完全没有文档承诺过这个副作用
|
|
268
|
-
// (`docs-site/zh/
|
|
268
|
+
// (`docs-site/zh/how-to/write-experiment.mdx` 对 `budget` 的描述只有一句「这一格配置的预算
|
|
269
269
|
// 上限」)。新语义:已完成 attempt 的花费加总一旦到顶,就不再放新 attempt 起飞(已经在飞的
|
|
270
270
|
// 照常跑完,不会被中途打断);到顶之前不做任何预测性限流,并发完全由 globalSem / runSem 决定。
|
|
271
271
|
// 代价是「已花 + 在飞未结算」的总花费可能短暂超出 budget——这是有意识的取舍:budget 是防止
|
|
@@ -47,7 +47,7 @@ export async function runSandboxCommand(
|
|
|
47
47
|
const root = await resolveRegistryRoot(cwd, flags.run);
|
|
48
48
|
if (root === undefined) {
|
|
49
49
|
io.err(
|
|
50
|
-
`No .niceeval directory found from ${cwd} upward. Run this inside the project, or pass --
|
|
50
|
+
`No .niceeval directory found from ${cwd} upward. Run this inside the project, or pass --results <results-root> to point at it.\n`,
|
|
51
51
|
);
|
|
52
52
|
return 1;
|
|
53
53
|
}
|
|
@@ -71,7 +71,7 @@ export async function runSandboxCommand(
|
|
|
71
71
|
async function resolveRegistryRoot(cwd: string, runFlag: string | undefined): Promise<string | undefined> {
|
|
72
72
|
if (runFlag !== undefined) {
|
|
73
73
|
const base = resolve(cwd, runFlag);
|
|
74
|
-
// --
|
|
74
|
+
// --results 可以指 .niceeval 本身或它的父目录。
|
|
75
75
|
return base.endsWith(".niceeval") ? base : `${base}/.niceeval`;
|
|
76
76
|
}
|
|
77
77
|
return findNiceevalRoot(cwd);
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// judge 解析与请求材料的单测:端点/凭据/模型解析结果必须进入真实请求,低分过不了 .gate()。
|
|
3
3
|
// fixture judge client = 截获 globalThis.fetch(autoevals 底层 openai client 走全局 fetch),
|
|
4
4
|
// 不起 HTTP server、不 spawn CLI。契约见 docs/feature/scoring/library/judge.md 与
|
|
5
|
-
// docs-site/zh/
|
|
5
|
+
// docs-site/zh/explanation/judge.mdx 的解析优先级表;用例登记在
|
|
6
6
|
// docs/engineering/unit-tests/scoring/cases.md 的 Judge 分区。
|
|
7
7
|
|
|
8
8
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
package/src/shared/aggregate.ts
CHANGED
|
@@ -45,13 +45,14 @@ export function experimentGroupOf(experimentId: string): string | undefined {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
|
-
* eval id 前缀过滤,同 CLI
|
|
49
|
-
* "algebra"
|
|
48
|
+
* eval id 前缀过滤,同 CLI 位置参数语义(docs/feature/reports/show.md「打开与收窄」):
|
|
49
|
+
* eval 位置参数是收窄过滤,按**裸前缀宽松匹配**——"algebra" 命中 "algebra"、"algebra/..."
|
|
50
|
+
* 也命中 "algebra2",多命中正是它的用途(与 `--experiment` 的按路径段匹配有意不同)。
|
|
50
51
|
*/
|
|
51
52
|
export function evalPrefixPredicate(evals?: string | string[]): (id: string) => boolean {
|
|
52
53
|
if (evals === undefined) return () => true;
|
|
53
|
-
const prefixes =
|
|
54
|
-
return (id) => prefixes.some((prefix) => id
|
|
54
|
+
const prefixes = Array.isArray(evals) ? evals : [evals];
|
|
55
|
+
return (id) => prefixes.some((prefix) => id.startsWith(prefix));
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
/** 无 experimentId 时的兜底标签。 */
|
package/src/show/compose.ts
CHANGED
|
@@ -1,95 +1,78 @@
|
|
|
1
|
-
// show
|
|
1
|
+
// show 专属的执行时间轴口径(--history;契约:docs/feature/reports/show.md「--history:一个 eval
|
|
2
|
+
// 的执行时间轴」)。逐 attempt 而非逐快照:对 Scope 中匹配的每个 experimentId + evalId 分节,
|
|
3
|
+
// 节内按 startedAt 升序列出跨快照按 attempt 身份键去重后的历次 attempt——时间、verdict、
|
|
4
|
+
// 单行结果摘要(Scoring display 契约)、耗时、成本与 locator。resume 携带的复印件不占行。
|
|
2
5
|
//
|
|
3
|
-
// 现刻水位
|
|
4
|
-
// 本文件只留 show 独有的时间轴计算:每个快照 / 每次真实执行一行,resume 携带的复印件不占行。
|
|
6
|
+
// 现刻水位 Scope(两个宿主共用)住在 ../results/select.ts;本文件只留 show 独有的时间轴计算。
|
|
5
7
|
// 数据只消费 niceeval/results 的读取面。
|
|
6
8
|
|
|
7
|
-
import { foldEvalVerdict } from "../shared/verdict.ts";
|
|
8
9
|
import { attemptCostUSD } from "../report/metrics.ts";
|
|
9
|
-
import
|
|
10
|
+
import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../scoring/display.ts";
|
|
11
|
+
import type { EvalResult, Verdict } from "../types.ts";
|
|
10
12
|
import type { AttemptHandle, Experiment } from "../results/index.ts";
|
|
11
13
|
|
|
12
14
|
// ───────────────────────── 时间轴(--history)─────────────────────────
|
|
13
15
|
|
|
14
|
-
export interface
|
|
15
|
-
/**
|
|
16
|
-
startedAt
|
|
16
|
+
export interface AttemptHistoryRow {
|
|
17
|
+
/** 该 attempt 自己的开始时刻(ISO);第三方落盘可能缺失,如实缺省、排序沉底。 */
|
|
18
|
+
startedAt?: string;
|
|
17
19
|
verdict: Verdict;
|
|
18
|
-
|
|
20
|
+
/** 单行结果摘要(display 契约):主失败断言 / 结构化 error 一层摘要 / skip 理由;passed 缺省。 */
|
|
21
|
+
summary?: string;
|
|
22
|
+
durationMs: number;
|
|
19
23
|
costUSD: number | null;
|
|
20
|
-
/**
|
|
21
|
-
|
|
22
|
-
/** errored 时的错误摘要。 */
|
|
23
|
-
error?: string;
|
|
24
|
+
/** attempt 的稳定引用(`@` 前缀),复制给 `niceeval show @<locator>` 下钻。 */
|
|
25
|
+
locator?: string;
|
|
24
26
|
}
|
|
25
27
|
|
|
26
|
-
/** attempt 的身份键(去重口径与 dedupeAttempts 一致);缺 startedAt 时不参与去重(宁可多列不误删)。 */
|
|
28
|
+
/** attempt 的身份键(去重口径与 results 的 dedupeAttempts 一致);缺 startedAt 时不参与去重(宁可多列不误删)。 */
|
|
27
29
|
function attemptKey(attempt: AttemptHandle): string | undefined {
|
|
28
30
|
const r = attempt.result;
|
|
29
31
|
return r.startedAt === undefined ? undefined : `${r.attempt}|${r.startedAt}`;
|
|
30
32
|
}
|
|
31
33
|
|
|
32
34
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
+
* 单行结果摘要:与榜单 Result 单元格同一条 display 契约(docs/feature/scoring/library/display.md)——
|
|
36
|
+
* 结构化 error 取一层 message 摘要,skipped 取理由,failed 取主失败断言的紧凑单行;passed 无摘要。
|
|
35
37
|
*/
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
function rowSummary(result: EvalResult): string | undefined {
|
|
39
|
+
if (result.error !== undefined) return summaryText(result.error.message);
|
|
40
|
+
if (result.skipReason !== undefined) return summaryText(result.skipReason);
|
|
41
|
+
const summary = primaryAssertionSummary(result.assertions, result.verdict);
|
|
42
|
+
return summary === undefined ? undefined : compactAssertionSummary(summary);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* 一个 experimentId + evalId 的执行时间轴:跨快照收集全部 attempt,按身份键去重
|
|
47
|
+
* (--resume 携带的复印件不占行),startedAt 升序;缺 startedAt 的行按发现顺序沉底。
|
|
48
|
+
*/
|
|
49
|
+
export function attemptHistory(exp: Experiment, evalId: string): AttemptHistoryRow[] {
|
|
38
50
|
const seen = new Set<string>();
|
|
39
|
-
|
|
40
|
-
|
|
51
|
+
const dated: AttemptHistoryRow[] = [];
|
|
52
|
+
const undated: AttemptHistoryRow[] = [];
|
|
53
|
+
// 新→旧扫描(exp.snapshots 已按新→旧排序):同一身份键保留最新落盘里的那份
|
|
54
|
+
// (locator 在携带条目上原样复制,取哪份行内容相同;取最新与 dedupeAttempts 口径一致)。
|
|
55
|
+
for (const snapshot of exp.snapshots) {
|
|
41
56
|
const ev = snapshot.evals.find((e) => e.id === evalId);
|
|
42
57
|
if (!ev) continue;
|
|
43
|
-
const fresh: AttemptHandle[] = [];
|
|
44
58
|
for (const attempt of ev.attempts) {
|
|
45
59
|
const key = attemptKey(attempt);
|
|
46
|
-
if (key
|
|
47
|
-
|
|
48
|
-
|
|
60
|
+
if (key !== undefined) {
|
|
61
|
+
if (seen.has(key)) continue;
|
|
62
|
+
seen.add(key);
|
|
49
63
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
64
|
+
const r = attempt.result;
|
|
65
|
+
const row: AttemptHistoryRow = {
|
|
66
|
+
...(r.startedAt !== undefined ? { startedAt: r.startedAt } : {}),
|
|
67
|
+
verdict: r.verdict,
|
|
68
|
+
...(rowSummary(r) !== undefined ? { summary: rowSummary(r) } : {}),
|
|
69
|
+
durationMs: r.durationMs,
|
|
70
|
+
costUSD: attemptCostUSD(r),
|
|
71
|
+
...(attempt.locator !== undefined ? { locator: attempt.locator } : {}),
|
|
72
|
+
};
|
|
73
|
+
(row.startedAt === undefined ? undated : dated).push(row);
|
|
59
74
|
}
|
|
60
|
-
const latest = fresh[fresh.length - 1];
|
|
61
|
-
const failed = latest.result.assertions.find((a) => a.outcome !== "passed");
|
|
62
|
-
rows.push({
|
|
63
|
-
startedAt: snapshot.startedAt,
|
|
64
|
-
verdict: foldEvalVerdict(fresh.map((a) => a.result)),
|
|
65
|
-
attempts: fresh.length,
|
|
66
|
-
costUSD: cost,
|
|
67
|
-
...(failed ? { failedAssertion: `${failed.severity} ${failed.name}` } : {}),
|
|
68
|
-
...(latest.result.error !== undefined ? { error: latest.result.error.message } : {}),
|
|
69
|
-
});
|
|
70
75
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
export interface ExperimentHistoryRow {
|
|
75
|
-
startedAt: string;
|
|
76
|
-
passedEvals: number;
|
|
77
|
-
totalEvals: number;
|
|
78
|
-
costUSD: number | null;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** 实验级 per-run 通过率序列(裸 `show --history`):每个快照一行,新→旧。 */
|
|
82
|
-
export function experimentHistory(exp: Experiment): ExperimentHistoryRow[] {
|
|
83
|
-
return exp.snapshots.map((snapshot) => {
|
|
84
|
-
let passed = 0;
|
|
85
|
-
for (const ev of snapshot.evals) {
|
|
86
|
-
if (foldEvalVerdict(ev.attempts.map((a) => a.result)) === "passed") passed += 1;
|
|
87
|
-
}
|
|
88
|
-
let cost: number | null = null;
|
|
89
|
-
for (const attempt of snapshot.attempts) {
|
|
90
|
-
const c = attemptCostUSD(attempt.result);
|
|
91
|
-
if (c !== null) cost = (cost ?? 0) + c;
|
|
92
|
-
}
|
|
93
|
-
return { startedAt: snapshot.startedAt, passedEvals: passed, totalEvals: snapshot.evals.length, costUSD: cost };
|
|
94
|
-
});
|
|
76
|
+
dated.sort((a, b) => (a.startedAt! < b.startedAt! ? -1 : a.startedAt! > b.startedAt! ? 1 : 0));
|
|
77
|
+
return [...dated, ...undated.reverse()];
|
|
95
78
|
}
|