niceeval 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +5 -5
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +11 -11
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +12 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +54 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +34 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/results/select.ts
CHANGED
|
@@ -1,30 +1,31 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 快照 Selection 与 attempt 去重(定稿见 docs/feature/results/library.md「选择快照」「身份键与去重」)。
|
|
2
2
|
//
|
|
3
3
|
// 选择器只有一个(latest),长在集合上;它不是 DSL,只是最常用的那次筛选。
|
|
4
|
-
//
|
|
4
|
+
// 选择器必须诚实:残缺、落后、未收尾都被算出来,以结构化 warnings 随 Selection 走 ——
|
|
5
5
|
// 渲染与否在消费方(message 是渲染好的英文句子),但缺口不静默。
|
|
6
6
|
|
|
7
|
-
import { experimentKeyOf } from "./format.ts";
|
|
8
7
|
import type {
|
|
9
8
|
AttemptHandle,
|
|
10
9
|
DedupeWarning,
|
|
10
|
+
Eval,
|
|
11
11
|
Experiment,
|
|
12
|
-
|
|
12
|
+
Results,
|
|
13
13
|
Selection,
|
|
14
14
|
SelectionWarning,
|
|
15
15
|
Snapshot,
|
|
16
16
|
} from "./types.ts";
|
|
17
|
+
import { evalPrefixPredicate } from "../shared/aggregate.ts";
|
|
17
18
|
|
|
18
19
|
/** Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。 */
|
|
19
20
|
export function selectLatest(
|
|
20
21
|
experiments: Experiment[],
|
|
21
22
|
opts?: { experiments?: string | string[] },
|
|
22
23
|
): Selection {
|
|
23
|
-
const selected =
|
|
24
|
+
const selected = filterExperiments(experiments, opts?.experiments);
|
|
24
25
|
const snapshots = selected.map((exp) => exp.latest);
|
|
25
26
|
const warnings: SelectionWarning[] = [];
|
|
26
27
|
|
|
27
|
-
// stale
|
|
28
|
+
// stale 的基准:Selection 中最新的落盘(无阈值,如实触发;要阈值消费方按字段自比)。
|
|
28
29
|
let latestStartedAt = "";
|
|
29
30
|
for (const snapshot of snapshots) {
|
|
30
31
|
if (snapshot.startedAt > latestStartedAt) latestStartedAt = snapshot.startedAt;
|
|
@@ -37,16 +38,12 @@ export function selectLatest(
|
|
|
37
38
|
const covered = snapshot.evals.length;
|
|
38
39
|
const total = exp.evalIds.length;
|
|
39
40
|
if (covered < total) {
|
|
40
|
-
// 合成键不是真实 experiment id,拼不出可执行的 `niceeval exp` 命令,提示退化成中性说法。
|
|
41
|
-
const hint = snapshot.synthetic
|
|
42
|
-
? "re-run the experiment for a full snapshot"
|
|
43
|
-
: `re-run \`niceeval exp ${exp.id}\` for a full snapshot`;
|
|
44
41
|
warnings.push({
|
|
45
42
|
kind: "partial-coverage",
|
|
46
43
|
experimentId: exp.id,
|
|
47
44
|
covered,
|
|
48
45
|
total,
|
|
49
|
-
message: `snapshot covers ${covered} of ${total} evals seen in history; ${
|
|
46
|
+
message: `snapshot covers ${covered} of ${total} evals seen in history; re-run \`niceeval exp ${exp.id}\` for a full snapshot`,
|
|
50
47
|
});
|
|
51
48
|
}
|
|
52
49
|
if (snapshot.startedAt < latestStartedAt) {
|
|
@@ -58,18 +55,126 @@ export function selectLatest(
|
|
|
58
55
|
message: `snapshot "${exp.id}" (${snapshot.startedAt}) predates the latest run in this selection by ${humanizeGap(snapshot.startedAt, latestStartedAt)}`,
|
|
59
56
|
});
|
|
60
57
|
}
|
|
61
|
-
if (snapshot.
|
|
58
|
+
if (!snapshot.completedAt) {
|
|
62
59
|
warnings.push({
|
|
63
|
-
kind: "
|
|
60
|
+
kind: "unfinished-snapshot",
|
|
64
61
|
experimentId: exp.id,
|
|
65
|
-
|
|
66
|
-
|
|
62
|
+
startedAt: snapshot.startedAt,
|
|
63
|
+
dir: snapshot.dir,
|
|
64
|
+
message: `snapshot "${exp.id}" (${snapshot.startedAt}) has no completedAt — the run was interrupted; results may be incomplete`,
|
|
67
65
|
});
|
|
68
66
|
}
|
|
69
67
|
}
|
|
70
68
|
return makeSelection(snapshots, warnings);
|
|
71
69
|
}
|
|
72
70
|
|
|
71
|
+
/** selectCurrentResults 的范围输入:experiment id 前缀与 eval id 前缀,都可缺省。 */
|
|
72
|
+
export interface ResultScope {
|
|
73
|
+
/** experiment id 前缀(--experiment),分段匹配语义同 filterExperiments。 */
|
|
74
|
+
experiment?: string;
|
|
75
|
+
/** eval id 前缀(位置参数),收窄 Selection 覆盖的 eval;覆盖警告分母同步收窄到范围内。 */
|
|
76
|
+
patterns?: string[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* 两个宿主(show / view)共用的现刻水位选择器:每个 experiment × eval 取时间上最新的那份
|
|
81
|
+
* 判定,跨 run 合成。results.latest() 只挑「每实验最新快照」,带 eval 前缀的局部重跑会产出
|
|
82
|
+
* 残缺快照;现刻水位承诺「不会因为一次局部重跑变残缺」,所以在实验的全部历史快照上逐 eval
|
|
83
|
+
* 向更早的 run 补齐,再把合成好的 Selection 交给宿主注入报告槽——内置默认报告与 --report 吃
|
|
84
|
+
* 同一份。
|
|
85
|
+
*
|
|
86
|
+
* 同一 eval 的全部 attempts 必须整批取自包含它的最新快照,不把历史快照的 attempts 平铺后
|
|
87
|
+
* 按 eval 聚合——否则会把不同运行的重试混成一次虚构运行。合成快照的 dir/元数据只服务报告
|
|
88
|
+
* 分组与来源展示,证据身份一律来自 attempt 自己的 ref。
|
|
89
|
+
* 警告随 Selection 重算:partial-coverage 的分母 = 已知并集 ∩ 范围(范围收窄时分母同步收窄,
|
|
90
|
+
* 不让范围外的缺口刷屏);stale / unfinished 与 results.latest() 同口径。
|
|
91
|
+
*/
|
|
92
|
+
export function selectCurrentResults(results: Results, scope: ResultScope = {}): Selection {
|
|
93
|
+
const match =
|
|
94
|
+
scope.patterns && scope.patterns.length > 0 ? evalPrefixPredicate(scope.patterns) : () => true;
|
|
95
|
+
const experiments = filterExperiments(results.experiments, scope.experiment);
|
|
96
|
+
|
|
97
|
+
const snapshots: Snapshot[] = [];
|
|
98
|
+
const warnings: SelectionWarning[] = [];
|
|
99
|
+
|
|
100
|
+
for (const exp of experiments) {
|
|
101
|
+
// 逐题取最新:快照按最新在前,首个出现即最新判定
|
|
102
|
+
const taken = new Map<string, { ev: Eval; snapshot: Snapshot }>();
|
|
103
|
+
for (const snapshot of exp.snapshots) {
|
|
104
|
+
for (const ev of snapshot.evals) {
|
|
105
|
+
if (!match(ev.id) || taken.has(ev.id)) continue;
|
|
106
|
+
taken.set(ev.id, { ev, snapshot });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
if (taken.size === 0) continue;
|
|
110
|
+
|
|
111
|
+
const picks = [...taken.values()].sort((a, b) => a.ev.id.localeCompare(b.ev.id));
|
|
112
|
+
let startedAt = "";
|
|
113
|
+
let newest: Snapshot = picks[0].snapshot;
|
|
114
|
+
for (const pick of picks) {
|
|
115
|
+
if (pick.snapshot.startedAt > startedAt) {
|
|
116
|
+
startedAt = pick.snapshot.startedAt;
|
|
117
|
+
newest = pick.snapshot;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const evals = picks.map((p) => p.ev);
|
|
121
|
+
const base = exp.latest;
|
|
122
|
+
snapshots.push({
|
|
123
|
+
experimentId: exp.id,
|
|
124
|
+
startedAt,
|
|
125
|
+
agent: base.agent,
|
|
126
|
+
...(base.model !== undefined ? { model: base.model } : {}),
|
|
127
|
+
producer: base.producer,
|
|
128
|
+
schemaVersion: base.schemaVersion,
|
|
129
|
+
evals,
|
|
130
|
+
attempts: evals.flatMap((ev) => ev.attempts),
|
|
131
|
+
dir: newest.dir,
|
|
132
|
+
...(newest.completedAt !== undefined ? { completedAt: newest.completedAt } : {}),
|
|
133
|
+
...(base.knownEvalIds ? { knownEvalIds: [...base.knownEvalIds] } : {}),
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
// 残缺检测:跨快照补齐后仍缺,只可能是历史上见过(或 knownEvalIds 声明过)
|
|
137
|
+
// 却从未在可读落盘里出现的题 —— 分母收窄到范围内,不让范围外的缺口刷屏。
|
|
138
|
+
const total = exp.evalIds.filter(match).length;
|
|
139
|
+
if (evals.length < total) {
|
|
140
|
+
warnings.push({
|
|
141
|
+
kind: "partial-coverage",
|
|
142
|
+
experimentId: exp.id,
|
|
143
|
+
covered: evals.length,
|
|
144
|
+
total,
|
|
145
|
+
message: `verdicts cover ${evals.length} of ${total} evals seen in history; re-run \`niceeval exp ${exp.id}\` for a full snapshot`,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
let latestStartedAt = "";
|
|
151
|
+
for (const snapshot of snapshots) {
|
|
152
|
+
if (snapshot.startedAt > latestStartedAt) latestStartedAt = snapshot.startedAt;
|
|
153
|
+
}
|
|
154
|
+
for (const snapshot of snapshots) {
|
|
155
|
+
if (snapshot.startedAt < latestStartedAt) {
|
|
156
|
+
warnings.push({
|
|
157
|
+
kind: "stale-snapshot",
|
|
158
|
+
experimentId: snapshot.experimentId,
|
|
159
|
+
startedAt: snapshot.startedAt,
|
|
160
|
+
latestStartedAt,
|
|
161
|
+
message: `verdicts for "${snapshot.experimentId}" were produced at ${snapshot.startedAt}, before the latest run in this selection (${latestStartedAt})`,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
if (snapshot.completedAt === undefined) {
|
|
165
|
+
warnings.push({
|
|
166
|
+
kind: "unfinished-snapshot",
|
|
167
|
+
experimentId: snapshot.experimentId,
|
|
168
|
+
startedAt: snapshot.startedAt,
|
|
169
|
+
dir: snapshot.dir,
|
|
170
|
+
message: `snapshot "${snapshot.experimentId}" (${snapshot.startedAt}) is unfinished (the process was interrupted); completed attempts are read as-is, but the set may be incomplete`,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return makeSelection(snapshots, warnings);
|
|
176
|
+
}
|
|
177
|
+
|
|
73
178
|
/**
|
|
74
179
|
* Selection 构造:filter 只删不换 —— 快照删减,warnings 修剪规则是
|
|
75
180
|
* 「experimentId 不在幸存快照中的丢弃,非实验作用域的保留」(为将来非 per-experiment 的 kind 留位置)。
|
|
@@ -92,8 +197,8 @@ export function makeSelection(snapshots: Snapshot[], warnings: SelectionWarning[
|
|
|
92
197
|
|
|
93
198
|
/**
|
|
94
199
|
* 跨快照聚合前的身份键去重:(experimentId, evalId, attempt, startedAt)。
|
|
95
|
-
* --resume
|
|
96
|
-
*
|
|
200
|
+
* --resume 会把上一轮已通过的结果原样合入新快照,同一 attempt 因此存在于多份落盘;
|
|
201
|
+
* 重复时保留最新快照里的那份(内容相同,取新快照的副本让 ref 落在最新落盘上;
|
|
97
202
|
* 位置取首次出现处,顺序稳定)。startedAt 缺失时宁可不去重也不误删,记入 warnings。
|
|
98
203
|
*/
|
|
99
204
|
export function dedupeAttempts(attempts: AttemptHandle[]): { attempts: AttemptHandle[]; warnings: DedupeWarning[] } {
|
|
@@ -113,26 +218,27 @@ export function dedupeAttempts(attempts: AttemptHandle[]): { attempts: AttemptHa
|
|
|
113
218
|
deduped.push(attempt);
|
|
114
219
|
continue;
|
|
115
220
|
}
|
|
116
|
-
const key = JSON.stringify([
|
|
221
|
+
const key = JSON.stringify([attempt.experimentId, r.id, r.attempt, r.startedAt]);
|
|
117
222
|
const existing = indexByKey.get(key);
|
|
118
223
|
if (existing === undefined) {
|
|
119
224
|
indexByKey.set(key, deduped.length);
|
|
120
225
|
deduped.push(attempt);
|
|
121
|
-
} else if (
|
|
226
|
+
} else if (isNewerSnapshot(attempt.snapshot, deduped[existing].snapshot)) {
|
|
122
227
|
deduped[existing] = attempt;
|
|
123
228
|
}
|
|
124
229
|
}
|
|
125
230
|
return { attempts: deduped, warnings };
|
|
126
231
|
}
|
|
127
232
|
|
|
128
|
-
/**
|
|
129
|
-
export function
|
|
130
|
-
const byStart = a.
|
|
233
|
+
/** 快照新旧比较:startedAt 优先,同刻按快照目录名(时间戳 + 随机后缀,字典序即时序)。 */
|
|
234
|
+
export function isNewerSnapshot(a: Snapshot, b: Snapshot): boolean {
|
|
235
|
+
const byStart = a.startedAt.localeCompare(b.startedAt);
|
|
131
236
|
if (byStart !== 0) return byStart > 0;
|
|
132
237
|
return a.dir.localeCompare(b.dir) > 0;
|
|
133
238
|
}
|
|
134
239
|
|
|
135
|
-
|
|
240
|
+
/** experiment id 分段前缀过滤(--experiment / latest({ experiments }) 同一语义);包内使用,不进公共 barrel。 */
|
|
241
|
+
export function filterExperiments(experiments: Experiment[], filter?: string | string[]): Experiment[] {
|
|
136
242
|
if (filter === undefined) return experiments;
|
|
137
243
|
// 允许 "compare/" 这种带尾斜杠的写法,与 "compare" 等价;分段匹配不误配 "compare2"。
|
|
138
244
|
const prefixes = (Array.isArray(filter) ? filter : [filter]).map((p) => p.replace(/\/+$/, ""));
|
|
Binary file
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Eval 源码的归一化与哈希:discovery 时的捕获(src/runner/eval-source.ts)与证据重建
|
|
2
|
+
// (annotated-source.ts 的 buildAnnotatedEvalSource)必须对同一份源码算出同一个 SHA-256,
|
|
3
|
+
// 所以归一化规则与哈希算法只住这一处 —— 两侧各自 import,不各写一份可能跑偏的实现。
|
|
4
|
+
//
|
|
5
|
+
// 只做归一化 + 哈希,不碰文件系统:住在 results/ 而不是 shared/,是因为 shared/ 的既有约定
|
|
6
|
+
// 是"环境无关、vite 前端直接打包"(见 shared/verdict.ts、shared/aggregate.ts 的注释),
|
|
7
|
+
// node:crypto 在浏览器打包下不成立;results/ 已经是纯 Node 库(writer.ts / open.ts 也用
|
|
8
|
+
// node:fs),多一个 node:crypto 不破坏任何边界。runner 已经单向依赖 results(reporters/
|
|
9
|
+
// artifacts.ts 用 createResultsWriter),所以 src/runner/eval-source.ts 反过来 import 这里
|
|
10
|
+
// 不新增循环依赖。
|
|
11
|
+
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* 归一化 eval 源码文本:去 UTF-8 BOM、把 CRLF / CR 统一成 LF。
|
|
16
|
+
* 幂等——已归一化的文本再跑一次结果不变,所以 capture 时归一化一次、重建时可以放心再跑一次,
|
|
17
|
+
* 两处算出的 SHA-256 恒相同。不做其它改写(不 trim 行尾空白、不折叠空行):
|
|
18
|
+
* 行号是断言 SourceLoc 的锚,任何改变行结构的归一化都会让映射错位。
|
|
19
|
+
*/
|
|
20
|
+
export function normalizeEvalSource(text: string): string {
|
|
21
|
+
const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
22
|
+
return withoutBom.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** 对(已归一化的)源码文本算 SHA-256 十六进制摘要。 */
|
|
26
|
+
export function hashEvalSource(content: string): string {
|
|
27
|
+
return createHash("sha256").update(content, "utf-8").digest("hex");
|
|
28
|
+
}
|
package/src/results/types.ts
CHANGED
|
@@ -1,59 +1,94 @@
|
|
|
1
|
-
// results 域类型:openResults
|
|
1
|
+
// results 域类型:openResults 的分层读取契约与 Selection(定稿见 docs/feature/results/library.md、docs/feature/results/architecture.md)。
|
|
2
2
|
//
|
|
3
|
-
// 结果数据类型(EvalResult /
|
|
3
|
+
// 结果数据类型(EvalResult / ExperimentRunInfo / StreamEvent / …)仍住在各自的域文件里,
|
|
4
4
|
// 这里只 import,不搬家 —— 「类型的家」迁移(facade 反向 re-export)是下一波,不在本次范围。
|
|
5
5
|
//
|
|
6
|
-
// 命名约定:Experiment / Snapshot / Eval
|
|
6
|
+
// 命名约定:Experiment / Snapshot / Eval 是纯数据,不带 Handle 后缀;
|
|
7
7
|
// 唯一叫 AttemptHandle 的是 attempt —— 它的方法真的会碰磁盘,后缀标记的就是这件事。
|
|
8
8
|
|
|
9
|
-
import type { EvalResult,
|
|
9
|
+
import type { EvalResult, ExperimentRunInfo, LocalizedText } from "../types.ts";
|
|
10
10
|
import type { O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
|
|
11
|
-
import type { DiffData, SourceArtifact } from "../types.ts";
|
|
11
|
+
import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
|
|
12
|
+
import type { AttemptLocator } from "./locator.ts";
|
|
12
13
|
|
|
13
|
-
/** attempt
|
|
14
|
-
export const ARTIFACT_KINDS = ["events", "trace", "o11y", "diff", "sources"] as const;
|
|
14
|
+
/** attempt 级 artifact 的种类;文件名见 format.ts 的 artifactFileOf,布局见 docs/feature/results/architecture.md。 */
|
|
15
|
+
export const ARTIFACT_KINDS = ["events", "trace", "o11y", "agentSetup", "diff", "sources"] as const;
|
|
15
16
|
export type ArtifactKind = (typeof ARTIFACT_KINDS)[number];
|
|
16
17
|
|
|
18
|
+
/** 写这份结果的工具:niceeval 自己,或经 niceeval/results 写入面转换的第三方 harness。 */
|
|
19
|
+
export interface Producer {
|
|
20
|
+
name: string;
|
|
21
|
+
version?: string;
|
|
22
|
+
commit?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
17
25
|
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
26
|
+
* `snapshot.json` 的持久化契约:快照元数据 —— 身份、快照级字段与版本元数据,
|
|
27
|
+
* 不含任何逐 attempt 数据。快照开始时写入;收尾时补写 `completedAt`。
|
|
28
|
+
* 字段规则与版本判定见 docs/feature/results/architecture.md「snapshot.json」「版本与升级设计」。
|
|
21
29
|
*/
|
|
22
|
-
export interface
|
|
23
|
-
|
|
24
|
-
|
|
30
|
+
export interface SnapshotMeta {
|
|
31
|
+
/** 恒为 "niceeval.results";和 schemaVersion、producer 一起构成持久化契约,永不移动或改名。 */
|
|
32
|
+
format: "niceeval.results";
|
|
33
|
+
/** 结果格式版本;与读取器不同即视为不兼容,提示用 producer.version 对应的 niceeval 查看。 */
|
|
34
|
+
schemaVersion: number;
|
|
35
|
+
producer: Producer;
|
|
36
|
+
/** 权威的实验身份;实验目录名是它的清洗投影。 */
|
|
37
|
+
experimentId: string;
|
|
38
|
+
/** 实验运行配置(flags / runs / earlyExit / sandbox / timeoutMs / budget),快照内全部 attempt 共享。 */
|
|
39
|
+
experiment?: ExperimentRunInfo;
|
|
40
|
+
agent: string;
|
|
41
|
+
model?: string;
|
|
42
|
+
startedAt: string;
|
|
43
|
+
/** 收尾时补写;缺失 = 快照未收尾(进程中断),已落盘的 attempt 照常可读。 */
|
|
44
|
+
completedAt?: string;
|
|
45
|
+
/** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
|
|
46
|
+
knownEvalIds?: string[];
|
|
47
|
+
/** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
|
|
48
|
+
name?: LocalizedText;
|
|
25
49
|
}
|
|
26
50
|
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
51
|
+
/**
|
|
52
|
+
* 回到证据的引用:快照目录(根相对)+ 快照内 attempt 目录(快照相对)。
|
|
53
|
+
* 字段名(snapshot / attempt)是 view 深链 `#/attempt/<snapshot>/<attempt>` 的持久化路由契约,
|
|
54
|
+
* 不随句柄改名;`snapshot` 恒为两段(`<实验目录>/<快照目录>`),`attempt` 是 `<evalId 路径>/a<n>`,
|
|
55
|
+
* 路由按「前两段 = 快照」解析。Reports 的 MetricCell.refs 用的是同一个身份。
|
|
56
|
+
*/
|
|
57
|
+
export interface AttemptRef {
|
|
58
|
+
/** 根相对快照目录:`<experiment-dir>/<快照目录>`。 */
|
|
59
|
+
snapshot: string;
|
|
60
|
+
/** 快照相对 attempt 目录:`<evalId 路径>/a<n>`。 */
|
|
61
|
+
attempt: string;
|
|
35
62
|
}
|
|
36
63
|
|
|
37
64
|
/**
|
|
38
|
-
* 单个 eval attempt:瘦身条目 +
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* (--resume
|
|
65
|
+
* 单个 eval attempt:瘦身条目 + 重 artifact 的懒加载方法。
|
|
66
|
+
* 懒加载即存在性判断: artifact 缺失返回 null,不抛错;同一 handle 内读过一次即记忆化。
|
|
67
|
+
* artifact 定位按候选顺序回退:先本 attempt 目录,再 artifactBase 指向的原快照 attempt 目录
|
|
68
|
+
* (--resume 合入条目的 artifact 留在原快照里);原快照被清理后如实返回 null。
|
|
42
69
|
*/
|
|
43
70
|
export interface AttemptHandle {
|
|
44
71
|
/** 属于哪道题 —— 直达字段,不绕 result。 */
|
|
45
72
|
evalId: string;
|
|
46
|
-
/**
|
|
73
|
+
/** 属于哪个实验。 */
|
|
47
74
|
experimentId: string;
|
|
48
|
-
/** EvalResult
|
|
75
|
+
/** EvalResult 瘦身条目:判定、断言、用量、成本(快照级字段已拼合)。 */
|
|
49
76
|
result: EvalResult;
|
|
50
|
-
/** 证据引用,指条目所在的落盘(
|
|
77
|
+
/** 证据引用,指条目所在的落盘(合入后的新快照); artifact 经候选回退仍可达。 */
|
|
51
78
|
ref: AttemptRef;
|
|
52
|
-
/**
|
|
53
|
-
|
|
79
|
+
/** 所属快照(反向引用);去重「保留最新快照里的那份」靠它比较新旧。 */
|
|
80
|
+
snapshot: Snapshot;
|
|
81
|
+
/**
|
|
82
|
+
* 不透明的 Attempt 定位符(见 `locator.ts`);openResults() 产出的 handle 恒会填这个字段
|
|
83
|
+
* (result.json 里有就直接用,没有则按当前身份兜底算,见 open.ts 的回填逻辑)。可选只是为了
|
|
84
|
+
* 不强制手工构造的 AttemptHandle(测试里的内存 fake)也必须带上——真实读取路径永远有值。
|
|
85
|
+
*/
|
|
86
|
+
locator?: AttemptLocator;
|
|
54
87
|
events(): Promise<StreamEvent[] | null>;
|
|
55
88
|
trace(): Promise<TraceSpan[] | null>;
|
|
56
89
|
o11y(): Promise<O11ySummary | null>;
|
|
90
|
+
/** 这次 attempt 的 agent setup 装了什么(Skill / native plugin / MCP / Python plugin);没装扩展的 agent 恒为 null。 */
|
|
91
|
+
agentSetup(): Promise<AgentSetupManifest | null>;
|
|
57
92
|
diff(): Promise<DiffData | null>;
|
|
58
93
|
sources(): Promise<SourceArtifact[] | null>;
|
|
59
94
|
}
|
|
@@ -64,29 +99,30 @@ export interface Eval {
|
|
|
64
99
|
attempts: AttemptHandle[];
|
|
65
100
|
}
|
|
66
101
|
|
|
67
|
-
/**
|
|
68
|
-
* 快照 = 单次跑的实验(experiment × run 切片)。一个 run 目录可以装多个 experiment
|
|
69
|
-
* (`niceeval exp compare`),所以「每个 experiment 最新一次」只能用快照粒度表达。
|
|
70
|
-
*/
|
|
102
|
+
/** 快照 = 单次跑的实验,物理上就是一个快照目录,没有更低一层。 */
|
|
71
103
|
export interface Snapshot {
|
|
72
|
-
/**
|
|
104
|
+
/** 权威身份(snapshot.json 字段;实验目录名只是它的清洗投影)。 */
|
|
73
105
|
experimentId: string;
|
|
74
106
|
startedAt: string;
|
|
75
|
-
/**
|
|
107
|
+
/** 缺失 = 未收尾(进程中断);已落盘 attempt 照常在下面读到。 */
|
|
108
|
+
completedAt?: string;
|
|
109
|
+
/** 本快照自己的 agent。 */
|
|
76
110
|
agent: string;
|
|
77
111
|
model?: string;
|
|
78
|
-
/**
|
|
79
|
-
|
|
80
|
-
/**
|
|
112
|
+
/** 实验运行配置(flags / runs / budget …),快照内全部 attempt 共享。 */
|
|
113
|
+
experiment?: ExperimentRunInfo;
|
|
114
|
+
/** 谁写的这份结果(niceeval 或第三方 harness)。 */
|
|
115
|
+
producer: Producer;
|
|
116
|
+
/** 结果格式版本(能读进来的恒为当前版本;不兼容的在 skipped)。 */
|
|
81
117
|
schemaVersion: number;
|
|
118
|
+
/** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
|
|
119
|
+
name?: LocalizedText;
|
|
82
120
|
/** 每道题一项:{ id, attempts };残缺检测 / 逐题遍历从这里走。 */
|
|
83
121
|
evals: Eval[];
|
|
84
122
|
/** 全部 attempt 平铺(= evals 逐题展开),不关心题目边界的聚合消费用。 */
|
|
85
123
|
attempts: AttemptHandle[];
|
|
86
|
-
/**
|
|
87
|
-
|
|
88
|
-
/** experimentId 是合成键(结果里没有 experimentId)时为 true。 */
|
|
89
|
-
synthetic?: boolean;
|
|
124
|
+
/** 快照目录的绝对路径(物理落盘就是快照本身,没有更低一层)。 */
|
|
125
|
+
dir: string;
|
|
90
126
|
/** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
|
|
91
127
|
knownEvalIds?: string[];
|
|
92
128
|
}
|
|
@@ -103,31 +139,30 @@ export interface Experiment {
|
|
|
103
139
|
}
|
|
104
140
|
|
|
105
141
|
/** 目录扫描里读不了、但必须让调用方知道的落盘;无关 JSON 不记(静默忽略)。 */
|
|
106
|
-
export interface
|
|
107
|
-
/**
|
|
142
|
+
export interface SkippedDir {
|
|
143
|
+
/** 落盘目录的绝对路径。 */
|
|
108
144
|
dir: string;
|
|
109
145
|
/**
|
|
110
146
|
* incompatible-version:schemaVersion 与读取器不同(不解析、不迁移、不降级);
|
|
111
|
-
* malformed:summary.json
|
|
112
|
-
* incomplete:有 attempt
|
|
113
|
-
*
|
|
147
|
+
* malformed:snapshot.json(或历史版本的 summary.json)是坏数据;
|
|
148
|
+
* incomplete:有 attempt 落盘、没有 snapshot.json —— 快照目录建好但元数据没写完的极小窗口,
|
|
149
|
+
* 或人为删文件。snapshot.json 是收尾事实,reader 不读无 snapshot.json 的目录;
|
|
150
|
+
* 已完成的 attempt 留在盘上供手工排查。
|
|
114
151
|
*/
|
|
115
152
|
reason: "incompatible-version" | "malformed" | "incomplete";
|
|
116
|
-
/** 那份结果声明的 schemaVersion(incomplete 没有
|
|
153
|
+
/** 那份结果声明的 schemaVersion(incomplete 没有 snapshot.json,自然缺失)。 */
|
|
117
154
|
schemaVersion?: number;
|
|
118
155
|
/** 完整的 producer(name + version):只有 name === "niceeval" 才能拼 npx 提示,第三方如实报名字。 */
|
|
119
|
-
producer?:
|
|
156
|
+
producer?: Producer;
|
|
120
157
|
/** malformed:一句英文诊断。 */
|
|
121
158
|
detail?: string;
|
|
122
159
|
}
|
|
123
160
|
|
|
124
|
-
/** openResults 的返回:experiments 分层;skipped
|
|
161
|
+
/** openResults 的返回:experiments 分层;skipped 不静默丢。 */
|
|
125
162
|
export interface Results {
|
|
126
163
|
/** 每个实验一项,挂着自己的全部历史(id 字典序)。 */
|
|
127
164
|
experiments: Experiment[];
|
|
128
|
-
skipped:
|
|
129
|
-
/** 低层忠实磁盘面:物理落盘目录,新→旧;多数消费方不碰。 */
|
|
130
|
-
runDirs: RunDir[];
|
|
165
|
+
skipped: SkippedDir[];
|
|
131
166
|
/**
|
|
132
167
|
* 每个实验取最新一次快照,返回 Selection(快照与挑选警告绑在一起走)。
|
|
133
168
|
* `experiments` 是 experiment id 前缀过滤(string | string[]),同 CLI 位置参数语义。
|
|
@@ -136,7 +171,7 @@ export interface Results {
|
|
|
136
171
|
}
|
|
137
172
|
|
|
138
173
|
/**
|
|
139
|
-
*
|
|
174
|
+
* Selection:选出的快照 + 挑选过程算出的警告。渲染与否在消费方,但缺口永远被算出来。
|
|
140
175
|
* 下游(Reports 计算函数、copySnapshots)收 `Selection | Snapshot[]`;
|
|
141
176
|
* 手工挑的裸数组没有挑选过程,自然没有 warnings 可带,也如实。
|
|
142
177
|
*/
|
|
@@ -152,8 +187,8 @@ export interface Selection {
|
|
|
152
187
|
}
|
|
153
188
|
|
|
154
189
|
/**
|
|
155
|
-
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message
|
|
156
|
-
* kind
|
|
190
|
+
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message;
|
|
191
|
+
* kind 是契约的一部分,全集与触发条件见 docs/feature/results/library.md「警告 kind 全集」。
|
|
157
192
|
*/
|
|
158
193
|
export type SelectionWarning =
|
|
159
194
|
| {
|
|
@@ -165,7 +200,7 @@ export type SelectionWarning =
|
|
|
165
200
|
message: string;
|
|
166
201
|
}
|
|
167
202
|
| {
|
|
168
|
-
/**
|
|
203
|
+
/** 该实验选中的快照早于 Selection 中最新的落盘;无阈值,如实触发,要阈值消费方按字段自比。 */
|
|
169
204
|
kind: "stale-snapshot";
|
|
170
205
|
experimentId: string;
|
|
171
206
|
startedAt: string;
|
|
@@ -173,11 +208,12 @@ export type SelectionWarning =
|
|
|
173
208
|
message: string;
|
|
174
209
|
}
|
|
175
210
|
| {
|
|
176
|
-
/**
|
|
177
|
-
kind: "
|
|
211
|
+
/** 选中快照缺 completedAt(进程中断,未收尾);已落盘 attempt 照常读出,警告提示集合可能不完整。 */
|
|
212
|
+
kind: "unfinished-snapshot";
|
|
178
213
|
experimentId: string;
|
|
179
|
-
|
|
180
|
-
|
|
214
|
+
startedAt: string;
|
|
215
|
+
/** 该快照目录的绝对路径。 */
|
|
216
|
+
dir: string;
|
|
181
217
|
message: string;
|
|
182
218
|
};
|
|
183
219
|
|