niceeval 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +5 -5
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +11 -11
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +12 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +54 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +34 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/view/data.ts
CHANGED
|
@@ -1,30 +1,25 @@
|
|
|
1
|
-
// view 的数据层:读取经 niceeval/results 的 openResults(布局/版本知识只住在那)
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
// 见 docs/view.md「用 Reports 积木重建 view」迁移顺序 1–2。
|
|
1
|
+
// view 的数据层:读取经 niceeval/results 的 openResults(布局/版本知识只住在那)。
|
|
2
|
+
// 这里只做编排:报告槽 Selection 恒经 selectCurrentResults(现刻水位;与 show 调同一个函数,
|
|
3
|
+
// 两扇门判定不分叉)、快照明细注入(locator / artifactBase)、skipped 透传、报告槽渲染
|
|
4
|
+
// (renderReportSlot:裸跑填充 CostPassRateComparison,--report 整槽替换,en / zh-CN 双语各渲染一遍)。
|
|
5
|
+
// --report 只换报告定义,注入的 Selection 与裸跑同一份。统计口径整体住在报告槽里
|
|
6
|
+
// (报告组件的官方计算函数),viewData 不再携带 overview / 榜单这类统计产物,
|
|
7
|
+
// 见 docs/feature/reports/view.md「打开与收窄」。
|
|
9
8
|
|
|
10
9
|
import { statSync } from "node:fs";
|
|
11
10
|
import { basename, dirname, join, relative, resolve } from "node:path";
|
|
12
11
|
import { dedupeAttempts, openResults } from "../results/index.ts";
|
|
13
|
-
import type { AttemptHandle, Results,
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
import { loadReportFile } from "
|
|
17
|
-
import {
|
|
18
|
-
import type { EvalResult
|
|
19
|
-
import type { SkippedRunNotice, ViewData, ViewEvalResult, ViewSnapshot } from "./shared/types.ts";
|
|
12
|
+
import type { AttemptHandle, Results, Selection, Snapshot, SkippedDir } from "../results/index.ts";
|
|
13
|
+
// dist-sourced: ReportDefinition/ReportLoadError identity must match the built-ins/web.ts
|
|
14
|
+
// module instance renderReportSlot() dynamically imports below (see its comment).
|
|
15
|
+
import { loadReportFile } from "../../dist/report/load.js";
|
|
16
|
+
import { selectCurrentResults, filterExperiments } from "../results/select.ts";
|
|
17
|
+
import type { EvalResult } from "../types.ts";
|
|
18
|
+
import type { ReportSlotHtml, SkippedRunNotice, ViewData, ViewEvalResult, ViewSnapshot } from "./shared/types.ts";
|
|
20
19
|
import { t } from "../i18n/index.ts";
|
|
21
20
|
import { RESULTS_SCHEMA_VERSION } from "../types.ts";
|
|
22
21
|
|
|
23
|
-
|
|
24
|
-
const BOARD_COLUMNS = [passRate, durationMs, tokens, costUSD] as const;
|
|
25
|
-
|
|
26
|
-
/** hero 整体通过率的常量维度:全部选中 attempt 一组,走同一台两级聚合引擎。 */
|
|
27
|
-
const OVERALL_DIMENSION = { name: "overall", of: () => "all" } as const;
|
|
22
|
+
export type { ReportSlotHtml } from "./shared/types.ts";
|
|
28
23
|
|
|
29
24
|
export interface ViewScan {
|
|
30
25
|
viewData: ViewData;
|
|
@@ -34,18 +29,26 @@ export interface ViewScan {
|
|
|
34
29
|
*/
|
|
35
30
|
artifactDirs: Map<string, string>;
|
|
36
31
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
32
|
+
* artifactBase → AttemptHandle。sources.json 在盘上是去重后的引用(`{path, sha256}[]`),
|
|
33
|
+
* 不能像其它 artifact 那样直接 copyFile / piping 原字节——必须经 `AttemptHandle.sources()`
|
|
34
|
+
* 解引用出完整内容(`{path, content}[]`)才能给浏览器用。这份索引专为那一种 artifact 的特判
|
|
35
|
+
* 准备(`copyFetchedArtifacts`),events.json / trace.json 仍走 `artifactDirs` 的原文件路径。
|
|
36
|
+
*/
|
|
37
|
+
attemptsByBase: Map<string, AttemptHandle>;
|
|
38
|
+
/**
|
|
39
|
+
* 报告槽:报告树经 renderReportToStaticHtml 渲染出的静态 HTML(en / zh-CN 各一份),
|
|
40
|
+
* 裸跑填充内置默认报告 CostPassRateComparison,--report 整槽替换。作为两个 <template> 静态块
|
|
41
|
+
* 烘进页面(与 __NICEEVAL_VIEW_DATA__ 相邻),不进 viewData —— 前端只负责把当前
|
|
42
|
+
* 界面语言对应的那块 HTML 摆进报告槽位置,不解析。
|
|
40
43
|
*/
|
|
41
|
-
reportHtml
|
|
44
|
+
reportHtml: ReportSlotHtml;
|
|
42
45
|
}
|
|
43
46
|
|
|
44
|
-
/** view 宿主输入的组合语义(与 show 对齐,docs/reports.md
|
|
47
|
+
/** view 宿主输入的组合语义(与 show 对齐,docs/feature/reports/architecture.md「Selection 是计算入口」)。 */
|
|
45
48
|
export interface ViewScanOptions {
|
|
46
|
-
/** eval id 前缀(位置参数)
|
|
49
|
+
/** eval id 前缀(位置参数):收窄报告槽 Selection;证据室(快照明细)不收窄,深链恒可达。 */
|
|
47
50
|
patterns?: string[];
|
|
48
|
-
/** experiment id 前缀(--experiment)
|
|
51
|
+
/** experiment id 前缀(--experiment):Selection 只留该实验。 */
|
|
49
52
|
experiment?: string;
|
|
50
53
|
/** --report 报告文件:相对 cwd 的路径。装载失败抛 ReportLoadError(CLI 打印后退出)。 */
|
|
51
54
|
report?: { path: string; cwd: string };
|
|
@@ -54,12 +57,12 @@ export interface ViewScanOptions {
|
|
|
54
57
|
/** 可预期的用户输入错误:CLI 打一句英文直说问题与下一步,退出码 1,不抛堆栈。 */
|
|
55
58
|
export class ViewInputError extends Error {}
|
|
56
59
|
|
|
57
|
-
/**
|
|
60
|
+
/** 版本不同、按设计直接不兼容的落盘;只占位提示,不解析内容。 */
|
|
58
61
|
export interface IncompatibleRun {
|
|
59
|
-
/**
|
|
62
|
+
/** 快照目录,相对 cwd;直接可拼进 npx 命令。 */
|
|
60
63
|
dir: string;
|
|
61
64
|
schemaVersion: number;
|
|
62
|
-
producer?:
|
|
65
|
+
producer?: Snapshot["producer"];
|
|
63
66
|
}
|
|
64
67
|
|
|
65
68
|
/** 用能读这份报告的 niceeval 版本查看的命令;第三方 harness 的落盘拼不出 npx,返回 undefined。 */
|
|
@@ -72,7 +75,7 @@ export function incompatibleViewCommand(run: IncompatibleRun): string | undefine
|
|
|
72
75
|
export function incompatibleHint(run: IncompatibleRun): string {
|
|
73
76
|
const command = incompatibleViewCommand(run);
|
|
74
77
|
if (command === undefined) {
|
|
75
|
-
// 第三方 harness:如实报名字和版本,不拼 npx(docs/results
|
|
78
|
+
// 第三方 harness:如实报名字和版本,不拼 npx(docs/feature/results/library.md 的裁决)。
|
|
76
79
|
return t("cli.view.incompatibleForeign", {
|
|
77
80
|
dir: run.dir,
|
|
78
81
|
name: run.producer?.name ?? "?",
|
|
@@ -90,7 +93,7 @@ export function incompatibleHint(run: IncompatibleRun): string {
|
|
|
90
93
|
});
|
|
91
94
|
}
|
|
92
95
|
|
|
93
|
-
/** 单文件模式读到版本不同的
|
|
96
|
+
/** 单文件模式读到版本不同的 snapshot 时抛出;CLI 捕获后打印提示退出,不当成普通错误堆栈。 */
|
|
94
97
|
export class IncompatibleResultsError extends Error {
|
|
95
98
|
constructor(readonly run: IncompatibleRun) {
|
|
96
99
|
super(incompatibleHint(run));
|
|
@@ -98,48 +101,61 @@ export class IncompatibleResultsError extends Error {
|
|
|
98
101
|
}
|
|
99
102
|
}
|
|
100
103
|
|
|
101
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* 服务/解析 artifact 的根目录:输入是目录就用它,是文件就上跳两层(snapshot.json 恒在
|
|
106
|
+
* `<root>/<experiment-dir>/<snapshot-dir>/snapshot.json`,root 是 AttemptRef.snapshot
|
|
107
|
+
* 两段路径的锚点)。
|
|
108
|
+
*/
|
|
102
109
|
export function viewRoot(input?: string): string {
|
|
103
110
|
const target = resolve(input ?? ".niceeval");
|
|
104
111
|
try {
|
|
105
|
-
return statSync(target).isFile() ? dirname(target) : target;
|
|
112
|
+
return statSync(target).isFile() ? dirname(dirname(target)) : target;
|
|
106
113
|
} catch {
|
|
107
114
|
return target;
|
|
108
115
|
}
|
|
109
116
|
}
|
|
110
117
|
|
|
118
|
+
/** 携带条目 / 新算的 artifactBase:`result.artifactBase ?? `${ref.snapshot}/${ref.attempt}``(posix 拼接)。 */
|
|
119
|
+
function withArtifactBase(attempt: AttemptHandle): EvalResult {
|
|
120
|
+
const r = attempt.result;
|
|
121
|
+
if (r.artifactBase !== undefined) return r;
|
|
122
|
+
return { ...r, artifactBase: `${attempt.ref.snapshot}/${attempt.ref.attempt}` };
|
|
123
|
+
}
|
|
124
|
+
|
|
111
125
|
/**
|
|
112
126
|
* 读跨历史「每 (experimentId, evalId) 最新一份」的 EvalResult,供续跑携带已通过结果。
|
|
113
|
-
*
|
|
127
|
+
* 只看最近一个快照不行:部分补跑(位置参数只跑几道题)会把携带基线换成那个部分快照,
|
|
114
128
|
* 之后重跑任何实验都携带不到东西,`exp <组>` 的「补齐缺失」语义随之失效。
|
|
115
|
-
* 同一 (experimentId, evalId) 的多个 attempt
|
|
116
|
-
* 携带条目要能被 view
|
|
129
|
+
* 同一 (experimentId, evalId) 的多个 attempt 整批取自含它的最新快照,不跨快照混装。
|
|
130
|
+
* 携带条目要能被 view 找回 artifact,这里同时把 artifactBase(相对结果根)拼好(runner 依赖它)。
|
|
117
131
|
*/
|
|
118
132
|
export async function loadLatestResultsPerEval(root = ".niceeval"): Promise<EvalResult[]> {
|
|
119
133
|
const results = await openResults(root);
|
|
120
|
-
const rootAbs = resolve(root);
|
|
121
|
-
const claimed = new Set<string>();
|
|
122
134
|
const out: EvalResult[] = [];
|
|
123
|
-
for (const
|
|
124
|
-
//
|
|
125
|
-
// 保证同 (experiment, eval) 的多 attempt
|
|
126
|
-
const
|
|
127
|
-
for (const
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
135
|
+
for (const exp of results.experiments) {
|
|
136
|
+
// exp.snapshots 已按新→旧排序;同一快照内先收本轮的 eval id,收完再整体入 claimed,
|
|
137
|
+
// 保证同 (experiment, eval) 的多 attempt 整批取自同一个快照。
|
|
138
|
+
const claimed = new Set<string>();
|
|
139
|
+
for (const snapshot of exp.snapshots) {
|
|
140
|
+
const takenThisSnapshot = new Set<string>();
|
|
141
|
+
for (const ev of snapshot.evals) {
|
|
142
|
+
if (claimed.has(ev.id)) continue;
|
|
143
|
+
takenThisSnapshot.add(ev.id);
|
|
144
|
+
for (const attempt of ev.attempts) out.push(withArtifactBase(attempt));
|
|
145
|
+
}
|
|
146
|
+
for (const id of takenThisSnapshot) claimed.add(id);
|
|
132
147
|
}
|
|
133
|
-
for (const key of takenThisRun) claimed.add(key);
|
|
134
148
|
}
|
|
135
149
|
return out;
|
|
136
150
|
}
|
|
137
151
|
|
|
138
152
|
/**
|
|
139
153
|
* `niceeval view` 的数据装载入口:server 每次请求现读现算,`--out` 导出用同一份。
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
154
|
+
* 报告槽 Selection 恒经 selectCurrentResults 合成(现刻水位;与 `niceeval show` 调同一个
|
|
155
|
+
* 函数,裸跑与局部收窄不分叉),位置前缀 / --experiment 只作为 scope 传入,不切换选择口径。
|
|
156
|
+
* --report 本身不改挑选——它只换报告槽的填充,注入的 Selection 与裸跑同一份,
|
|
157
|
+
* 「裸跑 ≡ --report <CostPassRateComparison>」靠这条成立(docs/feature/reports/architecture.md「Selection 是计算入口」)。
|
|
158
|
+
* 证据室数据(快照明细 / skipped)恒为全量,深链在任何收窄下都可达。
|
|
143
159
|
* 零可读结果一律抛 ViewInputError,不渲染/导出空页面(server 起不来,--out 非零退出)。
|
|
144
160
|
*/
|
|
145
161
|
export async function loadViewScan(input?: string, opts: ViewScanOptions = {}): Promise<ViewScan> {
|
|
@@ -149,7 +165,6 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
149
165
|
assertSingleFileReadable(results, target);
|
|
150
166
|
|
|
151
167
|
const patterns = opts.patterns ?? [];
|
|
152
|
-
const narrowed = patterns.length > 0 || opts.experiment !== undefined || opts.report !== undefined;
|
|
153
168
|
|
|
154
169
|
// 零可读结果直说,不渲染/导出一张空页面(与 show 的「匹配不到直说」同一原则;
|
|
155
170
|
// CI 静态发布还靠这个非零退出保住上一次部署,空报告不顶上线)。零可读最常见的
|
|
@@ -170,10 +185,11 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
170
185
|
);
|
|
171
186
|
}
|
|
172
187
|
|
|
173
|
-
|
|
174
|
-
const selection =
|
|
175
|
-
|
|
176
|
-
|
|
188
|
+
// 报告槽 Selection:恒经现刻水位选择器合成,与 show 裸跑同口径(两扇门判定不分叉)。
|
|
189
|
+
const selection = selectCurrentResults(results, { experiment: opts.experiment, patterns });
|
|
190
|
+
// latestPerExperiment 只服务证据室 UI 的 latest 标记(ViewSnapshot.latest / viewData.snapshots),
|
|
191
|
+
// 与报告槽 Selection 完全无关,绝不复用为报告 Selection。
|
|
192
|
+
const latestPerExperiment = results.latest();
|
|
177
193
|
|
|
178
194
|
if (patterns.length > 0 && selection.snapshots.every((s) => s.evals.length === 0)) {
|
|
179
195
|
const known = [
|
|
@@ -184,23 +200,17 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
184
200
|
);
|
|
185
201
|
}
|
|
186
202
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
await tableData(selection, { rows: OVERALL_DIMENSION, columns: [passRate] }),
|
|
191
|
-
];
|
|
203
|
+
// 报告槽:裸跑填充 CostPassRateComparison,--report 整槽替换。报告吃同一份注入 Selection,
|
|
204
|
+
// web 面在计算侧静态渲染成 HTML(en / zh-CN 各一遍,切界面语言不重算数据)。
|
|
205
|
+
const reportHtml = await renderReportSlot(opts.report, results, selection);
|
|
192
206
|
|
|
193
|
-
//
|
|
194
|
-
const reportHtml = opts.report
|
|
195
|
-
? await renderReportSlot(opts.report, results, selection)
|
|
196
|
-
: undefined;
|
|
197
|
-
|
|
198
|
-
// 跨快照按身份键去重:--resume 携带的条目在多份落盘里重复,只保留最新 run 里的那份
|
|
207
|
+
// 跨快照按身份键去重:--resume 携带的条目在多份落盘里重复,只保留最新快照里的那份
|
|
199
208
|
// (与官方计算函数的聚合口径一致,Runs / Traces 的计数因此不被复印件灌票)。
|
|
200
209
|
const artifactDirs = new Map<string, string>();
|
|
210
|
+
const attemptsByBase = new Map<string, AttemptHandle>();
|
|
201
211
|
// latest 标记恒按 results.latest() 口径打(ViewSnapshot.latest 的声明语义),
|
|
202
|
-
//
|
|
203
|
-
const latestSet = new Set(
|
|
212
|
+
// 与报告槽 Selection(现刻水位,可能合成自更早快照)是两个独立概念,不混用。
|
|
213
|
+
const latestSet = new Set(latestPerExperiment.snapshots);
|
|
204
214
|
const allAttempts: AttemptHandle[] = [];
|
|
205
215
|
for (const exp of results.experiments) {
|
|
206
216
|
for (const snap of exp.snapshots) allAttempts.push(...snap.attempts);
|
|
@@ -216,59 +226,100 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
216
226
|
if (kept.length === 0 && !latest) continue;
|
|
217
227
|
snapshots.push({
|
|
218
228
|
experimentId: snap.experimentId,
|
|
219
|
-
...(snap.synthetic ? { synthetic: true } : {}),
|
|
220
229
|
agent: snap.agent,
|
|
221
230
|
...(snap.model !== undefined ? { model: snap.model } : {}),
|
|
222
231
|
startedAt: snap.startedAt,
|
|
223
|
-
|
|
232
|
+
// 与 reader 的 AttemptRef.snapshot 同一公式(祖父目录名/自身目录名),不依赖 root——
|
|
233
|
+
// 单文件模式的 root 是从目标文件上跳算出的,两条计算各自独立更不容易踩偏差。
|
|
234
|
+
run: `${basename(dirname(snap.dir))}/${basename(snap.dir)}`,
|
|
224
235
|
latest,
|
|
225
236
|
results: kept.map((a) => {
|
|
226
|
-
const { annotated, base, abs } = annotateResult(a,
|
|
227
|
-
|
|
237
|
+
const { annotated, base, abs } = annotateResult(a, root);
|
|
238
|
+
artifactDirs.set(base, abs);
|
|
239
|
+
attemptsByBase.set(base, a);
|
|
228
240
|
return annotated;
|
|
229
241
|
}),
|
|
230
242
|
});
|
|
231
243
|
}
|
|
232
244
|
}
|
|
233
245
|
|
|
234
|
-
|
|
246
|
+
// 全局最新快照(跨全部实验):hero 的项目名 / 最近一次运行时刻从这里取。
|
|
247
|
+
let latestSnapshot: Snapshot | undefined;
|
|
248
|
+
for (const exp of results.experiments) {
|
|
249
|
+
const candidate = exp.snapshots[0];
|
|
250
|
+
if (!candidate) continue;
|
|
251
|
+
if (!latestSnapshot || candidate.startedAt > latestSnapshot.startedAt) latestSnapshot = candidate;
|
|
252
|
+
}
|
|
253
|
+
|
|
235
254
|
const viewData: ViewData = {
|
|
236
|
-
...(
|
|
237
|
-
...(
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
: new Set(selection.snapshots.map((s) => s.runDir.dir)).size,
|
|
243
|
-
overview,
|
|
244
|
-
table,
|
|
245
|
-
overall,
|
|
255
|
+
...(latestSnapshot?.name !== undefined ? { name: latestSnapshot.name } : {}),
|
|
256
|
+
...(latestSnapshot ? { lastRunAt: latestSnapshot.startedAt } : {}),
|
|
257
|
+
// 合成 Selection 的快照是跨快照拼出来的,来源物理 run 数从 attempt 自己的 snapshot
|
|
258
|
+
// 反向引用取——每个 attempt 的 .snapshot 恒指向它真实所在的贡献快照(无论 Selection
|
|
259
|
+
// 是否合成),所以这条对裸跑与收窄一律成立,不需要分支。
|
|
260
|
+
composedRuns: new Set(selection.snapshots.flatMap((s) => s.attempts.map((a) => a.snapshot.dir))).size,
|
|
246
261
|
snapshots,
|
|
247
262
|
skippedRuns: results.skipped.map(toSkippedNotice),
|
|
248
263
|
};
|
|
249
|
-
return { viewData, artifactDirs,
|
|
264
|
+
return { viewData, artifactDirs, attemptsByBase, reportHtml };
|
|
250
265
|
}
|
|
251
266
|
|
|
252
267
|
/**
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
268
|
+
* base(withArtifactBase 同一公式)→ AttemptHandle 的轻量索引,供 server.ts 的 artifact 路由
|
|
269
|
+
* 按 URL 反查 attempt——只有 sources.json 请求需要这份索引(其余 artifact 直接读盘文件更便宜),
|
|
270
|
+
* 所以特意不跑 loadViewScan 的全套 Selection 合成 + 报告双语渲染,只做一次 openResults() 扫描。
|
|
271
|
+
* 不做去重收窄:遍历全部落盘 attempt,同一 base 只可能来自同一份物理落盘,历史快照里被去重
|
|
272
|
+
* 吸走的重复条目 base 天然不同,不会冲突;不会被 UI 引用的 base 混进来也无害(纯查表,
|
|
273
|
+
* 从不主动枚举 key)。
|
|
274
|
+
*/
|
|
275
|
+
export async function loadAttemptIndex(input?: string): Promise<Map<string, AttemptHandle>> {
|
|
276
|
+
const target = resolve(input ?? ".niceeval");
|
|
277
|
+
const results = await openResults(target);
|
|
278
|
+
const index = new Map<string, AttemptHandle>();
|
|
279
|
+
for (const exp of results.experiments) {
|
|
280
|
+
for (const snap of exp.snapshots) {
|
|
281
|
+
for (const attempt of snap.attempts) {
|
|
282
|
+
index.set(withArtifactBase(attempt).artifactBase!, attempt);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return index;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* 报告槽渲染:装载报告文件(--report;dev server 语义 —— 文件变更下次请求整页重算,
|
|
291
|
+
* 经 mtime cache-busting),缺省用内置默认报告 CostPassRateComparison → 注入 Selection →
|
|
292
|
+
* web 面 renderToStaticMarkup 成静态 HTML,en / zh-CN 各渲染一遍(chrome 文案按 locale)。
|
|
293
|
+
* react / react-dom 动态加载:data.ts 还被 runner 的续跑携带(loadLatestResultsPerEval)
|
|
294
|
+
* 消费,渲染依赖不进那条路径。attemptHref 缺省即 `#/attempt/@<locator>` 深链路由(web.ts 的默认值),
|
|
295
|
+
* 单段、不透明——证据室(`src/view/app/`)的 attempt 路由消费同一个格式(attempt-route.ts),
|
|
296
|
+
* 报告槽深链与证据室深链是同一条路由的两个来源。
|
|
257
297
|
*/
|
|
258
298
|
async function renderReportSlot(
|
|
259
|
-
report: { path: string; cwd: string },
|
|
299
|
+
report: { path: string; cwd: string } | undefined,
|
|
260
300
|
results: Results,
|
|
261
301
|
selection: Selection,
|
|
262
|
-
): Promise<
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
302
|
+
): Promise<ReportSlotHtml> {
|
|
303
|
+
// built-ins/index.ts 和 web.ts 都碰 JSX(.tsx / import react-dom);package-owned 的报告
|
|
304
|
+
// runtime 走预编译产物(dist/report/**,`pnpm run build:report` 产出),不受 view 消费方
|
|
305
|
+
// cwd/tsconfig 影响 —— 见 tsconfig.report-build.json。两个动态 import 必须落在同一份编译
|
|
306
|
+
// 产物图里(dist/report/tree.js 的 WebContext 模块级状态不能跨实例),不能一个走 dist
|
|
307
|
+
// 一个走 src。
|
|
308
|
+
const definition = report
|
|
309
|
+
? await loadReportFile(report.cwd, report.path, { freshImport: true })
|
|
310
|
+
: (await import("../../dist/report/built-ins/index.js")).CostPassRateComparison;
|
|
311
|
+
const { renderReportToStaticHtml } = await import("../../dist/report/web.js");
|
|
312
|
+
const ctx = { selection, results };
|
|
313
|
+
return {
|
|
314
|
+
en: await renderReportToStaticHtml(definition, ctx, { locale: "en" }),
|
|
315
|
+
"zh-CN": await renderReportToStaticHtml(definition, ctx, { locale: "zh-CN" }),
|
|
316
|
+
};
|
|
266
317
|
}
|
|
267
318
|
|
|
268
319
|
/**
|
|
269
|
-
* 单文件模式(`niceeval view path/to/
|
|
320
|
+
* 单文件模式(`niceeval view path/to/snapshot.json`)是用户明确指定的目标:
|
|
270
321
|
* 读不了就让命令失败并给可执行的下一步,不打开一个空页面。目录模式不走这里
|
|
271
|
-
* (读不了的进 skipped
|
|
322
|
+
* (读不了的进 skipped,页面顶部横幅展示,单个坏快照不拖垮整页)。
|
|
272
323
|
*/
|
|
273
324
|
function assertSingleFileReadable(results: Results, target: string): void {
|
|
274
325
|
let isFile = false;
|
|
@@ -277,7 +328,8 @@ function assertSingleFileReadable(results: Results, target: string): void {
|
|
|
277
328
|
} catch {
|
|
278
329
|
return; // 目标不存在:按空结果渲染(还没跑过 eval 不是错误)。
|
|
279
330
|
}
|
|
280
|
-
|
|
331
|
+
const hasSnapshots = results.experiments.some((e) => e.snapshots.length > 0);
|
|
332
|
+
if (!isFile || hasSnapshots) return;
|
|
281
333
|
const skip = results.skipped[0];
|
|
282
334
|
if (skip?.reason === "incompatible-version") {
|
|
283
335
|
throw new IncompatibleResultsError({
|
|
@@ -288,37 +340,31 @@ function assertSingleFileReadable(results: Results, target: string): void {
|
|
|
288
340
|
}
|
|
289
341
|
if (skip?.reason === "malformed") {
|
|
290
342
|
throw new Error(
|
|
291
|
-
`${target}: ${skip.detail ?? "unreadable report"}. The report may be corrupted; re-run the eval or delete this
|
|
343
|
+
`${target}: ${skip.detail ?? "unreadable report"}. The report may be corrupted; re-run the eval or delete this snapshot directory.`,
|
|
292
344
|
);
|
|
293
345
|
}
|
|
294
|
-
throw new Error(`${target} is not a niceeval
|
|
346
|
+
throw new Error(`${target} is not a niceeval snapshot`);
|
|
295
347
|
}
|
|
296
348
|
|
|
297
349
|
/**
|
|
298
350
|
* 给单条 attempt 注入 view 侧标注:
|
|
299
|
-
* -
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
351
|
+
* - locator:不透明的 AttemptLocator(与 Reports 的 MetricCell.refs / `ctx.attemptHref` 同一身份),
|
|
352
|
+
* `#/attempt/@<locator>` 深链路由的参数——证据室按它在 viewData.snapshots 里定位回同一条 attempt。
|
|
353
|
+
* - artifactBase:相对 view 根的 artifact 目录(前端据此 fetch trace.json 等)。本快照跑出的
|
|
354
|
+
* 条目落盘没有这个字段,按 `${ref.snapshot}/${ref.attempt}` 现算;携带条目(--resume 合入)
|
|
355
|
+
* 落盘自带 artifactBase,指向原快照,原样沿用。
|
|
356
|
+
* 返回新对象,不 mutate 读入的结果;宿主机绝对路径只回给调用方写进 artifactDirs
|
|
303
357
|
* (server 端内存),不挂到 result 上,避免随 viewData 进静态 HTML。
|
|
304
358
|
*/
|
|
305
359
|
function annotateResult(
|
|
306
360
|
attempt: AttemptHandle,
|
|
307
|
-
run: RunDir,
|
|
308
361
|
root: string,
|
|
309
|
-
): { annotated: ViewEvalResult; base
|
|
362
|
+
): { annotated: ViewEvalResult; base: string; abs: string } {
|
|
310
363
|
const r = attempt.result;
|
|
311
|
-
const
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
return { annotated: { ...annotated, artifactBase: base }, base, abs };
|
|
316
|
-
}
|
|
317
|
-
if (r.artifactBase) {
|
|
318
|
-
// 已是相对结果根的路径(携带条目),工件留在原 run 目录里。
|
|
319
|
-
return { annotated, base: r.artifactBase, abs: join(root, r.artifactBase) };
|
|
320
|
-
}
|
|
321
|
-
return { annotated };
|
|
364
|
+
const base = r.artifactBase ?? `${attempt.ref.snapshot}/${attempt.ref.attempt}`;
|
|
365
|
+
const abs = join(root, base);
|
|
366
|
+
const annotated: ViewEvalResult = { ...r, locator: attempt.locator, artifactBase: base };
|
|
367
|
+
return { annotated, base, abs };
|
|
322
368
|
}
|
|
323
369
|
|
|
324
370
|
/**
|
|
@@ -326,7 +372,7 @@ function annotateResult(
|
|
|
326
372
|
* 有 skipped 时逐条列目录与原因——schemaVersion 不兼容的 niceeval 落盘给出可跑的
|
|
327
373
|
* npx 命令,让「全被跳过」和「真没跑过」在错误里就能分清,不用进查看器排查。
|
|
328
374
|
*/
|
|
329
|
-
function noReadableResults(target: string, skipped:
|
|
375
|
+
function noReadableResults(target: string, skipped: SkippedDir[]): string {
|
|
330
376
|
if (skipped.length === 0) return t("cli.view.noResults", { root: target }).trimEnd();
|
|
331
377
|
const lines = skipped.map((skip) => {
|
|
332
378
|
const notice = toSkippedNotice(skip);
|
|
@@ -334,7 +380,7 @@ function noReadableResults(target: string, skipped: SkippedRun[]): string {
|
|
|
334
380
|
const hint = notice.command ? ` — view it with \`${notice.command.trim()}\`` : "";
|
|
335
381
|
return ` ${notice.dir}: ${notice.reason}${version}${hint}`;
|
|
336
382
|
});
|
|
337
|
-
const runs = skipped.length === 1 ? "1
|
|
383
|
+
const runs = skipped.length === 1 ? "1 snapshot directory was" : `${skipped.length} snapshot directories were`;
|
|
338
384
|
return [
|
|
339
385
|
`No readable results under ${target} — ${runs} skipped:`,
|
|
340
386
|
...lines,
|
|
@@ -342,7 +388,7 @@ function noReadableResults(target: string, skipped: SkippedRun[]): string {
|
|
|
342
388
|
].join("\n");
|
|
343
389
|
}
|
|
344
390
|
|
|
345
|
-
function toSkippedNotice(skip:
|
|
391
|
+
function toSkippedNotice(skip: SkippedDir): SkippedRunNotice {
|
|
346
392
|
const dir = relative(process.cwd(), skip.dir) || ".";
|
|
347
393
|
const command =
|
|
348
394
|
skip.reason === "incompatible-version" && skip.producer?.name === "niceeval" && skip.producer.version
|
package/src/view/index.ts
CHANGED
|
@@ -26,7 +26,7 @@ export {
|
|
|
26
26
|
* - 恰好一个位置参数且指向存在的文件 → 单文件模式(`niceeval view <summary.json>`),
|
|
27
27
|
* 不与 --run 或其它位置参数混用;
|
|
28
28
|
* - 指向存在的目录 → 报错直说:目录经 `--run <dir>` 递入,位置参数留给 eval id 前缀;
|
|
29
|
-
* - 其余 → eval id
|
|
29
|
+
* - 其余 → eval id 前缀,收窄报告槽 Selection(经 show 同一套 Selection 合成)。
|
|
30
30
|
*/
|
|
31
31
|
export function resolveViewInput(
|
|
32
32
|
cwd: string,
|
|
@@ -74,12 +74,12 @@ export function resolveViewInput(
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
/**
|
|
77
|
-
* 导出静态报告(--out):只有目录式一种形态。写 <dir>/index.html,并把前端会 fetch
|
|
77
|
+
* 导出静态报告(--out):只有目录式一种形态。写 <dir>/index.html,并把前端会 fetch 的 artifact
|
|
78
78
|
* (sources.json / events.json / trace.json)复制到 <dir>/artifact/<base>/——与本地
|
|
79
79
|
* server 的 /artifact/<rel> 路由同一布局,整个目录扔给任何静态托管即是完整体验。
|
|
80
|
-
*
|
|
81
|
-
* 单文件(*.html)导出已移除:代码/transcript/trace
|
|
82
|
-
* 存在本身就在诱导用户导出一份看不了证据的报告(docs/view.md「静态导出」)。
|
|
80
|
+
* 首页即报告槽(裸跑填充 CostPassRateComparison,--report 整槽替换),证据室同站。
|
|
81
|
+
* 单文件(*.html)导出已移除:代码/transcript/trace 视图依赖 artifact 文件,单文件注定残缺,
|
|
82
|
+
* 存在本身就在诱导用户导出一份看不了证据的报告(docs/feature/reports/view.md「静态导出」)。
|
|
83
83
|
*/
|
|
84
84
|
export async function buildView(opts: ViewOptions = {}): Promise<string> {
|
|
85
85
|
const out = resolve(opts.out ?? ".niceeval/site");
|
|
@@ -95,18 +95,30 @@ export async function buildView(opts: ViewOptions = {}): Promise<string> {
|
|
|
95
95
|
return out;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
//
|
|
99
|
-
// 且 diff 可达上百 MB,带进静态导出只会拖垮部署体积。
|
|
100
|
-
|
|
98
|
+
// 只导出前端会 fetch 的三类 artifact。diff.json / o11y.json 是运行侧产物,查看器从不读取,
|
|
99
|
+
// 且 diff 可达上百 MB,带进静态导出只会拖垮部署体积。events.json / trace.json 原字节复制;
|
|
100
|
+
// sources.json 是例外——盘上是去重后的引用(`{path, sha256}[]`),必须先经
|
|
101
|
+
// AttemptHandle.sources() 解引用出完整内容(`{path, content}[]`)再写出,否则浏览器端的
|
|
102
|
+
// isCodeSource 守卫因缺 content 字段判空,代码视图会误判「源码未捕获」(即便源码明明捕获了)。
|
|
103
|
+
const RAW_COPY_ARTIFACTS = ["events.json", "trace.json"];
|
|
101
104
|
|
|
102
105
|
async function copyFetchedArtifacts(scan: ViewScan, artifactRoot: string): Promise<void> {
|
|
103
106
|
for (const [base, srcDir] of scan.artifactDirs) {
|
|
104
107
|
const destDir = join(artifactRoot, base);
|
|
105
108
|
// 输入本身已经是导出布局(比如对着上次导出的目录重新生成 index.html)时不自拷。
|
|
106
109
|
if (resolve(srcDir) === resolve(destDir)) continue;
|
|
107
|
-
|
|
108
|
-
|
|
110
|
+
|
|
111
|
+
const rawFiles = RAW_COPY_ARTIFACTS.filter((name) => existsSync(join(srcDir, name)));
|
|
112
|
+
const hasSourcesRef = existsSync(join(srcDir, "sources.json"));
|
|
113
|
+
if (!rawFiles.length && !hasSourcesRef) continue;
|
|
114
|
+
|
|
109
115
|
await mkdir(destDir, { recursive: true });
|
|
110
|
-
await Promise.all(
|
|
116
|
+
await Promise.all(rawFiles.map((name) => copyFile(join(srcDir, name), join(destDir, name))));
|
|
117
|
+
|
|
118
|
+
if (hasSourcesRef) {
|
|
119
|
+
const attempt = scan.attemptsByBase.get(base);
|
|
120
|
+
const sources = attempt ? await attempt.sources() : null;
|
|
121
|
+
await writeFile(join(destDir, "sources.json"), JSON.stringify(sources ?? []), "utf-8");
|
|
122
|
+
}
|
|
111
123
|
}
|
|
112
124
|
}
|