niceeval 0.12.1-canary.24 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -2
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/en.d.ts +0 -19
- package/dist/i18n/en.js +8 -35
- package/dist/i18n/zh-CN.d.ts +2 -21
- package/dist/i18n/zh-CN.js +6 -33
- package/dist/record/open.js +2 -2
- package/dist/record/types.d.ts +7 -16
- package/dist/report/built-in/result-components.js +22 -8
- package/dist/report/components/entity-lists/compute.d.ts +6 -0
- package/dist/report/components/entity-lists/compute.js +62 -13
- package/dist/report/components/entity-lists/content.d.ts +2 -0
- package/dist/report/components/entity-lists/content.js +60 -26
- package/dist/report/components/entity-lists/index.d.ts +21 -1
- package/dist/report/components/entity-lists/index.js +38 -6
- package/dist/report/components/entity-lists/validate.js +22 -18
- package/dist/report/components/experiment-detail/compute.js +1 -1
- package/dist/report/components/fixtures.js +14 -7
- package/dist/report/components/shared-compute.d.ts +12 -0
- package/dist/report/components/shared-compute.js +22 -2
- package/dist/report/components/site-components/HeroCard.d.ts +2 -2
- package/dist/report/components/site-components/HeroCard.js +28 -4
- package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
- package/dist/report/components/site-components/scope-warnings.js +1 -1
- package/dist/report/components/summaries/compute.js +2 -2
- package/dist/report/components/summaries/index.js +26 -15
- package/dist/report/definition/cell.d.ts +15 -4
- package/dist/report/definition/cell.js +20 -13
- package/dist/report/definition/primitives.js +28 -13
- package/dist/report/index.d.ts +1 -1
- package/dist/report/model/aggregate.d.ts +22 -4
- package/dist/report/model/aggregate.js +35 -1
- package/dist/report/model/calculation.js +1 -5
- package/dist/report/model/evaluation-kind.js +2 -1
- package/dist/report/model/flag.js +2 -0
- package/dist/report/model/format.d.ts +2 -2
- package/dist/report/model/format.js +3 -4
- package/dist/report/model/locale.d.ts +15 -8
- package/dist/report/model/locale.js +25 -13
- package/dist/report/model/metrics.js +6 -9
- package/dist/report/model/types.d.ts +36 -11
- package/dist/report/react/index.d.ts +1 -1
- package/dist/report/slices/compute.js +55 -3
- package/dist/report/slices/content.js +15 -8
- package/dist/report/slices/validate.js +27 -2
- package/dist/report/tasks.d.ts +5 -0
- package/dist/report/tasks.js +3 -2
- package/dist/runner/eval-selection.d.ts +5 -0
- package/dist/runner/fingerprint.d.ts +1 -1
- package/dist/runner/session.d.ts +8 -8
- package/dist/runner/types.d.ts +8 -17
- package/dist/runner/types.js +0 -2
- package/dist/sample/index.d.ts +13 -4
- package/dist/sample/index.js +72 -57
- package/dist/sandbox/compose.d.ts +4 -11
- package/dist/sandbox/docker.d.ts +2 -74
- package/dist/sandbox/dockerfile-build.d.ts +1 -5
- package/dist/sandbox/layer.d.ts +0 -117
- package/dist/sandbox/registry.d.ts +2 -3
- package/docs-site/zh/examples/ai-agent-application.mdx +2 -2
- package/docs-site/zh/examples/index.mdx +0 -7
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +1 -1
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +1 -1
- package/docs-site/zh/examples/integrations/langgraph.mdx +1 -1
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +1 -1
- package/docs-site/zh/explanation/adapter.mdx +3 -3
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +3 -3
- package/docs-site/zh/reference/cli.mdx +3 -3
- package/docs-site/zh/reference/define-agent.mdx +7 -9
- package/docs-site/zh/reference/official-adapters.mdx +1 -1
- package/docs-site/zh/reference/results-data.mdx +4 -7
- package/docs-site/zh/tutorials/agent-onboarding.mdx +2 -2
- package/docs-site/zh/tutorials/authoring.mdx +3 -3
- package/docs-site/zh/tutorials/ci-integration.mdx +1 -1
- package/docs-site/zh/tutorials/connect-otel.mdx +4 -4
- package/docs-site/zh/tutorials/connect-your-agent.mdx +6 -6
- package/docs-site/zh/tutorials/custom-reports.mdx +1 -2
- package/docs-site/zh/tutorials/experiments.mdx +2 -2
- package/docs-site/zh/tutorials/quickstart.mdx +2 -2
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +4 -4
- package/docs-site/zh/tutorials/sandbox-providers.mdx +1 -7
- package/docs-site/zh/tutorials/viewing-results.mdx +3 -5
- package/docs-site/zh/tutorials/write-send.mdx +7 -7
- package/package.json +3 -4
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/index.ts +1 -1
- package/src/agents/openai-compat.test.ts +3 -3
- package/src/agents/types.ts +2 -2
- package/src/agents/ui-message-stream.ts +2 -2
- package/src/cli.ts +43 -312
- package/src/define-agent.test.ts +9 -13
- package/src/define.ts +2 -5
- package/src/i18n/en.ts +8 -44
- package/src/i18n/zh-CN.ts +6 -41
- package/src/loaders/criteria.test.ts +2 -2
- package/src/loaders/index.test.ts +2 -2
- package/src/loaders/private.test.ts +2 -2
- package/src/record/evidence-schema.test.ts +1 -1
- package/src/record/host-equivalence.test.ts +48 -36
- package/src/record/index.ts +0 -1
- package/src/record/open.ts +2 -2
- package/src/record/results.test.ts +6 -6
- package/src/record/types.ts +7 -17
- package/src/record/writer.ts +25 -1
- package/src/report/assets/styles.css +74 -1
- package/src/report/built-in/result-components.tsx +33 -1
- package/src/report/components/compute.test.ts +193 -138
- package/src/report/components/entity-lists/compute.ts +82 -13
- package/src/report/components/entity-lists/content.test.ts +34 -70
- package/src/report/components/entity-lists/content.ts +65 -32
- package/src/report/components/entity-lists/index.tsx +91 -7
- package/src/report/components/entity-lists/validate.test.ts +17 -18
- package/src/report/components/entity-lists/validate.ts +15 -15
- package/src/report/components/experiment-detail/compute.ts +1 -1
- package/src/report/components/experiment-detail/experiment-detail.test.tsx +2 -7
- package/src/report/components/fixtures.ts +14 -7
- package/src/report/components/shared-compute.ts +22 -2
- package/src/report/components/site-components/HeroCard.tsx +32 -5
- package/src/report/components/site-components/scope-warnings.ts +1 -1
- package/src/report/components/summaries/compute.ts +3 -2
- package/src/report/components/summaries/index.tsx +18 -1
- package/src/report/definition/cell.ts +35 -15
- package/src/report/definition/primitives.tsx +61 -18
- package/src/report/definition/table.test.tsx +0 -56
- package/src/report/index.ts +1 -0
- package/src/report/model/aggregate.ts +43 -4
- package/src/report/model/calculation.test.ts +7 -56
- package/src/report/model/calculation.ts +1 -7
- package/src/report/model/evaluation-kind.ts +2 -1
- package/src/report/model/flag.ts +2 -0
- package/src/report/model/format.ts +3 -4
- package/src/report/model/locale.ts +25 -13
- package/src/report/model/metrics.ts +6 -11
- package/src/report/model/types.ts +37 -11
- package/src/report/react/index.tsx +1 -0
- package/src/report/runtime/dual-render.test.tsx +42 -6
- package/src/report/slices/compute.ts +61 -2
- package/src/report/slices/content.ts +15 -7
- package/src/report/slices/delta-table.test.ts +101 -33
- package/src/report/slices/validate.test.ts +26 -4
- package/src/report/slices/validate.ts +19 -2
- package/src/report/tasks.ts +8 -2
- package/src/runner/accept.test.ts +5 -30
- package/src/runner/accept.ts +3 -34
- package/src/runner/attempt.ts +2 -1
- package/src/runner/config-identity.test.ts +6 -3
- package/src/runner/eval-selection.ts +15 -0
- package/src/runner/experiment-labels.test.ts +3 -2
- package/src/runner/feedback/human.test.ts +23 -23
- package/src/runner/feedback/human.ts +11 -13
- package/src/runner/feedback/json.ts +1 -1
- package/src/runner/fingerprint.test.ts +11 -8
- package/src/runner/fingerprint.ts +2 -2
- package/src/runner/gate-lease.test.ts +10 -10
- package/src/runner/gate-lease.ts +7 -7
- package/src/runner/lock.test.ts +8 -8
- package/src/runner/lock.ts +4 -4
- package/src/runner/reporters/artifacts.ts +17 -1
- package/src/runner/reporters/braintrust.test.ts +1 -1
- package/src/runner/run.ts +7 -7
- package/src/runner/session.test.ts +13 -15
- package/src/runner/session.ts +24 -24
- package/src/runner/teardown-registry.test.ts +9 -9
- package/src/runner/teardown-registry.ts +4 -4
- package/src/runner/types.ts +8 -18
- package/src/sample/index.ts +84 -63
- package/src/sample/sample.test.ts +89 -107
- package/src/sandbox/compose.test.ts +16 -519
- package/src/sandbox/compose.ts +17 -323
- package/src/sandbox/docker.test.ts +2 -209
- package/src/sandbox/docker.ts +50 -573
- package/src/sandbox/dockerfile-build.test.ts +1 -52
- package/src/sandbox/dockerfile-build.ts +7 -44
- package/src/sandbox/index.ts +0 -64
- package/src/sandbox/keep.test.ts +0 -87
- package/src/sandbox/keep.ts +0 -64
- package/src/sandbox/layer.test.ts +0 -37
- package/src/sandbox/layer.ts +17 -647
- package/src/sandbox/link.test.ts +2 -3
- package/src/sandbox/plan.test.ts +0 -77
- package/src/sandbox/registry.ts +3 -6
- package/src/sandbox/runtime.test.ts +0 -29
- package/src/sandbox/runtime.ts +19 -235
- package/src/show/index.test.ts +1 -0
- package/src/show/index.ts +24 -18
- package/src/show/json.test.ts +8 -1
- package/src/show/json.ts +3 -0
- package/src/view/data.test.ts +1 -0
- package/src/view/data.ts +5 -2
- package/src/view/server-rebuild.test.ts +1 -0
- package/src/view/site-param-pages.test.ts +1 -0
- package/dist/sandbox/dind-supervisor.d.ts +0 -12
- package/dist/sandbox/docker-profile/errors.d.ts +0 -25
- package/dist/sandbox/docker-profile/registry.d.ts +0 -43
- package/dist/sandbox/docker-profile/runtime.d.ts +0 -36
- package/dist/sandbox/docker-profile/schema.d.ts +0 -129
- package/docs-site/zh/tutorials/docker-in-docker.mdx +0 -223
- package/src/cli-exp-rename.test.ts +0 -272
- package/src/runner/rename-experiment.test.ts +0 -241
- package/src/runner/rename-experiment.ts +0 -489
- package/src/sandbox/dind-supervisor.test.ts +0 -30
- package/src/sandbox/dind-supervisor.ts +0 -125
- package/src/sandbox/docker-profile/cli.ts +0 -205
- package/src/sandbox/docker-profile/errors.ts +0 -45
- package/src/sandbox/docker-profile/registry.ts +0 -450
- package/src/sandbox/docker-profile/runtime.ts +0 -238
- package/src/sandbox/docker-profile/schema.ts +0 -798
- package/src/sandbox/docker-profile.test.ts +0 -501
- package/src/sandbox/registry.test.ts +0 -46
package/INDEX.md
CHANGED
|
@@ -28,7 +28,6 @@
|
|
|
28
28
|
- `docs-site/zh/tutorials/custom-reports.mdx` — 编写自定义报告:用普通 TypeScript 函数计算结果,并同时生成终端与网页报告。
|
|
29
29
|
- `docs-site/zh/tutorials/dataset-fanout.mdx` — 数据驱动测试(dataset fan-out):用多份数据运行同一套评估用例:从 .eval.ts 文件导出数组或 keyed record,将一套评估逻辑展开为多个 case。用 loadYaml 或 loadJson 读取外部测试集,并获得稳定 ID。
|
|
30
30
|
- `docs-site/zh/tutorials/deploy-report-site.mdx` — 把报告站发布上线:把 .niceeval 直接提交进仓库、配一份排除运行时状态的 .gitignore,再用不装整仓依赖的 CI 脚本和 Vercel CLI,在打 tag 时把报告站发布上线。
|
|
31
|
-
- `docs-site/zh/tutorials/docker-in-docker.mdx` — 让 Sandbox 使用 Docker:根据任务信任边界选择 Docker socket、raw privileged DinD 或受管 rootless DinD,并为 Agent 准备 Docker CLI 与 daemon。
|
|
32
31
|
- `docs-site/zh/tutorials/evaluation-kinds.mdx` — 题型、断言、Judge 和成本限制:值断言、作用域断言、LLM-as-judge、严重度怎么折成一个判定、计分制怎么给部分分,以及成本上限怎么写。
|
|
33
32
|
- `docs-site/zh/tutorials/experiments.mdx` — 实验矩阵:用运行矩阵比较 agents 和 models:使用 NiceEval experiments 让同一批评估用例横跨多个 agents、models 和 flags,比较 pass rate、成本和延迟。
|
|
34
33
|
- `docs-site/zh/tutorials/fixtures.mdx` — Sandbox Fixture:用任务评估 coding agents:用 .eval.ts 给 coding agent 准备隔离 workspace、发送真实任务,并用 Sandbox 文件、命令、diff 和 judge 验证结果。
|
|
@@ -62,7 +61,7 @@
|
|
|
62
61
|
- `docs-site/zh/reference/builtin-agents.mdx` — 内置 Agent 能力参考:NiceEval 内置的 claude-code、codex、bub 适配器分别做到了哪些能力,对应哪些断言,以及已知限制。
|
|
63
62
|
- `docs-site/zh/reference/capabilities.mdx` — 能力与证据覆盖参考:Agent 的构造方式、运行时行为与必填的六通道 evidenceCoverage 声明,怎样共同决定 NiceEval 的断言能相信什么。
|
|
64
63
|
- `docs-site/zh/reference/cli.mdx` — NiceEval CLI:命令、flags 和退出码参考:NiceEval CLI 参考:exp、show、view、init、list 和 clean 命令,以及结果下钻、experiment、评估用例过滤、并发、预算和 CI 输出。
|
|
65
|
-
- `docs-site/zh/reference/define-agent.mdx` —
|
|
64
|
+
- `docs-site/zh/reference/define-agent.mdx` — defineDirectAgent 和 defineSandboxAgent:Adapter 参考:defineDirectAgent 和 defineSandboxAgent 参考:AgentContext、AgentSession、Sandbox 接口和共享 Sandbox helpers。
|
|
66
65
|
- `docs-site/zh/reference/define-config.mdx` — defineConfig:项目默认配置:defineConfig 参考:judge、reporters、并发、超时和 sandbox 默认值。
|
|
67
66
|
- `docs-site/zh/reference/define-eval.mdx` — defineEval:声明、配置并运行 NiceEval 评估用例:defineEval 参考:选项、test context t、Turn 返回值、Sandbox helper,以及数组和 keyed record 测试集导出。
|
|
68
67
|
- `docs-site/zh/reference/events.mdx` — 标准事件流参考:StreamEvent 的十种事件:每种的字段、什么时候吐、哪些断言消费它。adapter 的核心工作就是产出这条流。
|
package/dist/agents/types.d.ts
CHANGED
|
@@ -139,7 +139,7 @@ export type EvidenceCoverageEntry = {
|
|
|
139
139
|
/**
|
|
140
140
|
* 覆盖声明(EvidenceCoverage):完整性不是口头承诺,是随数据走的声明
|
|
141
141
|
* (见 docs/feature/adapters/architecture/evidence.md)。两层:
|
|
142
|
-
* - Agent 级默认(`
|
|
142
|
+
* - Agent 级默认(`defineDirectAgent` / `defineSandboxAgent` 的 `evidenceCoverage`)必须逐一声明
|
|
143
143
|
* 六个通道;官方 SDK 适配器可使用 `completeEvidenceCoverage`。
|
|
144
144
|
* - Turn 级降级(`Turn.evidenceCoverage`)只用于相对 Agent 默认值降级。
|
|
145
145
|
*/
|
|
@@ -562,7 +562,7 @@ export interface SandboxAgentDef {
|
|
|
562
562
|
* 在 finally 里跑一次。 */
|
|
563
563
|
teardown?: AgentTeardown;
|
|
564
564
|
}
|
|
565
|
-
/** `
|
|
565
|
+
/** `defineDirectAgent()` 的入参形状(见 src/define.ts)——`kind: "direct"` 由 define 固定填入,不由用户声明。 */
|
|
566
566
|
export interface DirectAgentDef {
|
|
567
567
|
/** agent 的显示名/标识,原样进入 `Agent.name`——不是注册表查找 key,只用于展示、结果归属与去重指纹。 */
|
|
568
568
|
name: string;
|
package/dist/i18n/en.d.ts
CHANGED
|
@@ -82,25 +82,6 @@ export declare const en: {
|
|
|
82
82
|
"cli.accept.flagUnsupported": string;
|
|
83
83
|
"cli.accept.failed": string;
|
|
84
84
|
"cli.accept.done": string;
|
|
85
|
-
"cli.rename.usage": string;
|
|
86
|
-
"cli.rename.flagUnsupported": string;
|
|
87
|
-
"cli.rename.previewHeader": string;
|
|
88
|
-
"cli.rename.blocked": string;
|
|
89
|
-
"cli.rename.migratingHeader": string;
|
|
90
|
-
"cli.rename.migratingRow": string;
|
|
91
|
-
"cli.rename.excludedHeader": string;
|
|
92
|
-
"cli.rename.excludedRow": string;
|
|
93
|
-
"cli.rename.doneHeader": string;
|
|
94
|
-
"cli.rename.snapshotPath": string;
|
|
95
|
-
"cli.rename.doneRow": string;
|
|
96
|
-
"cli.rename.error.sourceEmpty": string;
|
|
97
|
-
"cli.rename.error.targetNotFound": string;
|
|
98
|
-
"cli.rename.error.targetHasResults": string;
|
|
99
|
-
"cli.rename.error.sourceUnreadable": string;
|
|
100
|
-
"cli.rename.error.artifactUnavailable": string;
|
|
101
|
-
"cli.rename.error.nothingToMigrate": string;
|
|
102
|
-
"cli.rename.conflicting": string;
|
|
103
|
-
"cli.rename.failed": string;
|
|
104
85
|
"cli.error": string;
|
|
105
86
|
"cli.flag.acceptNeedsSelector": string;
|
|
106
87
|
"cli.flag.acceptWithRerunAll": string;
|
package/dist/i18n/en.js
CHANGED
|
@@ -79,7 +79,7 @@ export const en = {
|
|
|
79
79
|
"cli.dry.unit.configs": "configs",
|
|
80
80
|
"cli.dry.affects": "affects {{evals}} · {{ids}}",
|
|
81
81
|
"cli.dry.acceptHint": "accept: {{command}}",
|
|
82
|
-
"cli.accept.choiceHeader": "
|
|
82
|
+
"cli.accept.choiceHeader": "stale {{selector}}{{change}} ({{evals}} evals)\n",
|
|
83
83
|
"cli.accept.prompt": " reuse these results? [y/N] ",
|
|
84
84
|
"cli.accept.nothingToAccept": "No difference in this plan can be accepted (nothing is blocked by the fingerprint gate).\n" +
|
|
85
85
|
"Running as planned.\n",
|
|
@@ -91,37 +91,9 @@ export const en = {
|
|
|
91
91
|
" fix: pass only @<locator> (and optionally --record <dir>)\n",
|
|
92
92
|
"cli.accept.failed": "error: could not accept result: {{error}}\n",
|
|
93
93
|
"cli.accept.done": "Accepted {{sourceLocator}}. New result locator: {{locator}}. Current fingerprint: {{fingerprint}}\n",
|
|
94
|
-
"cli.rename.usage": "error: niceeval exp rename expects exactly two arguments: an old id and a new id\n" +
|
|
95
|
-
" fix: niceeval exp rename <oldId> <newId> [--dry] [--json]\n",
|
|
96
|
-
"cli.rename.flagUnsupported": "error: {{flag}} is not valid with niceeval exp rename\n" +
|
|
97
|
-
" fix: pass only <oldId> <newId>, optionally --dry / --json\n",
|
|
98
|
-
"cli.rename.previewHeader": "exp rename preview: {{oldId}} -> {{newId}}\n",
|
|
99
|
-
"cli.rename.blocked": " blocked (nothing will be written): {{reason}}\n",
|
|
100
|
-
"cli.rename.migratingHeader": " {{count}} terminal results will migrate:\n",
|
|
101
|
-
"cli.rename.migratingRow": " {{evalId}} {{sourceLocator}} -> {{newId}}\n",
|
|
102
|
-
"cli.rename.excludedHeader": " {{count}} excluded (not migrated, does not block):\n",
|
|
103
|
-
"cli.rename.excludedRow": " {{evalId}} {{reason}}\n",
|
|
104
|
-
"cli.rename.doneHeader": "exp rename done: rebound {{count}} terminal results from {{oldId}} to {{newId}}.\n",
|
|
105
|
-
"cli.rename.snapshotPath": " new snapshot: {{path}}\n",
|
|
106
|
-
"cli.rename.doneRow": " {{evalId}} {{sourceLocator}} -> {{locator}}\n",
|
|
107
|
-
"cli.rename.error.sourceEmpty": "error: {{oldId}} has no readable terminal history to migrate to {{newId}}.\n" +
|
|
108
|
-
" fix: restore and verify {{oldId}}'s real results before retrying; with no old results, run `niceeval exp {{newId}}` and do not rename.\n" +
|
|
109
|
-
" exp rename does not move experiment source, nor delete or rewrite the old result tree.\n",
|
|
110
|
-
"cli.rename.error.targetNotFound": "error: new id \"{{newId}}\" is not discovered under this project's experiments/.\n" +
|
|
111
|
-
" fix: create or rename the experiment in experiments/ first (e.g. `git mv experiments/{{oldId}}.ts experiments/{{newId}}.ts`), then rerun.\n",
|
|
112
|
-
"cli.rename.error.targetHasResults": "error: {{newId}} already has terminal results for these evals; rename never overwrites existing results.\n" +
|
|
113
|
-
" fix: keep the target results, or explicitly clean the target history and re-preview; the command deletes nothing itself.\n",
|
|
114
|
-
"cli.rename.error.sourceUnreadable": "error: the Record for {{oldId}} is unreadable; cannot migrate to {{newId}}.\n" +
|
|
115
|
-
" fix: view this record with a niceeval version that reads its schemaVersion.\n",
|
|
116
|
-
"cli.rename.error.artifactUnavailable": "error: source evidence cannot be preserved ({{evalId}}); nothing will be written.\n" +
|
|
117
|
-
" fix: make the artifact reference and source locator readable, or rerun this eval.\n",
|
|
118
|
-
"cli.rename.error.nothingToMigrate": "error: nothing to migrate under {{oldId}}: no terminal passed/failed still selected by {{newId}}, or all excluded.\n" +
|
|
119
|
-
" fix: check that {{newId}}'s evals selector covers the old experiment's results.\n",
|
|
120
|
-
"cli.rename.conflicting": " conflicting evals: {{evals}}\n",
|
|
121
|
-
"cli.rename.failed": "error: exp rename failed: {{error}}\n",
|
|
122
94
|
"cli.error": "niceeval error: {{error}}\n",
|
|
123
95
|
"cli.flag.acceptNeedsSelector": "error: --accept needs a selector, for example --accept config:judge.model\n" +
|
|
124
|
-
" fix: run `niceeval exp <selection> --dry` first; every `
|
|
96
|
+
" fix: run `niceeval exp <selection> --dry` first; every `stale` line prints the selectors it can accept, copy one verbatim\n" +
|
|
125
97
|
" differences you can accept in this plan: {{available}}\n",
|
|
126
98
|
"cli.flag.acceptWithRerunAll": "--accept cannot be combined with --rerun all: one says trust nothing from cache, the other says trust this difference anyway.\n" +
|
|
127
99
|
"Drop --rerun all to accept the difference, or drop --accept to rerun everything.\n",
|
|
@@ -136,8 +108,8 @@ export const en = {
|
|
|
136
108
|
"runner.setupReturnedCleanup": "{{layer}} returned a function. setup does not carry cleanup and the returned value will not be executed — put the cleanup in the paired teardown of the same layer ({{hint}}); see the experiments tutorial on docs-site or docs/runner.md.\n",
|
|
137
109
|
"runner.experimentTeardownLate": "experiment {{experimentId}}'s teardown was not triggered by the normal countdown path; it has been executed by the end-of-run sweep instead. Record are unaffected; seeing this line means an unlocated intermittent scheduling issue fired — please record this run in the memory ledger.\n",
|
|
138
110
|
"runner.teardownRegistrationWriteFailed": "writing the crash-recovery teardown registration for experiment {{experimentId}} failed: {{message}}. The run continues normally, but a SIGKILL during this run cannot be recovered via `niceeval exp --teardown` or the startup self-heal — check disk space/permissions under .niceeval/teardowns/.\n",
|
|
139
|
-
"runner.lockTakenOver": "took over an expired case lock for {{experimentId}}/{{evalId}} (previously held by pid {{pid}} on {{host}}; its heartbeat
|
|
140
|
-
"runner.gateLeaseTakenOver": "took over an expired concurrency-slot lease for experiment {{experimentId}} (slot {{slot}}, previously held by pid {{pid}} on {{host}}; its heartbeat
|
|
111
|
+
"runner.lockTakenOver": "took over an expired case lock for {{experimentId}}/{{evalId}} (previously held by pid {{pid}} on {{host}}; its heartbeat went stale) — that run likely died without releasing it; this run now owns dispatching this case.\n",
|
|
112
|
+
"runner.gateLeaseTakenOver": "took over an expired concurrency-slot lease for experiment {{experimentId}} (slot {{slot}}, previously held by pid {{pid}} on {{host}}; its heartbeat went stale) — that run likely died without releasing it; this run now owns the slot.\n",
|
|
141
113
|
"runner.gateLeaseWaiting": "waiting on another run for experiment {{experimentId}}'s concurrency slots: all {{effectiveN}} in use ({{holders}}). Concurrent runs share this experiment's slots, and the smallest maxConcurrency in play wins — this run declared {{declaredN}}. Nothing dispatches until a slot frees up; the other run's slots release when its attempts finish, or 30s after it dies.\n",
|
|
142
114
|
"runner.dispatchHaltedExperiment": "experiment halted (dispatch-halted): {{message}}\n",
|
|
143
115
|
"runner.dispatchHaltedEval": "eval halted: {{message}}\n",
|
|
@@ -155,8 +127,6 @@ export const en = {
|
|
|
155
127
|
"Usage:\n" +
|
|
156
128
|
" niceeval exp [path|experiment] [eval-id-prefix…] run experiments\n" +
|
|
157
129
|
" niceeval exp list [experiment-prefix] list runnable configs (no dispatch)\n" +
|
|
158
|
-
" niceeval exp rename <oldId> <newId> rebind terminal results from old to new id\n" +
|
|
159
|
-
" --dry preview without writing; --json one JSON document\n" +
|
|
160
130
|
" --teardown recover a killed run: run only the selected experiments'\n" +
|
|
161
131
|
" teardown (no attempts, no setup); combining it with eval id prefixes is an error\n" +
|
|
162
132
|
" niceeval accept @<locator>... accept explicit historical results\n" +
|
|
@@ -186,6 +156,8 @@ export const en = {
|
|
|
186
156
|
" --record <dir> pin a record root --exp <id> repeatable; 2+ compares conditions\n" +
|
|
187
157
|
" --report <file> custom report --page <id> pick the initial page (multi-page\n" +
|
|
188
158
|
" reports render it, then list the rest as a page index with copyable commands)\n" +
|
|
159
|
+
" --fresh only count freshly executed attempts (excludes carried-over and\n" +
|
|
160
|
+
" historical stitched-in attempts); excluded evals show up as placeholder rows\n" +
|
|
189
161
|
" niceeval list list discovered evals\n" +
|
|
190
162
|
" niceeval session list [--all] [experiment-prefix] query Sessions (read-only)\n" +
|
|
191
163
|
" niceeval session show <sessionId> show one Session (read-only)\n" +
|
|
@@ -194,6 +166,7 @@ export const en = {
|
|
|
194
166
|
" (same file as show); --page <id> picks the initial page;\n" +
|
|
195
167
|
" --record <dir> pins a record root; --run <file> opens exactly\n" +
|
|
196
168
|
" one run; --exp <id> (repeatable) narrows to those experiments;\n" +
|
|
169
|
+
" --fresh only new executions\n" +
|
|
197
170
|
" --out <dir> exports a static site: index.html plus the viewer\n" +
|
|
198
171
|
" artifacts, ready for any static host\n" +
|
|
199
172
|
" niceeval sandbox list|enter|history|diff|stop inspect & destroy sandboxes kept by --keep-sandbox\n" +
|
|
@@ -277,7 +250,7 @@ export const en = {
|
|
|
277
250
|
"context.capabilityMissing": "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
|
|
278
251
|
" Docs: node_modules/niceeval/docs-site/zh/tutorials/sandbox-agent.mdx",
|
|
279
252
|
"context.skipEmpty": "skip() requires a non-empty reason.",
|
|
280
|
-
"define.agentNameRequired": "
|
|
253
|
+
"define.agentNameRequired": "defineDirectAgent requires name.",
|
|
281
254
|
"define.evalIdRejected": "defineEval does not accept id; ids are derived from file paths.",
|
|
282
255
|
"define.evalEnvironmentEmpty": "defineEval environment must be a non-empty profile id when provided.",
|
|
283
256
|
"define.evalTestRequired": "defineEval requires an async test(t) function.",
|
package/dist/i18n/zh-CN.d.ts
CHANGED
|
@@ -73,7 +73,7 @@ export declare const zhCN: {
|
|
|
73
73
|
readonly "cli.dry.unit.configs": "个运行配置";
|
|
74
74
|
readonly "cli.dry.affects": "影响 {{evals}} · {{ids}}";
|
|
75
75
|
readonly "cli.dry.acceptHint": "accept: {{command}}";
|
|
76
|
-
readonly "cli.accept.choiceHeader": "
|
|
76
|
+
readonly "cli.accept.choiceHeader": "stale {{selector}}{{change}} ({{evals}} 条 eval)\n";
|
|
77
77
|
readonly "cli.accept.prompt": " 复用这些结果吗? [y/N] ";
|
|
78
78
|
readonly "cli.accept.nothingToAccept": string;
|
|
79
79
|
readonly "cli.accept.equivalent": "等价命令: {{command}}\n";
|
|
@@ -82,25 +82,6 @@ export declare const zhCN: {
|
|
|
82
82
|
readonly "cli.accept.flagUnsupported": string;
|
|
83
83
|
readonly "cli.accept.failed": "error: 接受结果失败:{{error}}\n";
|
|
84
84
|
readonly "cli.accept.done": "已接受 {{sourceLocator}}。新结果 locator: {{locator}}。当前指纹:{{fingerprint}}\n";
|
|
85
|
-
readonly "cli.rename.usage": string;
|
|
86
|
-
readonly "cli.rename.flagUnsupported": string;
|
|
87
|
-
readonly "cli.rename.previewHeader": "exp rename 预览:{{oldId}} → {{newId}}\n";
|
|
88
|
-
readonly "cli.rename.blocked": " 整批阻断,零写入:{{reason}}\n";
|
|
89
|
-
readonly "cli.rename.migratingHeader": " 将迁移 {{count}} 条终态结果:\n";
|
|
90
|
-
readonly "cli.rename.migratingRow": " {{evalId}} {{sourceLocator}} → {{newId}}\n";
|
|
91
|
-
readonly "cli.rename.excludedHeader": " 排除 {{count}} 条(不迁移,不阻断):\n";
|
|
92
|
-
readonly "cli.rename.excludedRow": " {{evalId}} {{reason}}\n";
|
|
93
|
-
readonly "cli.rename.doneHeader": "exp rename 完成:把 {{oldId}} 的 {{count}} 条终态结果重绑到 {{newId}}。\n";
|
|
94
|
-
readonly "cli.rename.snapshotPath": " 新 snapshot:{{path}}\n";
|
|
95
|
-
readonly "cli.rename.doneRow": " {{evalId}} {{sourceLocator}} → {{locator}}\n";
|
|
96
|
-
readonly "cli.rename.error.sourceEmpty": string;
|
|
97
|
-
readonly "cli.rename.error.targetNotFound": string;
|
|
98
|
-
readonly "cli.rename.error.targetHasResults": string;
|
|
99
|
-
readonly "cli.rename.error.sourceUnreadable": string;
|
|
100
|
-
readonly "cli.rename.error.artifactUnavailable": string;
|
|
101
|
-
readonly "cli.rename.error.nothingToMigrate": string;
|
|
102
|
-
readonly "cli.rename.conflicting": " 冲突 eval:{{evals}}\n";
|
|
103
|
-
readonly "cli.rename.failed": "error: exp rename 失败:{{error}}\n";
|
|
104
85
|
readonly "cli.error": "niceeval 出错:{{error}}\n";
|
|
105
86
|
readonly "cli.flag.acceptNeedsSelector": string;
|
|
106
87
|
readonly "cli.flag.acceptWithRerunAll": string;
|
|
@@ -184,7 +165,7 @@ export declare const zhCN: {
|
|
|
184
165
|
readonly "codex.envPathManaged": string;
|
|
185
166
|
readonly "context.capabilityMissing": string;
|
|
186
167
|
readonly "context.skipEmpty": "skip() 需要一个非空理由。";
|
|
187
|
-
readonly "define.agentNameRequired": "
|
|
168
|
+
readonly "define.agentNameRequired": "defineDirectAgent 需要 name。";
|
|
188
169
|
readonly "define.evalIdRejected": "defineEval 不接受 id —— id 由文件路径推导。";
|
|
189
170
|
readonly "define.evalEnvironmentEmpty": "defineEval 的 environment 如有提供,必须是非空的 profile id。";
|
|
190
171
|
readonly "define.evalTestRequired": "defineEval 需要一个 async test(t) 函数。";
|
package/dist/i18n/zh-CN.js
CHANGED
|
@@ -79,7 +79,7 @@ export const zhCN = {
|
|
|
79
79
|
"cli.dry.unit.configs": "个运行配置",
|
|
80
80
|
"cli.dry.affects": "影响 {{evals}} · {{ids}}",
|
|
81
81
|
"cli.dry.acceptHint": "accept: {{command}}",
|
|
82
|
-
"cli.accept.choiceHeader": "
|
|
82
|
+
"cli.accept.choiceHeader": "stale {{selector}}{{change}} ({{evals}} 条 eval)\n",
|
|
83
83
|
"cli.accept.prompt": " 复用这些结果吗? [y/N] ",
|
|
84
84
|
"cli.accept.nothingToAccept": "本次计划里没有可授权的差异(没有条目卡在指纹门上)。\n" +
|
|
85
85
|
"按原计划运行。\n",
|
|
@@ -91,37 +91,9 @@ export const zhCN = {
|
|
|
91
91
|
" fix: 只传 @<locator>(可选 `--record <目录>`)\n",
|
|
92
92
|
"cli.accept.failed": "error: 接受结果失败:{{error}}\n",
|
|
93
93
|
"cli.accept.done": "已接受 {{sourceLocator}}。新结果 locator: {{locator}}。当前指纹:{{fingerprint}}\n",
|
|
94
|
-
"cli.rename.usage": "error: niceeval exp rename 需要恰好两个参数:旧 id 和新 id\n" +
|
|
95
|
-
" fix: niceeval exp rename <oldId> <newId> [--dry] [--json]\n",
|
|
96
|
-
"cli.rename.flagUnsupported": "error: {{flag}} 不能用于 niceeval exp rename\n" +
|
|
97
|
-
" fix: 只传 <oldId> <newId>,可选 --dry / --json\n",
|
|
98
|
-
"cli.rename.previewHeader": "exp rename 预览:{{oldId}} → {{newId}}\n",
|
|
99
|
-
"cli.rename.blocked": " 整批阻断,零写入:{{reason}}\n",
|
|
100
|
-
"cli.rename.migratingHeader": " 将迁移 {{count}} 条终态结果:\n",
|
|
101
|
-
"cli.rename.migratingRow": " {{evalId}} {{sourceLocator}} → {{newId}}\n",
|
|
102
|
-
"cli.rename.excludedHeader": " 排除 {{count}} 条(不迁移,不阻断):\n",
|
|
103
|
-
"cli.rename.excludedRow": " {{evalId}} {{reason}}\n",
|
|
104
|
-
"cli.rename.doneHeader": "exp rename 完成:把 {{oldId}} 的 {{count}} 条终态结果重绑到 {{newId}}。\n",
|
|
105
|
-
"cli.rename.snapshotPath": " 新 snapshot:{{path}}\n",
|
|
106
|
-
"cli.rename.doneRow": " {{evalId}} {{sourceLocator}} → {{locator}}\n",
|
|
107
|
-
"cli.rename.error.sourceEmpty": "error: {{oldId}} 下没有可读的终态历史,无法迁移到 {{newId}}。\n" +
|
|
108
|
-
" fix: 恢复并确认 {{oldId}} 的真实结果后重试;没有旧结果时直接运行 `niceeval exp {{newId}}`,不要再执行 rename。\n" +
|
|
109
|
-
" exp rename 不移动实验源码,也不删除或改写旧结果树。\n",
|
|
110
|
-
"cli.rename.error.targetNotFound": "error: 新 id \"{{newId}}\" 没有被当前项目的 experiments/ 发现。\n" +
|
|
111
|
-
" fix: 先在 experiments/ 创建或改名出新实验(如 `git mv experiments/{{oldId}}.ts experiments/{{newId}}.ts`),再重跑本命令。\n",
|
|
112
|
-
"cli.rename.error.targetHasResults": "error: {{newId}} 下已有这些 eval 的终态结果,改名不覆盖已有结果。\n" +
|
|
113
|
-
" fix: 保留目标结果继续用,或显式清理目标历史后重新预览;命令自身不删除数据。\n",
|
|
114
|
-
"cli.rename.error.sourceUnreadable": "error: {{oldId}} 的 Record 读不动,无法迁移到 {{newId}}。\n" +
|
|
115
|
-
" fix: 用能读该 schemaVersion 的 niceeval 版本查看这份记录。\n",
|
|
116
|
-
"cli.rename.error.artifactUnavailable": "error: 来源证据无法保留({{evalId}}),整批零写入。\n" +
|
|
117
|
-
" fix: 先确认 artifact 引用与来源 locator 可读,或对这条 eval 重新运行。\n",
|
|
118
|
-
"cli.rename.error.nothingToMigrate": "error: {{oldId}} 下没有可迁移的 eval:没有 {{newId}} 仍选中的终态 passed/failed,或全部被排除。\n" +
|
|
119
|
-
" fix: 检查 {{newId}} 的 evals 选择器是否覆盖旧实验结果。\n",
|
|
120
|
-
"cli.rename.conflicting": " 冲突 eval:{{evals}}\n",
|
|
121
|
-
"cli.rename.failed": "error: exp rename 失败:{{error}}\n",
|
|
122
94
|
"cli.error": "niceeval 出错:{{error}}\n",
|
|
123
95
|
"cli.flag.acceptNeedsSelector": "error: --accept 必须带 selector,例如 --accept config:judge.model\n" +
|
|
124
|
-
" fix: 先跑 `niceeval exp <选择> --dry`,每条 `
|
|
96
|
+
" fix: 先跑 `niceeval exp <选择> --dry`,每条 `stale` 行都打出了它可授权的 selector,原样复制一条\n" +
|
|
125
97
|
" 本次计划里可以授权的差异:{{available}}\n",
|
|
126
98
|
"cli.flag.acceptWithRerunAll": "--accept 不能与 --rerun all 同用:一边全不采信缓存,一边又要采信这条差异,方向自相矛盾。\n" +
|
|
127
99
|
"要授权这条差异就去掉 --rerun all;要全量重跑就去掉 --accept。\n",
|
|
@@ -155,8 +127,6 @@ export const zhCN = {
|
|
|
155
127
|
"用法:\n" +
|
|
156
128
|
" niceeval exp [路径|实验] [eval-id 前缀…] 跑实验\n" +
|
|
157
129
|
" niceeval exp list [实验前缀] 列出可运行的实验配置(不派发)\n" +
|
|
158
|
-
" niceeval exp rename <旧 id> <新 id> 把旧实验的终态结果重绑到新 id(显式审计迁移)\n" +
|
|
159
|
-
" --dry 只预览不写盘;--json 输出单份 JSON 文档\n" +
|
|
160
130
|
" --teardown 强杀后补收尾:只对选中的实验各执行一次 teardown(不派发\n" +
|
|
161
131
|
" attempt、不跑 setup);与 eval id 前缀组合是用法错误\n" +
|
|
162
132
|
" niceeval accept @<locator>... 接受明确列出的历史结果\n" +
|
|
@@ -183,6 +153,8 @@ export const zhCN = {
|
|
|
183
153
|
" --record <目录> 钉死记录根 --exp <id> 可重复,两个以上进入对照\n" +
|
|
184
154
|
" --report <文件> 自定义报告 --page <id> 定初始页(多页报告渲染该页,\n" +
|
|
185
155
|
" 尾部再附其余页索引)\n" +
|
|
156
|
+
" --fresh 只统计新执行的 attempt(排除携带条目与跨快照拼入的历史执行);\n" +
|
|
157
|
+
" 被排除的题转成占位行\n" +
|
|
186
158
|
" niceeval list 列出发现到的 eval\n" +
|
|
187
159
|
" niceeval session list [--all] [实验前缀] 查询 Session(只读)\n" +
|
|
188
160
|
" niceeval session show <sessionId> 查看一个 Session(只读)\n" +
|
|
@@ -190,6 +162,7 @@ export const zhCN = {
|
|
|
190
162
|
" 报告页 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
|
|
191
163
|
" --page <id> 定初始页 --record <目录> 钉死记录根\n" +
|
|
192
164
|
" --run <文件> 只打开这一份快照 --exp <id>(可重复)收窄到这些实验;\n" +
|
|
165
|
+
" --fresh 只看新执行\n" +
|
|
193
166
|
" --out <目录> 静态导出:index.html 连同查看器 artifact,可直接静态托管\n" +
|
|
194
167
|
" niceeval sandbox list|enter|history|diff|stop 查看与销毁 --keep-sandbox 留下的现场\n" +
|
|
195
168
|
" niceeval sandbox list --orphans / prune 核对并收回被强杀留下的无主实例\n" +
|
|
@@ -272,7 +245,7 @@ export const zhCN = {
|
|
|
272
245
|
"context.capabilityMissing": "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +
|
|
273
246
|
" 文档:node_modules/niceeval/docs-site/zh/tutorials/sandbox-agent.mdx",
|
|
274
247
|
"context.skipEmpty": "skip() 需要一个非空理由。",
|
|
275
|
-
"define.agentNameRequired": "
|
|
248
|
+
"define.agentNameRequired": "defineDirectAgent 需要 name。",
|
|
276
249
|
"define.evalIdRejected": "defineEval 不接受 id —— id 由文件路径推导。",
|
|
277
250
|
"define.evalEnvironmentEmpty": "defineEval 的 environment 如有提供,必须是非空的 profile id。",
|
|
278
251
|
"define.evalTestRequired": "defineEval 需要一个 async test(t) 函数。",
|
package/dist/record/open.js
CHANGED
|
@@ -30,8 +30,8 @@ const locatorIndexByResults = new WeakMap();
|
|
|
30
30
|
export class LocatorNotFoundError extends Error {
|
|
31
31
|
locator;
|
|
32
32
|
constructor(locator) {
|
|
33
|
-
super(`No attempt found for locator "${locator}" in this results root. It may
|
|
34
|
-
"(the run was deleted, or publish didn't include it) or
|
|
33
|
+
super(`No attempt found for locator "${locator}" in this results root. It may be stale ` +
|
|
34
|
+
"(the run was deleted, or publish didn't include it) or mistyped.");
|
|
35
35
|
this.locator = locator;
|
|
36
36
|
this.name = "LocatorNotFoundError";
|
|
37
37
|
}
|
package/dist/record/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DiagnosticRecord, EvalResult, ExperimentRunInfo, LocalizedText, SandboxBuildRecord, TimingActivity
|
|
1
|
+
import type { DiagnosticRecord, EvalResult, ExperimentRunInfo, LocalizedText, SandboxBuildRecord, TimingActivity } from "../types.ts";
|
|
2
2
|
import type { CommandExitEvidence, O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
|
|
3
3
|
import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
|
|
4
4
|
import type { AttemptIdentity, AttemptLocator } from "./locator.ts";
|
|
@@ -204,7 +204,7 @@ export interface Record {
|
|
|
204
204
|
}
|
|
205
205
|
/**
|
|
206
206
|
* 一个实验的覆盖事实:已知 eval 并集(分母)与当前口径下没有任何 attempt 的题。
|
|
207
|
-
* `
|
|
207
|
+
* `missingEvalIds` 永远被算出来,不静默——渲染面把它转成覆盖占位行
|
|
208
208
|
* (见 docs/feature/sample/library.md「选择快照」「时效:新执行与历史执行」)。
|
|
209
209
|
*
|
|
210
210
|
* `run` 是该 Experiment 的锚点 Run:零 attempt 的 Eval 按 agent / model / flags 归组时
|
|
@@ -217,20 +217,8 @@ export interface SampleCoverage {
|
|
|
217
217
|
run: Run;
|
|
218
218
|
/** 分母:本地历史 ∪ 各快照携带的 knownEvalIds,交命令行范围(与 `exp.knownEvalIds` 同源)。 */
|
|
219
219
|
knownEvalIds: string[];
|
|
220
|
-
/**
|
|
221
|
-
|
|
222
|
-
}
|
|
223
|
-
/** 当前结果缺口;原因只解释下一步,不构成另一种结果状态。 */
|
|
224
|
-
export interface SampleMissing {
|
|
225
|
-
evalId: string;
|
|
226
|
-
/** 从未有物理 Attempt,或有历史结果但没有一条能代表当前配置。 */
|
|
227
|
-
reason: "never-run" | "previous-result";
|
|
228
|
-
/** 最近一条旧结果的审计入口;它不参与当前统计,也不保证一定满足 accept 资格。 */
|
|
229
|
-
previous?: {
|
|
230
|
-
locator: AttemptLocator;
|
|
231
|
-
verdict: Verdict;
|
|
232
|
-
startedAt: string;
|
|
233
|
-
};
|
|
220
|
+
/** 当前口径下没有任何 attempt 的题(含 `fresh: true` 排除历史执行后新产生的缺口)。 */
|
|
221
|
+
missingEvalIds: string[];
|
|
234
222
|
}
|
|
235
223
|
/**
|
|
236
224
|
* Sample(范围):选出的快照 + 口径 + 已按口径物化的 attempt 全集 + 挑选警告。
|
|
@@ -240,6 +228,8 @@ export interface SampleMissing {
|
|
|
240
228
|
export interface Sample {
|
|
241
229
|
/** 这份 Sample 的口径,字面写在数据上。 */
|
|
242
230
|
mode: "latest-run" | "current";
|
|
231
|
+
/** 是否只保留各 Experiment 锚点中真实执行的 attempt。 */
|
|
232
|
+
fresh: boolean;
|
|
243
233
|
runs: Run[];
|
|
244
234
|
/**
|
|
245
235
|
* 按口径物化的 attempt 全集:消费 attempts 就自动正确,不需要自己 flatten runs,
|
|
@@ -261,6 +251,7 @@ export interface Sample {
|
|
|
261
251
|
evals?: string | string[];
|
|
262
252
|
}): Sample;
|
|
263
253
|
filter(predicate: (attempt: AttemptHandle) => boolean): Sample;
|
|
254
|
+
freshOnly(): Sample;
|
|
264
255
|
}
|
|
265
256
|
/**
|
|
266
257
|
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message;能用一条命令直接推进的
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
2
|
import { defineComponent } from "../definition/tree.js";
|
|
3
|
-
import { Callouts, Chart, Col, Conversation, CopyBlock, DiffView, Grid, Scatter, Series, SourceView, Stat, TableContentView, Waterfall, } from "../definition/primitives.js";
|
|
3
|
+
import { Callouts, Chart, Col, Conversation, CopyBlock, DiffView, Grid, Scatter, Series, SourceView, Stat, TableContentView, Text, Waterfall, } from "../definition/primitives.js";
|
|
4
4
|
import { HeroCard } from "../components/site-components/index.js";
|
|
5
5
|
import { AttemptSummary } from "../components/attempt-detail/index.js";
|
|
6
6
|
import { experimentListContent } from "../components/entity-lists/content.js";
|
|
7
|
+
import { ExperimentTableView } from "../components/entity-lists/index.js";
|
|
7
8
|
import { attemptAssertionsContent, attemptConversationContent, attemptDiffContent, embedConversationInSource, executionEvidenceUnavailableCallouts, attemptFixPromptContent, attemptNoticesContent, projectedSourceContent, attemptTimelineContent, } from "../components/attempt-detail/content.js";
|
|
8
9
|
import { stabilityMatrixContent } from "../slices/content.js";
|
|
10
|
+
import { formatInstant, formatReportDateTimeRange, } from "../model/format.js";
|
|
9
11
|
import { DEFAULT_REPORT_LOCALE, localeText } from "../model/locale.js";
|
|
10
12
|
function experimentTarget(point) {
|
|
11
13
|
return { page: "experiment", params: { experiment: point.key } };
|
|
@@ -13,15 +15,27 @@ function experimentTarget(point) {
|
|
|
13
15
|
function summaryView(result, locale) {
|
|
14
16
|
const snapshot = result.summary;
|
|
15
17
|
const tally = snapshot.evalVerdicts;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
const formattedRange = snapshot.range.earliestStartedAt !== null && snapshot.range.latestStartedAt !== null
|
|
19
|
+
? formatReportDateTimeRange(snapshot.range.earliestStartedAt, snapshot.range.latestStartedAt, locale)
|
|
20
|
+
: null;
|
|
21
|
+
return (_jsxs(Col, { className: "niceeval-sample-summary", children: [_jsxs(Grid, { children: [snapshot.evaluationKindComposition !== "points" ? (_jsx(Stat, { label: localeText(locale, "scopeSummary.passRate"), value: { kind: "metric", metric: snapshot.endToEndPassRate } })) : null, snapshot.totalScore !== undefined ? (_jsx(Stat, { label: localeText(locale, "scopeSummary.totalScore"), value: { kind: "metric", metric: snapshot.totalScore } })) : null, _jsx(Stat, { label: localeText(locale, "scopeSummary.experiments"), value: snapshot.experiments }), _jsx(Stat, { label: localeText(locale, "scopeSummary.evals"), value: snapshot.evals }), _jsx(Stat, { label: localeText(locale, "scopeSummary.attempts"), value: snapshot.attempts }), _jsx(Stat, { label: localeText(locale, "scopeSummary.votesEval"), value: { kind: "verdict", counts: tally } }), _jsx(Stat, { label: localeText(locale, "scopeSummary.totalCost"), value: { kind: "metric", metric: snapshot.totalCostUSD }, detail: snapshot.totalCostUSD.samples < snapshot.totalCostUSD.total
|
|
22
|
+
? localeText(locale, "scopeSummary.costCoverage", {
|
|
23
|
+
samples: snapshot.totalCostUSD.samples,
|
|
24
|
+
total: snapshot.totalCostUSD.total,
|
|
25
|
+
})
|
|
26
|
+
: undefined })] }), snapshot.range.latestStartedAt !== null ? (_jsx(Text, { className: "niceeval-sample-summary-range", children: snapshot.range.earliestStartedAt !== null &&
|
|
27
|
+
snapshot.range.earliestStartedAt !== snapshot.range.latestStartedAt
|
|
28
|
+
? localeText(locale, "scopeSummary.runRange", {
|
|
29
|
+
from: formattedRange.from,
|
|
30
|
+
to: formattedRange.to,
|
|
31
|
+
})
|
|
32
|
+
: localeText(locale, "scopeSummary.lastRun", {
|
|
33
|
+
time: formatInstant(snapshot.range.latestStartedAt, locale),
|
|
34
|
+
}) })) : null] }));
|
|
22
35
|
}
|
|
23
36
|
export const StandardOverviewResultView = defineComponent(async ({ result }, ctx) => {
|
|
24
37
|
const table = experimentListContent(result.experiments);
|
|
38
|
+
const freshTable = result.freshExperiments ? experimentListContent(result.freshExperiments) : null;
|
|
25
39
|
const hasPassRate = table.columns.some((column) => column.key === "passRate");
|
|
26
40
|
const hasTotalScore = table.columns.some((column) => column.key === "totalScore");
|
|
27
41
|
const defaultSort = hasPassRate === hasTotalScore
|
|
@@ -29,7 +43,7 @@ export const StandardOverviewResultView = defineComponent(async ({ result }, ctx
|
|
|
29
43
|
: hasPassRate
|
|
30
44
|
? "passRate"
|
|
31
45
|
: "totalScore";
|
|
32
|
-
return (_jsxs(Col, { children: [_jsx(HeroCard, { title: ctx.report.title, data: result.hero }), _jsx(Callouts, { items: result.notices }), _jsx(Callouts, { items: result.diagnostics }), _jsx(CopyBlock, { content: result.fixPrompt }), summaryView(result, DEFAULT_REPORT_LOCALE), _jsx(Col, { children: result.charts.map((chart) => chart.y === "passRate" ? (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "passRate", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y)) : (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "totalScore", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y))) }), _jsx(TableContentView, { data: table, sort: defaultSort, searchable: true })] }));
|
|
46
|
+
return (_jsxs(Col, { children: [_jsx(HeroCard, { title: ctx.report.title, data: result.hero }), _jsx(Callouts, { items: result.notices }), _jsx(Callouts, { items: result.diagnostics }), _jsx(CopyBlock, { content: result.fixPrompt }), summaryView(result, DEFAULT_REPORT_LOCALE), _jsx(Col, { children: result.charts.map((chart) => chart.y === "passRate" ? (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "passRate", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y)) : (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "totalScore", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y))) }), freshTable === null ? (_jsx(TableContentView, { data: table, sort: defaultSort, searchable: true })) : (_jsx(ExperimentTableView, { fullContent: table, freshContent: freshTable, sort: defaultSort, searchable: true }))] }));
|
|
33
47
|
});
|
|
34
48
|
StandardOverviewResultView.displayName = "StandardOverviewResultView";
|
|
35
49
|
function stabilityPresentation(result) {
|
|
@@ -11,6 +11,12 @@ export declare function failureSummaryOf(result: EvalResult): {
|
|
|
11
11
|
summary: string | null;
|
|
12
12
|
more: number;
|
|
13
13
|
};
|
|
14
|
+
/**
|
|
15
|
+
* 「只看新执行」开关在场的判据(docs/feature/reports/components/summaries/experiment-table.md
|
|
16
|
+
* 「只看新执行」):Sample 里既没有历史执行也没有过期结论时不画开关——一个永远不改变行集的
|
|
17
|
+
* 控件只会让人怀疑自己看漏了什么。`ExperimentTable` 与内建默认报告共用同一条判据。
|
|
18
|
+
*/
|
|
19
|
+
export declare function hasHistoricalOrStale(items: readonly ExperimentListItem[]): boolean;
|
|
14
20
|
/** 已选出的 AttemptHandle[] → 列表行;顺序保持传入顺序(不再次按 Sample 去重)。 */
|
|
15
21
|
export declare function attemptRowsOf(attempts: readonly AttemptHandle[]): Promise<AttemptListItem[]>;
|
|
16
22
|
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Sample 展平顺序(不重排)。 */
|
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
|
|
9
9
|
import { comparabilityConfigOf, deepEqualJson } from "../../../sample/index.js";
|
|
10
10
|
import { foldEvalVerdict } from "../../../shared/verdict.js";
|
|
11
|
-
import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems, locatorOf, resolveInput, } from "../../model/aggregate.js";
|
|
11
|
+
import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems, historicalOf, locatorOf, msSince, resolveInput, staleReferenceOf, } from "../../model/aggregate.js";
|
|
12
12
|
import { attemptCostUSD, costUSD, durationMs, examScore, passRate, tokens, totalScore } from "../../model/metrics.js";
|
|
13
13
|
import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../../../assertions/display.js";
|
|
14
14
|
import { firstLine } from "../../../util.js";
|
|
15
|
-
import { summarizeItems } from "../shared-compute.js";
|
|
15
|
+
import { selectedAttemptsOnly, summarizeItems } from "../shared-compute.js";
|
|
16
16
|
/**
|
|
17
17
|
* 一次 attempt 的单行结果摘要(断言摘要契约):failed 取主失败断言摘要(不含
|
|
18
18
|
* "+N more",N 单独进 moreFailures),errored 取结构化 error 的一层摘要
|
|
@@ -47,6 +47,11 @@ export function failureSummaryOf(result) {
|
|
|
47
47
|
async function attemptListItemOf(item) {
|
|
48
48
|
const result = item.attempt.result;
|
|
49
49
|
const { summary, more } = failureSummaryOf(result);
|
|
50
|
+
const historical = historicalOf(item);
|
|
51
|
+
// 缺 startedAt(legacy / 第三方落盘)时退化到所属快照的 startedAt——时效标注宁可粗一档
|
|
52
|
+
// 时距,不留空字段(与 dedupeAttempts「缺才不去重」同一条「不伪造」纪律,这里伪造的只是
|
|
53
|
+
// 展示粒度,不影响身份判定)。
|
|
54
|
+
const startedAt = result.startedAt ?? item.run.startedAt;
|
|
50
55
|
return {
|
|
51
56
|
experimentId: experimentIdOf(item),
|
|
52
57
|
evalId: evalIdOf(item),
|
|
@@ -58,13 +63,23 @@ async function attemptListItemOf(item) {
|
|
|
58
63
|
moreFailures: more,
|
|
59
64
|
examScore: await computeCell(examScore, [item]),
|
|
60
65
|
totalScore: await computeCell(totalScore, [item]),
|
|
61
|
-
tokens: await computeCell(tokens, [item]),
|
|
62
66
|
durationMs: result.durationMs,
|
|
63
67
|
costUSD: attemptCostUSD(result),
|
|
64
|
-
startedAt
|
|
68
|
+
startedAt,
|
|
69
|
+
historical,
|
|
70
|
+
...(historical ? { staleSinceMs: msSince(startedAt) } : {}),
|
|
65
71
|
locator: locatorOf(item),
|
|
66
72
|
};
|
|
67
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* 「只看新执行」开关在场的判据(docs/feature/reports/components/summaries/experiment-table.md
|
|
76
|
+
* 「只看新执行」):Sample 里既没有历史执行也没有过期结论时不画开关——一个永远不改变行集的
|
|
77
|
+
* 控件只会让人怀疑自己看漏了什么。`ExperimentTable` 与内建默认报告共用同一条判据。
|
|
78
|
+
*/
|
|
79
|
+
export function hasHistoricalOrStale(items) {
|
|
80
|
+
return items.some((item) => Object.keys(item.staleReferences).length > 0 ||
|
|
81
|
+
item.evalRows.some((row) => row.attempts.some((attempt) => attempt.historical)));
|
|
82
|
+
}
|
|
68
83
|
/** 已选出的 AttemptHandle[] → 列表行;顺序保持传入顺序(不再次按 Sample 去重)。 */
|
|
69
84
|
export async function attemptRowsOf(attempts) {
|
|
70
85
|
return Promise.all(attempts.map((attempt) => attemptListItemOf({ attempt, run: attempt.run, watermark: attempt.run })));
|
|
@@ -155,6 +170,38 @@ function byMetricDescThenId(valueOf) {
|
|
|
155
170
|
return vb - va || a.experimentId.localeCompare(b.experimentId);
|
|
156
171
|
};
|
|
157
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* 覆盖缺口两档占位的「过期结论」参考(docs/feature/reports/components/summaries/experiment-table.md
|
|
175
|
+
* 「覆盖缺口的两档占位行」):`missingEvalIds` 里、`historyAttempts` 中存在与 `anchorConfigHash`
|
|
176
|
+
* 不可比判定的题,取其中最近一条。`fresh` 为 true 时整份不给参考——读者已声明只看新执行,
|
|
177
|
+
* 占位行就不再把被排除的历史结论请回来。
|
|
178
|
+
*/
|
|
179
|
+
function staleReferencesFor(experimentId, missingEvalIds, historyAttempts, anchorConfigHash, fresh) {
|
|
180
|
+
if (fresh || missingEvalIds.length === 0)
|
|
181
|
+
return {};
|
|
182
|
+
const missing = new Set(missingEvalIds);
|
|
183
|
+
const candidatesByEval = new Map();
|
|
184
|
+
for (const attempt of historyAttempts) {
|
|
185
|
+
if (attempt.experimentId !== experimentId)
|
|
186
|
+
continue;
|
|
187
|
+
if (!missing.has(attempt.evalId))
|
|
188
|
+
continue;
|
|
189
|
+
if (attempt.run.configHash === anchorConfigHash)
|
|
190
|
+
continue; // 可比,不是「过期结论」候选
|
|
191
|
+
const list = candidatesByEval.get(attempt.evalId);
|
|
192
|
+
if (list)
|
|
193
|
+
list.push(attempt);
|
|
194
|
+
else
|
|
195
|
+
candidatesByEval.set(attempt.evalId, [attempt]);
|
|
196
|
+
}
|
|
197
|
+
const out = {};
|
|
198
|
+
for (const [evalId, candidates] of candidatesByEval) {
|
|
199
|
+
const reference = staleReferenceOf(candidates);
|
|
200
|
+
if (reference)
|
|
201
|
+
out[evalId] = reference;
|
|
202
|
+
}
|
|
203
|
+
return out;
|
|
204
|
+
}
|
|
158
205
|
/**
|
|
159
206
|
* `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始排序按这份列表
|
|
160
207
|
* 自身的题型构成选择主读数——纯通过制沿用端到端通过率降序,纯计分制改按总分降序(缺数据
|
|
@@ -165,7 +212,7 @@ function byMetricDescThenId(valueOf) {
|
|
|
165
212
|
* 看跨配置演化用 run 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
166
213
|
*/
|
|
167
214
|
export async function experimentListData(input) {
|
|
168
|
-
const { runs, attempts, coverage } = resolveInput(input);
|
|
215
|
+
const { runs, attempts, coverage, historyAttempts, fresh } = resolveInput(input);
|
|
169
216
|
const coverageByExperiment = new Map(coverage.map((c) => [c.experimentId, c]));
|
|
170
217
|
// 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
|
|
171
218
|
const configByExperiment = new Map();
|
|
@@ -182,7 +229,7 @@ export async function experimentListData(input) {
|
|
|
182
229
|
'use the "run" dimension or MetricLine; to show the current level, pass results.current() which selects a single config per experiment.');
|
|
183
230
|
}
|
|
184
231
|
}
|
|
185
|
-
const items = collectItems(runs, attempts);
|
|
232
|
+
const items = collectItems(runs, selectedAttemptsOnly(attempts));
|
|
186
233
|
const groups = groupItems(items, "experiment");
|
|
187
234
|
const out = [];
|
|
188
235
|
for (const [experimentId, group] of groups) {
|
|
@@ -213,7 +260,7 @@ export async function experimentListData(input) {
|
|
|
213
260
|
}
|
|
214
261
|
const experiment = newest.run.experiment ?? newest.attempt.result.experiment;
|
|
215
262
|
const model = newest.attempt.result.model ?? newest.run.model;
|
|
216
|
-
const
|
|
263
|
+
const missingEvalIds = coverageByExperiment.get(experimentId)?.missingEvalIds ?? [];
|
|
217
264
|
out.push({
|
|
218
265
|
experimentId,
|
|
219
266
|
agent: newest.run.agent || newest.attempt.result.agent,
|
|
@@ -228,14 +275,15 @@ export async function experimentListData(input) {
|
|
|
228
275
|
tokens: await computeCell(tokens, group),
|
|
229
276
|
evals: stats.evals,
|
|
230
277
|
attempts: stats.attempts,
|
|
231
|
-
|
|
232
|
-
|
|
278
|
+
historicalAttempts: group.filter(historicalOf).length,
|
|
279
|
+
missingEvalIds,
|
|
280
|
+
staleReferences: staleReferencesFor(experimentId, missingEvalIds, historyAttempts, coverageByExperiment.get(experimentId)?.run.configHash, fresh),
|
|
233
281
|
lastRunAt: stats.lastRunAt,
|
|
234
282
|
evalRows,
|
|
235
283
|
});
|
|
236
284
|
}
|
|
237
|
-
// coverage 不是 attempt
|
|
238
|
-
//
|
|
285
|
+
// coverage 不是 attempt 的附属品:--fresh 和 current() 都可能让一个实验当前口径下
|
|
286
|
+
// 零 attempt。仍然给它一行,让 missingEvalIds 的占位题可达,不能把整实验静默吞掉。
|
|
239
287
|
for (const coverageEntry of coverage) {
|
|
240
288
|
if (groups.has(coverageEntry.experimentId))
|
|
241
289
|
continue;
|
|
@@ -259,8 +307,9 @@ export async function experimentListData(input) {
|
|
|
259
307
|
tokens: await computeCell(tokens, emptyItems),
|
|
260
308
|
evals: 0,
|
|
261
309
|
attempts: 0,
|
|
262
|
-
|
|
263
|
-
|
|
310
|
+
historicalAttempts: 0,
|
|
311
|
+
missingEvalIds: coverageEntry.missingEvalIds,
|
|
312
|
+
staleReferences: staleReferencesFor(coverageEntry.experimentId, coverageEntry.missingEvalIds, historyAttempts, anchor.configHash, fresh),
|
|
264
313
|
lastRunAt: anchor.startedAt,
|
|
265
314
|
evalRows: [],
|
|
266
315
|
});
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { TableContent } from "../../definition/cell.ts";
|
|
2
2
|
import type { AttemptListItem, EvalListItem, ExperimentListItem } from "../../model/types.ts";
|
|
3
|
+
/** 覆盖构成副行的 key 前缀;测试与消费方靠它把这一行从 Eval / 组行里筛出去。 */
|
|
4
|
+
export declare const COVERAGE_ROW_PREFIX = "coverage:";
|
|
3
5
|
export declare function experimentListContent(items: readonly ExperimentListItem[]): TableContent;
|
|
4
6
|
export declare function evalListContent(items: readonly EvalListItem[]): TableContent;
|
|
5
7
|
export declare function attemptListContent(items: readonly AttemptListItem[]): TableContent;
|