niceeval 0.12.0 → 0.12.1-canary.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/i18n/en.d.ts +19 -0
- package/dist/i18n/en.js +34 -7
- package/dist/i18n/zh-CN.d.ts +20 -1
- package/dist/i18n/zh-CN.js +32 -5
- package/dist/record/open.js +2 -2
- package/dist/record/types.d.ts +16 -7
- package/dist/report/built-in/result-components.js +8 -22
- package/dist/report/components/entity-lists/compute.d.ts +0 -6
- package/dist/report/components/entity-lists/compute.js +13 -62
- package/dist/report/components/entity-lists/content.d.ts +0 -2
- package/dist/report/components/entity-lists/content.js +26 -60
- package/dist/report/components/entity-lists/index.d.ts +1 -21
- package/dist/report/components/entity-lists/index.js +6 -38
- package/dist/report/components/entity-lists/validate.js +18 -22
- package/dist/report/components/experiment-detail/compute.js +1 -1
- package/dist/report/components/fixtures.js +7 -14
- package/dist/report/components/shared-compute.d.ts +0 -12
- package/dist/report/components/shared-compute.js +2 -22
- package/dist/report/components/site-components/HeroCard.d.ts +2 -2
- package/dist/report/components/site-components/HeroCard.js +4 -28
- package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
- package/dist/report/components/site-components/scope-warnings.js +1 -1
- package/dist/report/components/summaries/compute.js +2 -2
- package/dist/report/components/summaries/index.js +15 -26
- package/dist/report/definition/cell.d.ts +4 -15
- package/dist/report/definition/cell.js +13 -20
- package/dist/report/definition/primitives.js +13 -28
- package/dist/report/index.d.ts +1 -1
- package/dist/report/model/aggregate.d.ts +4 -22
- package/dist/report/model/aggregate.js +1 -35
- package/dist/report/model/calculation.js +5 -1
- package/dist/report/model/evaluation-kind.js +1 -2
- package/dist/report/model/flag.js +0 -2
- package/dist/report/model/format.d.ts +2 -2
- package/dist/report/model/format.js +4 -3
- package/dist/report/model/locale.d.ts +8 -15
- package/dist/report/model/locale.js +13 -25
- package/dist/report/model/metrics.js +9 -6
- package/dist/report/model/types.d.ts +11 -36
- package/dist/report/react/index.d.ts +1 -1
- package/dist/report/slices/compute.js +3 -55
- package/dist/report/slices/content.js +8 -15
- package/dist/report/slices/validate.js +2 -27
- package/dist/report/tasks.d.ts +0 -5
- package/dist/report/tasks.js +2 -3
- package/dist/runner/eval-selection.d.ts +0 -5
- package/dist/runner/fingerprint.d.ts +1 -1
- package/dist/runner/session.d.ts +8 -8
- package/dist/runner/types.d.ts +16 -7
- package/dist/runner/types.js +2 -0
- package/dist/sample/index.d.ts +4 -13
- package/dist/sample/index.js +57 -72
- package/docs-site/zh/examples/index.mdx +7 -0
- package/docs-site/zh/reference/cli.mdx +2 -3
- package/docs-site/zh/reference/results-data.mdx +7 -4
- package/docs-site/zh/tutorials/agent-onboarding.mdx +1 -1
- package/docs-site/zh/tutorials/authoring.mdx +3 -3
- package/docs-site/zh/tutorials/ci-integration.mdx +1 -1
- package/docs-site/zh/tutorials/custom-reports.mdx +2 -1
- package/docs-site/zh/tutorials/experiments.mdx +2 -2
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +4 -4
- package/docs-site/zh/tutorials/viewing-results.mdx +5 -3
- package/package.json +2 -2
- package/src/cli-exp-rename.test.ts +271 -0
- package/src/cli.ts +296 -39
- package/src/i18n/en.ts +43 -7
- package/src/i18n/zh-CN.ts +40 -5
- package/src/record/evidence-schema.test.ts +1 -1
- package/src/record/host-equivalence.test.ts +36 -48
- package/src/record/index.ts +1 -0
- package/src/record/open.ts +2 -2
- package/src/record/results.test.ts +6 -6
- package/src/record/types.ts +17 -7
- package/src/record/writer.ts +1 -25
- package/src/report/assets/styles.css +1 -68
- package/src/report/built-in/result-components.tsx +1 -33
- package/src/report/components/compute.test.ts +138 -193
- package/src/report/components/entity-lists/compute.ts +13 -82
- package/src/report/components/entity-lists/content.test.ts +70 -34
- package/src/report/components/entity-lists/content.ts +32 -65
- package/src/report/components/entity-lists/index.tsx +7 -91
- package/src/report/components/entity-lists/validate.test.ts +18 -17
- package/src/report/components/entity-lists/validate.ts +15 -15
- package/src/report/components/experiment-detail/compute.ts +1 -1
- package/src/report/components/experiment-detail/experiment-detail.test.tsx +7 -2
- package/src/report/components/fixtures.ts +7 -14
- package/src/report/components/shared-compute.ts +2 -22
- package/src/report/components/site-components/HeroCard.tsx +5 -32
- package/src/report/components/site-components/scope-warnings.ts +1 -1
- package/src/report/components/summaries/compute.ts +2 -3
- package/src/report/components/summaries/index.tsx +1 -18
- package/src/report/definition/cell.ts +15 -35
- package/src/report/definition/primitives.tsx +18 -61
- package/src/report/definition/table.test.tsx +56 -0
- package/src/report/index.ts +0 -1
- package/src/report/model/aggregate.ts +4 -43
- package/src/report/model/calculation.test.ts +56 -7
- package/src/report/model/calculation.ts +7 -1
- package/src/report/model/evaluation-kind.ts +1 -2
- package/src/report/model/flag.ts +0 -2
- package/src/report/model/format.ts +4 -3
- package/src/report/model/locale.ts +13 -25
- package/src/report/model/metrics.ts +11 -6
- package/src/report/model/types.ts +11 -37
- package/src/report/react/index.tsx +0 -1
- package/src/report/runtime/dual-render.test.tsx +6 -42
- package/src/report/slices/compute.ts +2 -61
- package/src/report/slices/content.ts +7 -15
- package/src/report/slices/delta-table.test.ts +33 -101
- package/src/report/slices/validate.test.ts +4 -26
- package/src/report/slices/validate.ts +2 -19
- package/src/report/tasks.ts +2 -8
- package/src/runner/accept.test.ts +27 -2
- package/src/runner/accept.ts +34 -3
- package/src/runner/attempt.ts +1 -2
- package/src/runner/config-identity.test.ts +1 -4
- package/src/runner/eval-selection.ts +0 -15
- package/src/runner/experiment-labels.test.ts +0 -1
- package/src/runner/feedback/human.test.ts +23 -23
- package/src/runner/feedback/human.ts +13 -11
- package/src/runner/feedback/json.ts +1 -1
- package/src/runner/fingerprint.test.ts +6 -9
- package/src/runner/fingerprint.ts +2 -2
- package/src/runner/gate-lease.test.ts +10 -10
- package/src/runner/gate-lease.ts +7 -7
- package/src/runner/lock.test.ts +8 -8
- package/src/runner/lock.ts +4 -4
- package/src/runner/rename-experiment.test.ts +241 -0
- package/src/runner/rename-experiment.ts +489 -0
- package/src/runner/reporters/artifacts.ts +1 -17
- package/src/runner/reporters/braintrust.test.ts +1 -1
- package/src/runner/run.ts +7 -7
- package/src/runner/session.test.ts +15 -13
- package/src/runner/session.ts +24 -24
- package/src/runner/teardown-registry.test.ts +9 -9
- package/src/runner/teardown-registry.ts +4 -4
- package/src/runner/types.ts +17 -7
- package/src/sample/index.ts +63 -84
- package/src/sample/sample.test.ts +107 -89
- package/src/show/index.test.ts +0 -1
- package/src/show/index.ts +18 -24
- package/src/show/json.test.ts +1 -8
- package/src/show/json.ts +0 -3
- package/src/view/data.test.ts +0 -1
- package/src/view/data.ts +2 -5
- package/src/view/server-rebuild.test.ts +0 -1
- package/src/view/site-param-pages.test.ts +0 -1
package/dist/i18n/en.d.ts
CHANGED
|
@@ -82,6 +82,25 @@ export declare const en: {
|
|
|
82
82
|
"cli.accept.flagUnsupported": string;
|
|
83
83
|
"cli.accept.failed": string;
|
|
84
84
|
"cli.accept.done": string;
|
|
85
|
+
"cli.rename.usage": string;
|
|
86
|
+
"cli.rename.flagUnsupported": string;
|
|
87
|
+
"cli.rename.previewHeader": string;
|
|
88
|
+
"cli.rename.blocked": string;
|
|
89
|
+
"cli.rename.migratingHeader": string;
|
|
90
|
+
"cli.rename.migratingRow": string;
|
|
91
|
+
"cli.rename.excludedHeader": string;
|
|
92
|
+
"cli.rename.excludedRow": string;
|
|
93
|
+
"cli.rename.doneHeader": string;
|
|
94
|
+
"cli.rename.snapshotPath": string;
|
|
95
|
+
"cli.rename.doneRow": string;
|
|
96
|
+
"cli.rename.error.sourceEmpty": string;
|
|
97
|
+
"cli.rename.error.targetNotFound": string;
|
|
98
|
+
"cli.rename.error.targetHasResults": string;
|
|
99
|
+
"cli.rename.error.sourceUnreadable": string;
|
|
100
|
+
"cli.rename.error.artifactUnavailable": string;
|
|
101
|
+
"cli.rename.error.nothingToMigrate": string;
|
|
102
|
+
"cli.rename.conflicting": string;
|
|
103
|
+
"cli.rename.failed": string;
|
|
85
104
|
"cli.error": string;
|
|
86
105
|
"cli.flag.acceptNeedsSelector": string;
|
|
87
106
|
"cli.flag.acceptWithRerunAll": string;
|
package/dist/i18n/en.js
CHANGED
|
@@ -79,7 +79,7 @@ export const en = {
|
|
|
79
79
|
"cli.dry.unit.configs": "configs",
|
|
80
80
|
"cli.dry.affects": "affects {{evals}} · {{ids}}",
|
|
81
81
|
"cli.dry.acceptHint": "accept: {{command}}",
|
|
82
|
-
"cli.accept.choiceHeader": "
|
|
82
|
+
"cli.accept.choiceHeader": "previous-result {{selector}}{{change}} ({{evals}} evals)\n",
|
|
83
83
|
"cli.accept.prompt": " reuse these results? [y/N] ",
|
|
84
84
|
"cli.accept.nothingToAccept": "No difference in this plan can be accepted (nothing is blocked by the fingerprint gate).\n" +
|
|
85
85
|
"Running as planned.\n",
|
|
@@ -91,9 +91,37 @@ export const en = {
|
|
|
91
91
|
" fix: pass only @<locator> (and optionally --record <dir>)\n",
|
|
92
92
|
"cli.accept.failed": "error: could not accept result: {{error}}\n",
|
|
93
93
|
"cli.accept.done": "Accepted {{sourceLocator}}. New result locator: {{locator}}. Current fingerprint: {{fingerprint}}\n",
|
|
94
|
+
"cli.rename.usage": "error: niceeval exp rename expects exactly two arguments: an old id and a new id\n" +
|
|
95
|
+
" fix: niceeval exp rename <oldId> <newId> [--dry] [--json]\n",
|
|
96
|
+
"cli.rename.flagUnsupported": "error: {{flag}} is not valid with niceeval exp rename\n" +
|
|
97
|
+
" fix: pass only <oldId> <newId>, optionally --dry / --json\n",
|
|
98
|
+
"cli.rename.previewHeader": "exp rename preview: {{oldId}} -> {{newId}}\n",
|
|
99
|
+
"cli.rename.blocked": " blocked (nothing will be written): {{reason}}\n",
|
|
100
|
+
"cli.rename.migratingHeader": " {{count}} terminal results will migrate:\n",
|
|
101
|
+
"cli.rename.migratingRow": " {{evalId}} {{sourceLocator}} -> {{newId}}\n",
|
|
102
|
+
"cli.rename.excludedHeader": " {{count}} excluded (not migrated, does not block):\n",
|
|
103
|
+
"cli.rename.excludedRow": " {{evalId}} {{reason}}\n",
|
|
104
|
+
"cli.rename.doneHeader": "exp rename done: rebound {{count}} terminal results from {{oldId}} to {{newId}}.\n",
|
|
105
|
+
"cli.rename.snapshotPath": " new snapshot: {{path}}\n",
|
|
106
|
+
"cli.rename.doneRow": " {{evalId}} {{sourceLocator}} -> {{locator}}\n",
|
|
107
|
+
"cli.rename.error.sourceEmpty": "error: {{oldId}} has no readable terminal history to migrate to {{newId}}.\n" +
|
|
108
|
+
" fix: restore and verify {{oldId}}'s real results before retrying; with no old results, run `niceeval exp {{newId}}` and do not rename.\n" +
|
|
109
|
+
" exp rename does not move experiment source, nor delete or rewrite the old result tree.\n",
|
|
110
|
+
"cli.rename.error.targetNotFound": "error: new id \"{{newId}}\" is not discovered under this project's experiments/.\n" +
|
|
111
|
+
" fix: create or rename the experiment in experiments/ first (e.g. `git mv experiments/{{oldId}}.ts experiments/{{newId}}.ts`), then rerun.\n",
|
|
112
|
+
"cli.rename.error.targetHasResults": "error: {{newId}} already has terminal results for these evals; rename never overwrites existing results.\n" +
|
|
113
|
+
" fix: keep the target results, or explicitly clean the target history and re-preview; the command deletes nothing itself.\n",
|
|
114
|
+
"cli.rename.error.sourceUnreadable": "error: the Record for {{oldId}} is unreadable; cannot migrate to {{newId}}.\n" +
|
|
115
|
+
" fix: view this record with a niceeval version that reads its schemaVersion.\n",
|
|
116
|
+
"cli.rename.error.artifactUnavailable": "error: source evidence cannot be preserved ({{evalId}}); nothing will be written.\n" +
|
|
117
|
+
" fix: make the artifact reference and source locator readable, or rerun this eval.\n",
|
|
118
|
+
"cli.rename.error.nothingToMigrate": "error: nothing to migrate under {{oldId}}: no terminal passed/failed still selected by {{newId}}, or all excluded.\n" +
|
|
119
|
+
" fix: check that {{newId}}'s evals selector covers the old experiment's results.\n",
|
|
120
|
+
"cli.rename.conflicting": " conflicting evals: {{evals}}\n",
|
|
121
|
+
"cli.rename.failed": "error: exp rename failed: {{error}}\n",
|
|
94
122
|
"cli.error": "niceeval error: {{error}}\n",
|
|
95
123
|
"cli.flag.acceptNeedsSelector": "error: --accept needs a selector, for example --accept config:judge.model\n" +
|
|
96
|
-
" fix: run `niceeval exp <selection> --dry` first; every `
|
|
124
|
+
" fix: run `niceeval exp <selection> --dry` first; every `previous-result` line prints the selectors it can accept, copy one verbatim\n" +
|
|
97
125
|
" differences you can accept in this plan: {{available}}\n",
|
|
98
126
|
"cli.flag.acceptWithRerunAll": "--accept cannot be combined with --rerun all: one says trust nothing from cache, the other says trust this difference anyway.\n" +
|
|
99
127
|
"Drop --rerun all to accept the difference, or drop --accept to rerun everything.\n",
|
|
@@ -108,8 +136,8 @@ export const en = {
|
|
|
108
136
|
"runner.setupReturnedCleanup": "{{layer}} returned a function. setup does not carry cleanup and the returned value will not be executed — put the cleanup in the paired teardown of the same layer ({{hint}}); see the experiments tutorial on docs-site or docs/runner.md.\n",
|
|
109
137
|
"runner.experimentTeardownLate": "experiment {{experimentId}}'s teardown was not triggered by the normal countdown path; it has been executed by the end-of-run sweep instead. Record are unaffected; seeing this line means an unlocated intermittent scheduling issue fired — please record this run in the memory ledger.\n",
|
|
110
138
|
"runner.teardownRegistrationWriteFailed": "writing the crash-recovery teardown registration for experiment {{experimentId}} failed: {{message}}. The run continues normally, but a SIGKILL during this run cannot be recovered via `niceeval exp --teardown` or the startup self-heal — check disk space/permissions under .niceeval/teardowns/.\n",
|
|
111
|
-
"runner.lockTakenOver": "took over an expired case lock for {{experimentId}}/{{evalId}} (previously held by pid {{pid}} on {{host}}; its heartbeat
|
|
112
|
-
"runner.gateLeaseTakenOver": "took over an expired concurrency-slot lease for experiment {{experimentId}} (slot {{slot}}, previously held by pid {{pid}} on {{host}}; its heartbeat
|
|
139
|
+
"runner.lockTakenOver": "took over an expired case lock for {{experimentId}}/{{evalId}} (previously held by pid {{pid}} on {{host}}; its heartbeat expired) — that run likely died without releasing it; this run now owns dispatching this case.\n",
|
|
140
|
+
"runner.gateLeaseTakenOver": "took over an expired concurrency-slot lease for experiment {{experimentId}} (slot {{slot}}, previously held by pid {{pid}} on {{host}}; its heartbeat expired) — that run likely died without releasing it; this run now owns the slot.\n",
|
|
113
141
|
"runner.gateLeaseWaiting": "waiting on another run for experiment {{experimentId}}'s concurrency slots: all {{effectiveN}} in use ({{holders}}). Concurrent runs share this experiment's slots, and the smallest maxConcurrency in play wins — this run declared {{declaredN}}. Nothing dispatches until a slot frees up; the other run's slots release when its attempts finish, or 30s after it dies.\n",
|
|
114
142
|
"runner.dispatchHaltedExperiment": "experiment halted (dispatch-halted): {{message}}\n",
|
|
115
143
|
"runner.dispatchHaltedEval": "eval halted: {{message}}\n",
|
|
@@ -127,6 +155,8 @@ export const en = {
|
|
|
127
155
|
"Usage:\n" +
|
|
128
156
|
" niceeval exp [path|experiment] [eval-id-prefix…] run experiments\n" +
|
|
129
157
|
" niceeval exp list [experiment-prefix] list runnable configs (no dispatch)\n" +
|
|
158
|
+
" niceeval exp rename <oldId> <newId> rebind terminal results from old to new id\n" +
|
|
159
|
+
" --dry preview without writing; --json one JSON document\n" +
|
|
130
160
|
" --teardown recover a killed run: run only the selected experiments'\n" +
|
|
131
161
|
" teardown (no attempts, no setup); combining it with eval id prefixes is an error\n" +
|
|
132
162
|
" niceeval accept @<locator>... accept explicit historical results\n" +
|
|
@@ -156,8 +186,6 @@ export const en = {
|
|
|
156
186
|
" --record <dir> pin a record root --exp <id> repeatable; 2+ compares conditions\n" +
|
|
157
187
|
" --report <file> custom report --page <id> pick the initial page (multi-page\n" +
|
|
158
188
|
" reports render it, then list the rest as a page index with copyable commands)\n" +
|
|
159
|
-
" --fresh only count freshly executed attempts (excludes carried-over and\n" +
|
|
160
|
-
" historical stitched-in attempts); excluded evals show up as placeholder rows\n" +
|
|
161
189
|
" niceeval list list discovered evals\n" +
|
|
162
190
|
" niceeval session list [--all] [experiment-prefix] query Sessions (read-only)\n" +
|
|
163
191
|
" niceeval session show <sessionId> show one Session (read-only)\n" +
|
|
@@ -166,7 +194,6 @@ export const en = {
|
|
|
166
194
|
" (same file as show); --page <id> picks the initial page;\n" +
|
|
167
195
|
" --record <dir> pins a record root; --run <file> opens exactly\n" +
|
|
168
196
|
" one run; --exp <id> (repeatable) narrows to those experiments;\n" +
|
|
169
|
-
" --fresh only new executions\n" +
|
|
170
197
|
" --out <dir> exports a static site: index.html plus the viewer\n" +
|
|
171
198
|
" artifacts, ready for any static host\n" +
|
|
172
199
|
" niceeval sandbox list|enter|history|diff|stop inspect & destroy sandboxes kept by --keep-sandbox\n" +
|
package/dist/i18n/zh-CN.d.ts
CHANGED
|
@@ -73,7 +73,7 @@ export declare const zhCN: {
|
|
|
73
73
|
readonly "cli.dry.unit.configs": "个运行配置";
|
|
74
74
|
readonly "cli.dry.affects": "影响 {{evals}} · {{ids}}";
|
|
75
75
|
readonly "cli.dry.acceptHint": "accept: {{command}}";
|
|
76
|
-
readonly "cli.accept.choiceHeader": "
|
|
76
|
+
readonly "cli.accept.choiceHeader": "previous-result {{selector}}{{change}} ({{evals}} 条 eval)\n";
|
|
77
77
|
readonly "cli.accept.prompt": " 复用这些结果吗? [y/N] ";
|
|
78
78
|
readonly "cli.accept.nothingToAccept": string;
|
|
79
79
|
readonly "cli.accept.equivalent": "等价命令: {{command}}\n";
|
|
@@ -82,6 +82,25 @@ export declare const zhCN: {
|
|
|
82
82
|
readonly "cli.accept.flagUnsupported": string;
|
|
83
83
|
readonly "cli.accept.failed": "error: 接受结果失败:{{error}}\n";
|
|
84
84
|
readonly "cli.accept.done": "已接受 {{sourceLocator}}。新结果 locator: {{locator}}。当前指纹:{{fingerprint}}\n";
|
|
85
|
+
readonly "cli.rename.usage": string;
|
|
86
|
+
readonly "cli.rename.flagUnsupported": string;
|
|
87
|
+
readonly "cli.rename.previewHeader": "exp rename 预览:{{oldId}} → {{newId}}\n";
|
|
88
|
+
readonly "cli.rename.blocked": " 整批阻断,零写入:{{reason}}\n";
|
|
89
|
+
readonly "cli.rename.migratingHeader": " 将迁移 {{count}} 条终态结果:\n";
|
|
90
|
+
readonly "cli.rename.migratingRow": " {{evalId}} {{sourceLocator}} → {{newId}}\n";
|
|
91
|
+
readonly "cli.rename.excludedHeader": " 排除 {{count}} 条(不迁移,不阻断):\n";
|
|
92
|
+
readonly "cli.rename.excludedRow": " {{evalId}} {{reason}}\n";
|
|
93
|
+
readonly "cli.rename.doneHeader": "exp rename 完成:把 {{oldId}} 的 {{count}} 条终态结果重绑到 {{newId}}。\n";
|
|
94
|
+
readonly "cli.rename.snapshotPath": " 新 snapshot:{{path}}\n";
|
|
95
|
+
readonly "cli.rename.doneRow": " {{evalId}} {{sourceLocator}} → {{locator}}\n";
|
|
96
|
+
readonly "cli.rename.error.sourceEmpty": string;
|
|
97
|
+
readonly "cli.rename.error.targetNotFound": string;
|
|
98
|
+
readonly "cli.rename.error.targetHasResults": string;
|
|
99
|
+
readonly "cli.rename.error.sourceUnreadable": string;
|
|
100
|
+
readonly "cli.rename.error.artifactUnavailable": string;
|
|
101
|
+
readonly "cli.rename.error.nothingToMigrate": string;
|
|
102
|
+
readonly "cli.rename.conflicting": " 冲突 eval:{{evals}}\n";
|
|
103
|
+
readonly "cli.rename.failed": "error: exp rename 失败:{{error}}\n";
|
|
85
104
|
readonly "cli.error": "niceeval 出错:{{error}}\n";
|
|
86
105
|
readonly "cli.flag.acceptNeedsSelector": string;
|
|
87
106
|
readonly "cli.flag.acceptWithRerunAll": string;
|
package/dist/i18n/zh-CN.js
CHANGED
|
@@ -79,7 +79,7 @@ export const zhCN = {
|
|
|
79
79
|
"cli.dry.unit.configs": "个运行配置",
|
|
80
80
|
"cli.dry.affects": "影响 {{evals}} · {{ids}}",
|
|
81
81
|
"cli.dry.acceptHint": "accept: {{command}}",
|
|
82
|
-
"cli.accept.choiceHeader": "
|
|
82
|
+
"cli.accept.choiceHeader": "previous-result {{selector}}{{change}} ({{evals}} 条 eval)\n",
|
|
83
83
|
"cli.accept.prompt": " 复用这些结果吗? [y/N] ",
|
|
84
84
|
"cli.accept.nothingToAccept": "本次计划里没有可授权的差异(没有条目卡在指纹门上)。\n" +
|
|
85
85
|
"按原计划运行。\n",
|
|
@@ -91,9 +91,37 @@ export const zhCN = {
|
|
|
91
91
|
" fix: 只传 @<locator>(可选 `--record <目录>`)\n",
|
|
92
92
|
"cli.accept.failed": "error: 接受结果失败:{{error}}\n",
|
|
93
93
|
"cli.accept.done": "已接受 {{sourceLocator}}。新结果 locator: {{locator}}。当前指纹:{{fingerprint}}\n",
|
|
94
|
+
"cli.rename.usage": "error: niceeval exp rename 需要恰好两个参数:旧 id 和新 id\n" +
|
|
95
|
+
" fix: niceeval exp rename <oldId> <newId> [--dry] [--json]\n",
|
|
96
|
+
"cli.rename.flagUnsupported": "error: {{flag}} 不能用于 niceeval exp rename\n" +
|
|
97
|
+
" fix: 只传 <oldId> <newId>,可选 --dry / --json\n",
|
|
98
|
+
"cli.rename.previewHeader": "exp rename 预览:{{oldId}} → {{newId}}\n",
|
|
99
|
+
"cli.rename.blocked": " 整批阻断,零写入:{{reason}}\n",
|
|
100
|
+
"cli.rename.migratingHeader": " 将迁移 {{count}} 条终态结果:\n",
|
|
101
|
+
"cli.rename.migratingRow": " {{evalId}} {{sourceLocator}} → {{newId}}\n",
|
|
102
|
+
"cli.rename.excludedHeader": " 排除 {{count}} 条(不迁移,不阻断):\n",
|
|
103
|
+
"cli.rename.excludedRow": " {{evalId}} {{reason}}\n",
|
|
104
|
+
"cli.rename.doneHeader": "exp rename 完成:把 {{oldId}} 的 {{count}} 条终态结果重绑到 {{newId}}。\n",
|
|
105
|
+
"cli.rename.snapshotPath": " 新 snapshot:{{path}}\n",
|
|
106
|
+
"cli.rename.doneRow": " {{evalId}} {{sourceLocator}} → {{locator}}\n",
|
|
107
|
+
"cli.rename.error.sourceEmpty": "error: {{oldId}} 下没有可读的终态历史,无法迁移到 {{newId}}。\n" +
|
|
108
|
+
" fix: 恢复并确认 {{oldId}} 的真实结果后重试;没有旧结果时直接运行 `niceeval exp {{newId}}`,不要再执行 rename。\n" +
|
|
109
|
+
" exp rename 不移动实验源码,也不删除或改写旧结果树。\n",
|
|
110
|
+
"cli.rename.error.targetNotFound": "error: 新 id \"{{newId}}\" 没有被当前项目的 experiments/ 发现。\n" +
|
|
111
|
+
" fix: 先在 experiments/ 创建或改名出新实验(如 `git mv experiments/{{oldId}}.ts experiments/{{newId}}.ts`),再重跑本命令。\n",
|
|
112
|
+
"cli.rename.error.targetHasResults": "error: {{newId}} 下已有这些 eval 的终态结果,改名不覆盖已有结果。\n" +
|
|
113
|
+
" fix: 保留目标结果继续用,或显式清理目标历史后重新预览;命令自身不删除数据。\n",
|
|
114
|
+
"cli.rename.error.sourceUnreadable": "error: {{oldId}} 的 Record 读不动,无法迁移到 {{newId}}。\n" +
|
|
115
|
+
" fix: 用能读该 schemaVersion 的 niceeval 版本查看这份记录。\n",
|
|
116
|
+
"cli.rename.error.artifactUnavailable": "error: 来源证据无法保留({{evalId}}),整批零写入。\n" +
|
|
117
|
+
" fix: 先确认 artifact 引用与来源 locator 可读,或对这条 eval 重新运行。\n",
|
|
118
|
+
"cli.rename.error.nothingToMigrate": "error: {{oldId}} 下没有可迁移的 eval:没有 {{newId}} 仍选中的终态 passed/failed,或全部被排除。\n" +
|
|
119
|
+
" fix: 检查 {{newId}} 的 evals 选择器是否覆盖旧实验结果。\n",
|
|
120
|
+
"cli.rename.conflicting": " 冲突 eval:{{evals}}\n",
|
|
121
|
+
"cli.rename.failed": "error: exp rename 失败:{{error}}\n",
|
|
94
122
|
"cli.error": "niceeval 出错:{{error}}\n",
|
|
95
123
|
"cli.flag.acceptNeedsSelector": "error: --accept 必须带 selector,例如 --accept config:judge.model\n" +
|
|
96
|
-
" fix: 先跑 `niceeval exp <选择> --dry`,每条 `
|
|
124
|
+
" fix: 先跑 `niceeval exp <选择> --dry`,每条 `previous-result` 行都打出了它可授权的 selector,原样复制一条\n" +
|
|
97
125
|
" 本次计划里可以授权的差异:{{available}}\n",
|
|
98
126
|
"cli.flag.acceptWithRerunAll": "--accept 不能与 --rerun all 同用:一边全不采信缓存,一边又要采信这条差异,方向自相矛盾。\n" +
|
|
99
127
|
"要授权这条差异就去掉 --rerun all;要全量重跑就去掉 --accept。\n",
|
|
@@ -127,6 +155,8 @@ export const zhCN = {
|
|
|
127
155
|
"用法:\n" +
|
|
128
156
|
" niceeval exp [路径|实验] [eval-id 前缀…] 跑实验\n" +
|
|
129
157
|
" niceeval exp list [实验前缀] 列出可运行的实验配置(不派发)\n" +
|
|
158
|
+
" niceeval exp rename <旧 id> <新 id> 把旧实验的终态结果重绑到新 id(显式审计迁移)\n" +
|
|
159
|
+
" --dry 只预览不写盘;--json 输出单份 JSON 文档\n" +
|
|
130
160
|
" --teardown 强杀后补收尾:只对选中的实验各执行一次 teardown(不派发\n" +
|
|
131
161
|
" attempt、不跑 setup);与 eval id 前缀组合是用法错误\n" +
|
|
132
162
|
" niceeval accept @<locator>... 接受明确列出的历史结果\n" +
|
|
@@ -153,8 +183,6 @@ export const zhCN = {
|
|
|
153
183
|
" --record <目录> 钉死记录根 --exp <id> 可重复,两个以上进入对照\n" +
|
|
154
184
|
" --report <文件> 自定义报告 --page <id> 定初始页(多页报告渲染该页,\n" +
|
|
155
185
|
" 尾部再附其余页索引)\n" +
|
|
156
|
-
" --fresh 只统计新执行的 attempt(排除携带条目与跨快照拼入的历史执行);\n" +
|
|
157
|
-
" 被排除的题转成占位行\n" +
|
|
158
186
|
" niceeval list 列出发现到的 eval\n" +
|
|
159
187
|
" niceeval session list [--all] [实验前缀] 查询 Session(只读)\n" +
|
|
160
188
|
" niceeval session show <sessionId> 查看一个 Session(只读)\n" +
|
|
@@ -162,7 +190,6 @@ export const zhCN = {
|
|
|
162
190
|
" 报告页 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
|
|
163
191
|
" --page <id> 定初始页 --record <目录> 钉死记录根\n" +
|
|
164
192
|
" --run <文件> 只打开这一份快照 --exp <id>(可重复)收窄到这些实验;\n" +
|
|
165
|
-
" --fresh 只看新执行\n" +
|
|
166
193
|
" --out <目录> 静态导出:index.html 连同查看器 artifact,可直接静态托管\n" +
|
|
167
194
|
" niceeval sandbox list|enter|history|diff|stop 查看与销毁 --keep-sandbox 留下的现场\n" +
|
|
168
195
|
" niceeval sandbox list --orphans / prune 核对并收回被强杀留下的无主实例\n" +
|
package/dist/record/open.js
CHANGED
|
@@ -30,8 +30,8 @@ const locatorIndexByResults = new WeakMap();
|
|
|
30
30
|
export class LocatorNotFoundError extends Error {
|
|
31
31
|
locator;
|
|
32
32
|
constructor(locator) {
|
|
33
|
-
super(`No attempt found for locator "${locator}" in this results root. It may
|
|
34
|
-
"(the run was deleted, or publish didn't include it) or mistyped.");
|
|
33
|
+
super(`No attempt found for locator "${locator}" in this results root. It may refer to an older record ` +
|
|
34
|
+
"(the run was deleted, or publish didn't include it) or be mistyped.");
|
|
35
35
|
this.locator = locator;
|
|
36
36
|
this.name = "LocatorNotFoundError";
|
|
37
37
|
}
|
package/dist/record/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DiagnosticRecord, EvalResult, ExperimentRunInfo, LocalizedText, SandboxBuildRecord, TimingActivity } from "../types.ts";
|
|
1
|
+
import type { DiagnosticRecord, EvalResult, ExperimentRunInfo, LocalizedText, SandboxBuildRecord, TimingActivity, Verdict } from "../types.ts";
|
|
2
2
|
import type { CommandExitEvidence, O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
|
|
3
3
|
import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
|
|
4
4
|
import type { AttemptIdentity, AttemptLocator } from "./locator.ts";
|
|
@@ -204,7 +204,7 @@ export interface Record {
|
|
|
204
204
|
}
|
|
205
205
|
/**
|
|
206
206
|
* 一个实验的覆盖事实:已知 eval 并集(分母)与当前口径下没有任何 attempt 的题。
|
|
207
|
-
* `
|
|
207
|
+
* `missing` 永远被算出来,不静默——渲染面把它转成覆盖占位行
|
|
208
208
|
* (见 docs/feature/sample/library.md「选择快照」「时效:新执行与历史执行」)。
|
|
209
209
|
*
|
|
210
210
|
* `run` 是该 Experiment 的锚点 Run:零 attempt 的 Eval 按 agent / model / flags 归组时
|
|
@@ -217,8 +217,20 @@ export interface SampleCoverage {
|
|
|
217
217
|
run: Run;
|
|
218
218
|
/** 分母:本地历史 ∪ 各快照携带的 knownEvalIds,交命令行范围(与 `exp.knownEvalIds` 同源)。 */
|
|
219
219
|
knownEvalIds: string[];
|
|
220
|
-
/**
|
|
221
|
-
|
|
220
|
+
/** 当前配置下没有物理 Attempt 的题,以及帮助用户决定下一步的缺口原因。 */
|
|
221
|
+
missing: SampleMissing[];
|
|
222
|
+
}
|
|
223
|
+
/** 当前结果缺口;原因只解释下一步,不构成另一种结果状态。 */
|
|
224
|
+
export interface SampleMissing {
|
|
225
|
+
evalId: string;
|
|
226
|
+
/** 从未有物理 Attempt,或有历史结果但没有一条能代表当前配置。 */
|
|
227
|
+
reason: "never-run" | "previous-result";
|
|
228
|
+
/** 最近一条旧结果的审计入口;它不参与当前统计,也不保证一定满足 accept 资格。 */
|
|
229
|
+
previous?: {
|
|
230
|
+
locator: AttemptLocator;
|
|
231
|
+
verdict: Verdict;
|
|
232
|
+
startedAt: string;
|
|
233
|
+
};
|
|
222
234
|
}
|
|
223
235
|
/**
|
|
224
236
|
* Sample(范围):选出的快照 + 口径 + 已按口径物化的 attempt 全集 + 挑选警告。
|
|
@@ -228,8 +240,6 @@ export interface SampleCoverage {
|
|
|
228
240
|
export interface Sample {
|
|
229
241
|
/** 这份 Sample 的口径,字面写在数据上。 */
|
|
230
242
|
mode: "latest-run" | "current";
|
|
231
|
-
/** 是否只保留各 Experiment 锚点中真实执行的 attempt。 */
|
|
232
|
-
fresh: boolean;
|
|
233
243
|
runs: Run[];
|
|
234
244
|
/**
|
|
235
245
|
* 按口径物化的 attempt 全集:消费 attempts 就自动正确,不需要自己 flatten runs,
|
|
@@ -251,7 +261,6 @@ export interface Sample {
|
|
|
251
261
|
evals?: string | string[];
|
|
252
262
|
}): Sample;
|
|
253
263
|
filter(predicate: (attempt: AttemptHandle) => boolean): Sample;
|
|
254
|
-
freshOnly(): Sample;
|
|
255
264
|
}
|
|
256
265
|
/**
|
|
257
266
|
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message;能用一条命令直接推进的
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
2
|
import { defineComponent } from "../definition/tree.js";
|
|
3
|
-
import { Callouts, Chart, Col, Conversation, CopyBlock, DiffView, Grid, Scatter, Series, SourceView, Stat, TableContentView,
|
|
3
|
+
import { Callouts, Chart, Col, Conversation, CopyBlock, DiffView, Grid, Scatter, Series, SourceView, Stat, TableContentView, Waterfall, } from "../definition/primitives.js";
|
|
4
4
|
import { HeroCard } from "../components/site-components/index.js";
|
|
5
5
|
import { AttemptSummary } from "../components/attempt-detail/index.js";
|
|
6
6
|
import { experimentListContent } from "../components/entity-lists/content.js";
|
|
7
|
-
import { ExperimentTableView } from "../components/entity-lists/index.js";
|
|
8
7
|
import { attemptAssertionsContent, attemptConversationContent, attemptDiffContent, embedConversationInSource, executionEvidenceUnavailableCallouts, attemptFixPromptContent, attemptNoticesContent, projectedSourceContent, attemptTimelineContent, } from "../components/attempt-detail/content.js";
|
|
9
8
|
import { stabilityMatrixContent } from "../slices/content.js";
|
|
10
|
-
import { formatInstant, formatReportDateTimeRange, } from "../model/format.js";
|
|
11
9
|
import { DEFAULT_REPORT_LOCALE, localeText } from "../model/locale.js";
|
|
12
10
|
function experimentTarget(point) {
|
|
13
11
|
return { page: "experiment", params: { experiment: point.key } };
|
|
@@ -15,27 +13,15 @@ function experimentTarget(point) {
|
|
|
15
13
|
function summaryView(result, locale) {
|
|
16
14
|
const snapshot = result.summary;
|
|
17
15
|
const tally = snapshot.evalVerdicts;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
total: snapshot.totalCostUSD.total,
|
|
25
|
-
})
|
|
26
|
-
: undefined })] }), snapshot.range.latestStartedAt !== null ? (_jsx(Text, { className: "niceeval-sample-summary-range", children: snapshot.range.earliestStartedAt !== null &&
|
|
27
|
-
snapshot.range.earliestStartedAt !== snapshot.range.latestStartedAt
|
|
28
|
-
? localeText(locale, "scopeSummary.runRange", {
|
|
29
|
-
from: formattedRange.from,
|
|
30
|
-
to: formattedRange.to,
|
|
31
|
-
})
|
|
32
|
-
: localeText(locale, "scopeSummary.lastRun", {
|
|
33
|
-
time: formatInstant(snapshot.range.latestStartedAt, locale),
|
|
34
|
-
}) })) : null] }));
|
|
16
|
+
return (_jsx(Col, { className: "niceeval-sample-summary", children: _jsxs(Grid, { children: [snapshot.evaluationKindComposition !== "points" ? (_jsx(Stat, { label: localeText(locale, "scopeSummary.passRate"), value: { kind: "metric", metric: snapshot.endToEndPassRate } })) : null, snapshot.totalScore !== undefined ? (_jsx(Stat, { label: localeText(locale, "scopeSummary.totalScore"), value: { kind: "metric", metric: snapshot.totalScore } })) : null, _jsx(Stat, { label: localeText(locale, "scopeSummary.experiments"), value: snapshot.experiments }), _jsx(Stat, { label: localeText(locale, "scopeSummary.evals"), value: snapshot.evals }), _jsx(Stat, { label: localeText(locale, "scopeSummary.attempts"), value: snapshot.attempts }), _jsx(Stat, { label: localeText(locale, "scopeSummary.votesEval"), value: { kind: "verdict", counts: tally } }), _jsx(Stat, { label: localeText(locale, "scopeSummary.totalCost"), value: { kind: "metric", metric: snapshot.totalCostUSD }, detail: snapshot.totalCostUSD.samples < snapshot.totalCostUSD.total
|
|
17
|
+
? localeText(locale, "scopeSummary.costCoverage", {
|
|
18
|
+
samples: snapshot.totalCostUSD.samples,
|
|
19
|
+
total: snapshot.totalCostUSD.total,
|
|
20
|
+
})
|
|
21
|
+
: undefined })] }) }));
|
|
35
22
|
}
|
|
36
23
|
export const StandardOverviewResultView = defineComponent(async ({ result }, ctx) => {
|
|
37
24
|
const table = experimentListContent(result.experiments);
|
|
38
|
-
const freshTable = result.freshExperiments ? experimentListContent(result.freshExperiments) : null;
|
|
39
25
|
const hasPassRate = table.columns.some((column) => column.key === "passRate");
|
|
40
26
|
const hasTotalScore = table.columns.some((column) => column.key === "totalScore");
|
|
41
27
|
const defaultSort = hasPassRate === hasTotalScore
|
|
@@ -43,7 +29,7 @@ export const StandardOverviewResultView = defineComponent(async ({ result }, ctx
|
|
|
43
29
|
: hasPassRate
|
|
44
30
|
? "passRate"
|
|
45
31
|
: "totalScore";
|
|
46
|
-
return (_jsxs(Col, { children: [_jsx(HeroCard, { title: ctx.report.title, data: result.hero }), _jsx(Callouts, { items: result.notices }), _jsx(Callouts, { items: result.diagnostics }), _jsx(CopyBlock, { content: result.fixPrompt }), summaryView(result, DEFAULT_REPORT_LOCALE), _jsx(Col, { children: result.charts.map((chart) => chart.y === "passRate" ? (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "passRate", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y)) : (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "totalScore", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y))) }),
|
|
32
|
+
return (_jsxs(Col, { children: [_jsx(HeroCard, { title: ctx.report.title, data: result.hero }), _jsx(Callouts, { items: result.notices }), _jsx(Callouts, { items: result.diagnostics }), _jsx(CopyBlock, { content: result.fixPrompt }), summaryView(result, DEFAULT_REPORT_LOCALE), _jsx(Col, { children: result.charts.map((chart) => chart.y === "passRate" ? (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "passRate", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y)) : (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "totalScore", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y))) }), _jsx(TableContentView, { data: table, sort: defaultSort, searchable: true })] }));
|
|
47
33
|
});
|
|
48
34
|
StandardOverviewResultView.displayName = "StandardOverviewResultView";
|
|
49
35
|
function stabilityPresentation(result) {
|
|
@@ -11,12 +11,6 @@ export declare function failureSummaryOf(result: EvalResult): {
|
|
|
11
11
|
summary: string | null;
|
|
12
12
|
more: number;
|
|
13
13
|
};
|
|
14
|
-
/**
|
|
15
|
-
* 「只看新执行」开关在场的判据(docs/feature/reports/components/summaries/experiment-table.md
|
|
16
|
-
* 「只看新执行」):Sample 里既没有历史执行也没有过期结论时不画开关——一个永远不改变行集的
|
|
17
|
-
* 控件只会让人怀疑自己看漏了什么。`ExperimentTable` 与内建默认报告共用同一条判据。
|
|
18
|
-
*/
|
|
19
|
-
export declare function hasHistoricalOrStale(items: readonly ExperimentListItem[]): boolean;
|
|
20
14
|
/** 已选出的 AttemptHandle[] → 列表行;顺序保持传入顺序(不再次按 Sample 去重)。 */
|
|
21
15
|
export declare function attemptRowsOf(attempts: readonly AttemptHandle[]): Promise<AttemptListItem[]>;
|
|
22
16
|
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Sample 展平顺序(不重排)。 */
|
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
|
|
9
9
|
import { comparabilityConfigOf, deepEqualJson } from "../../../sample/index.js";
|
|
10
10
|
import { foldEvalVerdict } from "../../../shared/verdict.js";
|
|
11
|
-
import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems,
|
|
11
|
+
import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems, locatorOf, resolveInput, } from "../../model/aggregate.js";
|
|
12
12
|
import { attemptCostUSD, costUSD, durationMs, examScore, passRate, tokens, totalScore } from "../../model/metrics.js";
|
|
13
13
|
import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../../../assertions/display.js";
|
|
14
14
|
import { firstLine } from "../../../util.js";
|
|
15
|
-
import {
|
|
15
|
+
import { summarizeItems } from "../shared-compute.js";
|
|
16
16
|
/**
|
|
17
17
|
* 一次 attempt 的单行结果摘要(断言摘要契约):failed 取主失败断言摘要(不含
|
|
18
18
|
* "+N more",N 单独进 moreFailures),errored 取结构化 error 的一层摘要
|
|
@@ -47,11 +47,6 @@ export function failureSummaryOf(result) {
|
|
|
47
47
|
async function attemptListItemOf(item) {
|
|
48
48
|
const result = item.attempt.result;
|
|
49
49
|
const { summary, more } = failureSummaryOf(result);
|
|
50
|
-
const historical = historicalOf(item);
|
|
51
|
-
// 缺 startedAt(legacy / 第三方落盘)时退化到所属快照的 startedAt——时效标注宁可粗一档
|
|
52
|
-
// 时距,不留空字段(与 dedupeAttempts「缺才不去重」同一条「不伪造」纪律,这里伪造的只是
|
|
53
|
-
// 展示粒度,不影响身份判定)。
|
|
54
|
-
const startedAt = result.startedAt ?? item.run.startedAt;
|
|
55
50
|
return {
|
|
56
51
|
experimentId: experimentIdOf(item),
|
|
57
52
|
evalId: evalIdOf(item),
|
|
@@ -63,23 +58,13 @@ async function attemptListItemOf(item) {
|
|
|
63
58
|
moreFailures: more,
|
|
64
59
|
examScore: await computeCell(examScore, [item]),
|
|
65
60
|
totalScore: await computeCell(totalScore, [item]),
|
|
61
|
+
tokens: await computeCell(tokens, [item]),
|
|
66
62
|
durationMs: result.durationMs,
|
|
67
63
|
costUSD: attemptCostUSD(result),
|
|
68
|
-
startedAt,
|
|
69
|
-
historical,
|
|
70
|
-
...(historical ? { staleSinceMs: msSince(startedAt) } : {}),
|
|
64
|
+
startedAt: result.startedAt ?? item.run.startedAt,
|
|
71
65
|
locator: locatorOf(item),
|
|
72
66
|
};
|
|
73
67
|
}
|
|
74
|
-
/**
|
|
75
|
-
* 「只看新执行」开关在场的判据(docs/feature/reports/components/summaries/experiment-table.md
|
|
76
|
-
* 「只看新执行」):Sample 里既没有历史执行也没有过期结论时不画开关——一个永远不改变行集的
|
|
77
|
-
* 控件只会让人怀疑自己看漏了什么。`ExperimentTable` 与内建默认报告共用同一条判据。
|
|
78
|
-
*/
|
|
79
|
-
export function hasHistoricalOrStale(items) {
|
|
80
|
-
return items.some((item) => Object.keys(item.staleReferences).length > 0 ||
|
|
81
|
-
item.evalRows.some((row) => row.attempts.some((attempt) => attempt.historical)));
|
|
82
|
-
}
|
|
83
68
|
/** 已选出的 AttemptHandle[] → 列表行;顺序保持传入顺序(不再次按 Sample 去重)。 */
|
|
84
69
|
export async function attemptRowsOf(attempts) {
|
|
85
70
|
return Promise.all(attempts.map((attempt) => attemptListItemOf({ attempt, run: attempt.run, watermark: attempt.run })));
|
|
@@ -170,38 +155,6 @@ function byMetricDescThenId(valueOf) {
|
|
|
170
155
|
return vb - va || a.experimentId.localeCompare(b.experimentId);
|
|
171
156
|
};
|
|
172
157
|
}
|
|
173
|
-
/**
|
|
174
|
-
* 覆盖缺口两档占位的「过期结论」参考(docs/feature/reports/components/summaries/experiment-table.md
|
|
175
|
-
* 「覆盖缺口的两档占位行」):`missingEvalIds` 里、`historyAttempts` 中存在与 `anchorConfigHash`
|
|
176
|
-
* 不可比判定的题,取其中最近一条。`fresh` 为 true 时整份不给参考——读者已声明只看新执行,
|
|
177
|
-
* 占位行就不再把被排除的历史结论请回来。
|
|
178
|
-
*/
|
|
179
|
-
function staleReferencesFor(experimentId, missingEvalIds, historyAttempts, anchorConfigHash, fresh) {
|
|
180
|
-
if (fresh || missingEvalIds.length === 0)
|
|
181
|
-
return {};
|
|
182
|
-
const missing = new Set(missingEvalIds);
|
|
183
|
-
const candidatesByEval = new Map();
|
|
184
|
-
for (const attempt of historyAttempts) {
|
|
185
|
-
if (attempt.experimentId !== experimentId)
|
|
186
|
-
continue;
|
|
187
|
-
if (!missing.has(attempt.evalId))
|
|
188
|
-
continue;
|
|
189
|
-
if (attempt.run.configHash === anchorConfigHash)
|
|
190
|
-
continue; // 可比,不是「过期结论」候选
|
|
191
|
-
const list = candidatesByEval.get(attempt.evalId);
|
|
192
|
-
if (list)
|
|
193
|
-
list.push(attempt);
|
|
194
|
-
else
|
|
195
|
-
candidatesByEval.set(attempt.evalId, [attempt]);
|
|
196
|
-
}
|
|
197
|
-
const out = {};
|
|
198
|
-
for (const [evalId, candidates] of candidatesByEval) {
|
|
199
|
-
const reference = staleReferenceOf(candidates);
|
|
200
|
-
if (reference)
|
|
201
|
-
out[evalId] = reference;
|
|
202
|
-
}
|
|
203
|
-
return out;
|
|
204
|
-
}
|
|
205
158
|
/**
|
|
206
159
|
* `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始排序按这份列表
|
|
207
160
|
* 自身的题型构成选择主读数——纯通过制沿用端到端通过率降序,纯计分制改按总分降序(缺数据
|
|
@@ -212,7 +165,7 @@ function staleReferencesFor(experimentId, missingEvalIds, historyAttempts, ancho
|
|
|
212
165
|
* 看跨配置演化用 run 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
213
166
|
*/
|
|
214
167
|
export async function experimentListData(input) {
|
|
215
|
-
const { runs, attempts, coverage
|
|
168
|
+
const { runs, attempts, coverage } = resolveInput(input);
|
|
216
169
|
const coverageByExperiment = new Map(coverage.map((c) => [c.experimentId, c]));
|
|
217
170
|
// 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
|
|
218
171
|
const configByExperiment = new Map();
|
|
@@ -229,7 +182,7 @@ export async function experimentListData(input) {
|
|
|
229
182
|
'use the "run" dimension or MetricLine; to show the current level, pass results.current() which selects a single config per experiment.');
|
|
230
183
|
}
|
|
231
184
|
}
|
|
232
|
-
const items = collectItems(runs,
|
|
185
|
+
const items = collectItems(runs, attempts);
|
|
233
186
|
const groups = groupItems(items, "experiment");
|
|
234
187
|
const out = [];
|
|
235
188
|
for (const [experimentId, group] of groups) {
|
|
@@ -260,7 +213,7 @@ export async function experimentListData(input) {
|
|
|
260
213
|
}
|
|
261
214
|
const experiment = newest.run.experiment ?? newest.attempt.result.experiment;
|
|
262
215
|
const model = newest.attempt.result.model ?? newest.run.model;
|
|
263
|
-
const
|
|
216
|
+
const coverageEntry = coverageByExperiment.get(experimentId);
|
|
264
217
|
out.push({
|
|
265
218
|
experimentId,
|
|
266
219
|
agent: newest.run.agent || newest.attempt.result.agent,
|
|
@@ -275,15 +228,14 @@ export async function experimentListData(input) {
|
|
|
275
228
|
tokens: await computeCell(tokens, group),
|
|
276
229
|
evals: stats.evals,
|
|
277
230
|
attempts: stats.attempts,
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
staleReferences: staleReferencesFor(experimentId, missingEvalIds, historyAttempts, coverageByExperiment.get(experimentId)?.run.configHash, fresh),
|
|
231
|
+
knownEvalIds: coverageEntry?.knownEvalIds ?? [],
|
|
232
|
+
missing: coverageEntry?.missing ?? [],
|
|
281
233
|
lastRunAt: stats.lastRunAt,
|
|
282
234
|
evalRows,
|
|
283
235
|
});
|
|
284
236
|
}
|
|
285
|
-
// coverage 不是 attempt
|
|
286
|
-
//
|
|
237
|
+
// coverage 不是 attempt 的附属品:current() 可能让一个实验当前口径下零 attempt。仍然给它
|
|
238
|
+
// 一行,让 missing 的占位题可达,不能把整实验静默吞掉。
|
|
287
239
|
for (const coverageEntry of coverage) {
|
|
288
240
|
if (groups.has(coverageEntry.experimentId))
|
|
289
241
|
continue;
|
|
@@ -307,9 +259,8 @@ export async function experimentListData(input) {
|
|
|
307
259
|
tokens: await computeCell(tokens, emptyItems),
|
|
308
260
|
evals: 0,
|
|
309
261
|
attempts: 0,
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
staleReferences: staleReferencesFor(coverageEntry.experimentId, coverageEntry.missingEvalIds, historyAttempts, anchor.configHash, fresh),
|
|
262
|
+
knownEvalIds: coverageEntry.knownEvalIds,
|
|
263
|
+
missing: coverageEntry.missing,
|
|
313
264
|
lastRunAt: anchor.startedAt,
|
|
314
265
|
evalRows: [],
|
|
315
266
|
});
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { TableContent } from "../../definition/cell.ts";
|
|
2
2
|
import type { AttemptListItem, EvalListItem, ExperimentListItem } from "../../model/types.ts";
|
|
3
|
-
/** 覆盖构成副行的 key 前缀;测试与消费方靠它把这一行从 Eval / 组行里筛出去。 */
|
|
4
|
-
export declare const COVERAGE_ROW_PREFIX = "coverage:";
|
|
5
3
|
export declare function experimentListContent(items: readonly ExperimentListItem[]): TableContent;
|
|
6
4
|
export declare function evalListContent(items: readonly EvalListItem[]): TableContent;
|
|
7
5
|
export declare function attemptListContent(items: readonly AttemptListItem[]): TableContent;
|