niceeval 0.11.4-canary.20 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/dist/i18n/en.d.ts +9 -1
- package/dist/i18n/en.js +11 -2
- package/dist/i18n/zh-CN.d.ts +10 -2
- package/dist/i18n/zh-CN.js +11 -2
- package/dist/record/attempt-evidence.d.ts +1 -1
- package/dist/record/open.js +25 -0
- package/dist/report/assets/series-encoding.d.ts +36 -0
- package/dist/report/assets/series-encoding.js +104 -0
- package/dist/report/built-in/result-components.js +12 -2
- package/dist/report/components/attempt-detail/compute.d.ts +8 -1
- package/dist/report/components/attempt-detail/compute.js +19 -1
- package/dist/report/components/attempt-detail/index.js +25 -3
- package/dist/report/components/attempt-detail/validate.js +3 -2
- package/dist/report/components/entity-lists/compute.d.ts +6 -0
- package/dist/report/components/entity-lists/compute.js +55 -8
- package/dist/report/components/entity-lists/content.d.ts +2 -0
- package/dist/report/components/entity-lists/content.js +59 -6
- package/dist/report/components/entity-lists/index.d.ts +21 -1
- package/dist/report/components/entity-lists/index.js +40 -7
- package/dist/report/components/entity-lists/validate.js +21 -0
- package/dist/report/components/fixtures.js +2 -0
- package/dist/report/definition/cell.d.ts +22 -0
- package/dist/report/definition/cell.js +23 -9
- package/dist/report/definition/primitives/chart.js +125 -25
- package/dist/report/definition/primitives/conversation.d.ts +1 -1
- package/dist/report/definition/primitives/conversation.js +5 -3
- package/dist/report/definition/primitives.js +32 -7
- package/dist/report/definition/tree.d.ts +5 -2
- package/dist/report/definition/tree.js +1 -0
- package/dist/report/index.d.ts +5 -5
- package/dist/report/index.js +2 -2
- package/dist/report/model/aggregate.d.ts +14 -1
- package/dist/report/model/aggregate.js +34 -2
- package/dist/report/model/conversions.d.ts +1 -0
- package/dist/report/model/conversions.js +4 -1
- package/dist/report/model/format.d.ts +6 -5
- package/dist/report/model/format.js +30 -9
- package/dist/report/model/locale.d.ts +19 -3
- package/dist/report/model/locale.js +31 -5
- package/dist/report/model/types.d.ts +41 -4
- package/dist/report/presentation.d.ts +53 -6
- package/dist/report/presentation.js +40 -5
- package/dist/report/react/index.d.ts +3 -3
- package/dist/report/react/index.js +1 -1
- package/dist/report/slices/compute.js +52 -4
- package/dist/report/slices/content.d.ts +2 -1
- package/dist/report/slices/content.js +48 -15
- package/dist/report/slices/validate.js +23 -0
- package/dist/report/tasks.d.ts +7 -1
- package/dist/report/tasks.js +5 -3
- package/dist/runner/types.d.ts +30 -10
- package/dist/sample/index.d.ts +0 -9
- package/dist/sample/index.js +0 -18
- package/dist/sandbox/backend.d.ts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.d.ts +2 -1
- package/dist/sandbox/docker.d.ts +28 -7
- package/dist/sandbox/e2b.d.ts +11 -0
- package/dist/sandbox/identity.d.ts +6 -1
- package/dist/sandbox/layer.d.ts +36 -3
- package/dist/sandbox/local.d.ts +9 -0
- package/dist/sandbox/types.d.ts +14 -8
- package/dist/sandbox/vercel.d.ts +9 -0
- package/docs-site/zh/reference/builtin-agents.mdx +2 -0
- package/docs-site/zh/reference/cli.mdx +1 -0
- package/docs-site/zh/reference/define-agent.mdx +13 -9
- package/docs-site/zh/reference/define-config.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +1 -1
- package/docs-site/zh/reference/results-data.mdx +1 -1
- package/docs-site/zh/tutorials/deploy-report-site.mdx +157 -0
- package/docs-site/zh/tutorials/publish-report.mdx +1 -1
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-providers.mdx +19 -4
- package/package.json +1 -1
- package/src/agents/codex-env.test.ts +8 -1
- package/src/agents/codex.ts +8 -0
- package/src/agents/coding-cli-versions.ts +22 -6
- package/src/cli.ts +23 -1
- package/src/i18n/en.ts +14 -3
- package/src/i18n/zh-CN.ts +13 -2
- package/src/record/attempt-evidence.ts +1 -1
- package/src/record/open.ts +21 -0
- package/src/record/results.test.ts +74 -13
- package/src/record/truncate.ts +32 -7
- package/src/record/writer.ts +39 -8
- package/src/report/assets/series-encoding.tsx +177 -0
- package/src/report/assets/styles.css +97 -5
- package/src/report/built-in/result-components.tsx +23 -2
- package/src/report/components/attempt-detail/attempt-components.test.tsx +41 -0
- package/src/report/components/attempt-detail/compute.ts +20 -2
- package/src/report/components/attempt-detail/index.tsx +32 -2
- package/src/report/components/attempt-detail/validate.test.ts +1 -0
- package/src/report/components/attempt-detail/validate.tsx +3 -1
- package/src/report/components/compute.test.ts +196 -0
- package/src/report/components/entity-lists/compute.ts +73 -7
- package/src/report/components/entity-lists/content.test.ts +14 -8
- package/src/report/components/entity-lists/content.ts +63 -6
- package/src/report/components/entity-lists/index.tsx +93 -8
- package/src/report/components/entity-lists/validate.test.ts +9 -0
- package/src/report/components/entity-lists/validate.ts +16 -0
- package/src/report/components/fixtures.ts +2 -0
- package/src/report/definition/cell.ts +46 -11
- package/src/report/definition/conversation.test.tsx +5 -1
- package/src/report/definition/primitives/chart.tsx +197 -32
- package/src/report/definition/primitives/conversation.tsx +7 -4
- package/src/report/definition/primitives.tsx +81 -9
- package/src/report/definition/tree.ts +6 -2
- package/src/report/index.ts +15 -2
- package/src/report/model/aggregate.ts +40 -2
- package/src/report/model/conversions.ts +5 -0
- package/src/report/model/format.ts +32 -9
- package/src/report/model/locale.ts +35 -5
- package/src/report/model/presentation.test.ts +23 -1
- package/src/report/model/types.ts +40 -4
- package/src/report/presentation.test.tsx +179 -19
- package/src/report/presentation.ts +114 -11
- package/src/report/react/index.tsx +13 -2
- package/src/report/runtime/dual-render.test.tsx +38 -2
- package/src/report/slices/compute.ts +59 -4
- package/src/report/slices/content.ts +55 -16
- package/src/report/slices/delta-table.test.ts +138 -3
- package/src/report/slices/validate.test.ts +29 -0
- package/src/report/slices/validate.ts +16 -0
- package/src/report/tasks.ts +12 -2
- package/src/runner/accept.test.ts +328 -10
- package/src/runner/accept.ts +239 -72
- package/src/runner/attempt.test.ts +25 -13
- package/src/runner/attempt.ts +9 -9
- package/src/runner/config-identity.test.ts +25 -0
- package/src/runner/config-identity.ts +6 -7
- package/src/runner/feedback/coordinator.test.ts +51 -0
- package/src/runner/feedback/coordinator.ts +23 -0
- package/src/runner/feedback/failure.test.ts +15 -0
- package/src/runner/feedback/failure.ts +7 -0
- package/src/runner/feedback/human.test.ts +548 -27
- package/src/runner/feedback/human.ts +321 -47
- package/src/runner/feedback/index.ts +10 -0
- package/src/runner/feedback/input-guard.test.ts +251 -0
- package/src/runner/feedback/input-guard.ts +171 -0
- package/src/runner/feedback/reducer.ts +1 -0
- package/src/runner/fingerprint.test.ts +24 -0
- package/src/runner/gate-lease.test.ts +38 -14
- package/src/runner/ledger.test.ts +10 -7
- package/src/runner/ledger.ts +9 -2
- package/src/runner/lock.test.ts +40 -16
- package/src/runner/manifest.ts +8 -9
- package/src/runner/reporters/artifacts.ts +25 -3
- package/src/runner/run.test.ts +61 -0
- package/src/runner/run.ts +34 -9
- package/src/runner/sandbox-pool.test.ts +216 -12
- package/src/runner/sandbox-pool.ts +20 -2
- package/src/runner/types.ts +30 -10
- package/src/sample/index.ts +0 -16
- package/src/sample/sample.test.ts +78 -0
- package/src/sandbox/backend.ts +1 -1
- package/src/sandbox/commands.ts +4 -7
- package/src/sandbox/compose.test.ts +26 -1
- package/src/sandbox/compose.ts +7 -3
- package/src/sandbox/docker.test.ts +66 -1
- package/src/sandbox/docker.ts +89 -65
- package/src/sandbox/dockerfile-build.ts +4 -1
- package/src/sandbox/e2b-agent-template.test.ts +18 -1
- package/src/sandbox/e2b-agent-template.ts +22 -6
- package/src/sandbox/e2b.test.ts +64 -0
- package/src/sandbox/e2b.ts +27 -5
- package/src/sandbox/identity.ts +7 -2
- package/src/sandbox/layer.test.ts +23 -4
- package/src/sandbox/layer.ts +127 -43
- package/src/sandbox/link.test.ts +5 -2
- package/src/sandbox/local.test.ts +8 -8
- package/src/sandbox/local.ts +28 -6
- package/src/sandbox/operations.test.ts +4 -4
- package/src/sandbox/operations.ts +5 -5
- package/src/sandbox/runtime.ts +18 -3
- package/src/sandbox/types.ts +14 -8
- package/src/sandbox/vercel.test.ts +58 -3
- package/src/sandbox/vercel.ts +38 -6
- package/src/show/index.ts +11 -30
- package/src/show/render.test.ts +23 -1
- package/src/show/render.ts +6 -3
package/INDEX.md
CHANGED
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
- `docs-site/zh/tutorials/criteria-files.mdx` — 隐藏测试判分:让判据文件进缓存 fingerprint:用 loadText 在模块顶层读入隐藏测试、跑测脚本等判据文件,文件内容进入 fingerprint:改一字节自动重跑,不靠人工 --rerun。
|
|
28
28
|
- `docs-site/zh/tutorials/custom-reports.mdx` — 编写自定义报告:用普通 TypeScript 函数计算结果,并同时生成终端与网页报告。
|
|
29
29
|
- `docs-site/zh/tutorials/dataset-fanout.mdx` — 数据驱动测试(dataset fan-out):用多份数据运行同一套评估用例:从 .eval.ts 文件导出数组或 keyed record,将一套评估逻辑展开为多个 case。用 loadYaml 或 loadJson 读取外部测试集,并获得稳定 ID。
|
|
30
|
+
- `docs-site/zh/tutorials/deploy-report-site.mdx` — 把报告站发布上线:把 .niceeval 直接提交进仓库、配一份排除运行时状态的 .gitignore,再用不装整仓依赖的 CI 脚本和 Vercel CLI,在打 tag 时把报告站发布上线。
|
|
30
31
|
- `docs-site/zh/tutorials/evaluation-kinds.mdx` — 题型、断言、Judge 和成本限制:值断言、作用域断言、LLM-as-judge、严重度怎么折成一个判定、计分制怎么给部分分,以及成本上限怎么写。
|
|
31
32
|
- `docs-site/zh/tutorials/experiments.mdx` — 实验矩阵:用运行矩阵比较 agents 和 models:使用 NiceEval experiments 让同一批评估用例横跨多个 agents、models 和 flags,比较 pass rate、成本和延迟。
|
|
32
33
|
- `docs-site/zh/tutorials/fixtures.mdx` — Sandbox Fixture:用任务评估 coding agents:用 .eval.ts 给 coding agent 准备隔离 workspace、发送真实任务,并用 Sandbox 文件、命令、diff 和 judge 验证结果。
|
package/dist/i18n/en.d.ts
CHANGED
|
@@ -162,6 +162,7 @@ export declare const en: {
|
|
|
162
162
|
"cli.view.noResults": string;
|
|
163
163
|
"cli.view.incompatibleForeign": string;
|
|
164
164
|
"cli.view.urls": string;
|
|
165
|
+
"codex.envPathManaged": string;
|
|
165
166
|
"context.capabilityMissing": string;
|
|
166
167
|
"context.skipEmpty": string;
|
|
167
168
|
"define.agentNameRequired": string;
|
|
@@ -197,11 +198,16 @@ export declare const en: {
|
|
|
197
198
|
"feedback.human.counts": string;
|
|
198
199
|
"feedback.human.diffHint": string;
|
|
199
200
|
"feedback.human.evalHint": string;
|
|
201
|
+
"feedback.human.exampleLocator": string;
|
|
202
|
+
"feedback.human.failureFacts": string;
|
|
200
203
|
"feedback.human.failuresHeader": string;
|
|
204
|
+
"feedback.human.failuresSoFar": string;
|
|
205
|
+
"feedback.human.failuresTotalKinds": string;
|
|
201
206
|
"feedback.human.heartbeat": string;
|
|
202
207
|
"feedback.human.inspect": string;
|
|
203
208
|
"feedback.human.keptSandboxesHeader": string;
|
|
204
209
|
"feedback.human.moreActive": string;
|
|
210
|
+
"feedback.human.moreFailureKinds": string;
|
|
205
211
|
"feedback.human.nextHeader": string;
|
|
206
212
|
"feedback.human.plan": string;
|
|
207
213
|
"feedback.human.planHeader": string;
|
|
@@ -217,6 +223,7 @@ export declare const en: {
|
|
|
217
223
|
"feedback.human.summaryAllReusedLine": string;
|
|
218
224
|
"feedback.human.suppressedFailures": string;
|
|
219
225
|
"feedback.human.trace": string;
|
|
226
|
+
"feedback.human.warningsHeader": string;
|
|
220
227
|
"feedback.phase.agentSetup": string;
|
|
221
228
|
"feedback.phase.agentEnsure": string;
|
|
222
229
|
"feedback.phase.stateLoad": string;
|
|
@@ -269,7 +276,7 @@ export declare const en: {
|
|
|
269
276
|
"local.dirMissing": string;
|
|
270
277
|
"local.dirNotWritable": string;
|
|
271
278
|
"local.notARepo": string;
|
|
272
|
-
"local.
|
|
279
|
+
"local.userUnsupported": string;
|
|
273
280
|
"report.assertionThreshold": string;
|
|
274
281
|
"report.error": string;
|
|
275
282
|
"report.errored": string;
|
|
@@ -354,4 +361,5 @@ export declare const en: {
|
|
|
354
361
|
"vercel.fileNotFound": string;
|
|
355
362
|
"vercel.rotateFailed": string;
|
|
356
363
|
"vercel.rotated": string;
|
|
364
|
+
"vercel.userUnsupported": string;
|
|
357
365
|
};
|
package/dist/i18n/en.js
CHANGED
|
@@ -228,7 +228,7 @@ export const en = {
|
|
|
228
228
|
"cli.experimentGroup": " path",
|
|
229
229
|
"cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
|
|
230
230
|
"cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
|
|
231
|
-
"cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (
|
|
231
|
+
"cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (tells coding agents to read the bundled docs before writing evals).\n",
|
|
232
232
|
"cli.init.esmHint": "tip: this project's package.json has no \"type\": \"module\". niceeval runs either way, but CommonJS mode disallows top-level await in config/eval files — add \"type\": \"module\" to package.json for the smoothest path.\n",
|
|
233
233
|
"cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
|
|
234
234
|
"cli.list.header": "Discovered {{count}} evals:\n",
|
|
@@ -245,6 +245,8 @@ export const en = {
|
|
|
245
245
|
"cli.view.noResults": "No results found under {{root}}. Run `niceeval exp` first, then `niceeval view`.\n",
|
|
246
246
|
"cli.view.incompatibleForeign": "{{dir}}: written by {{name}} {{version}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nOpen this report with the tool that produced it.\n",
|
|
247
247
|
"cli.view.urls": "niceeval view — open in a browser:\n{{urls}}\n",
|
|
248
|
+
"codex.envPathManaged": "codexAgent config.env.PATH is not supported: PATH is a Sandbox-managed variable, so silently dropping or overriding it would break hooks and child processes without any error. " +
|
|
249
|
+
"Prepend directories to it with the Sandbox factory's pathPrepend option instead (see docs/feature/sandbox/library.md).",
|
|
248
250
|
"context.capabilityMissing": "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
|
|
249
251
|
" Docs: node_modules/niceeval/docs-site/zh/tutorials/sandbox-agent.mdx",
|
|
250
252
|
"context.skipEmpty": "skip() requires a non-empty reason.",
|
|
@@ -281,11 +283,16 @@ export const en = {
|
|
|
281
283
|
"feedback.human.counts": "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{passed}} passed · {{failed}} failed · {{errored}} errored · {{skipped}} skipped",
|
|
282
284
|
"feedback.human.diffHint": "Diff: niceeval show {{locator}} --diff",
|
|
283
285
|
"feedback.human.evalHint": "Eval: niceeval show {{locator}} --source",
|
|
286
|
+
"feedback.human.exampleLocator": "e.g. {{locator}}",
|
|
287
|
+
"feedback.human.failureFacts": "facts ×{{count}}",
|
|
284
288
|
"feedback.human.failuresHeader": "FAILURES",
|
|
289
|
+
"feedback.human.failuresSoFar": "{{count}} so far",
|
|
290
|
+
"feedback.human.failuresTotalKinds": "{{total}} total · {{kinds}} kinds",
|
|
285
291
|
"feedback.human.heartbeat": "{{elapsed}} elapsed · {{counts}}",
|
|
286
292
|
"feedback.human.inspect": "Inspect: niceeval show {{locator}}",
|
|
287
293
|
"feedback.human.keptSandboxesHeader": "KEPT SANDBOXES",
|
|
288
294
|
"feedback.human.moreActive": "… {{count}} more active",
|
|
295
|
+
"feedback.human.moreFailureKinds": "+{{count}} more kinds — niceeval view",
|
|
289
296
|
"feedback.human.nextHeader": "NEXT",
|
|
290
297
|
"feedback.human.plan": "{{total}} attempts · {{evals}} evals × {{configs}} configs · concurrency {{concurrency}}",
|
|
291
298
|
"feedback.human.planHeader": "PLAN",
|
|
@@ -301,6 +308,7 @@ export const en = {
|
|
|
301
308
|
"feedback.human.summaryAllReusedLine": "{{passed}} passed · {{failed}} failed · {{errored}} errored (all {{reused}} reused)",
|
|
302
309
|
"feedback.human.suppressedFailures": "… {{count}} more failures suppressed",
|
|
303
310
|
"feedback.human.trace": "Trace: niceeval show {{locator}} --execution",
|
|
311
|
+
"feedback.human.warningsHeader": "WARNINGS",
|
|
304
312
|
"feedback.phase.agentSetup": "agent setup",
|
|
305
313
|
"feedback.phase.agentEnsure": "preparing agent",
|
|
306
314
|
"feedback.phase.stateLoad": "loading experiment state",
|
|
@@ -353,7 +361,7 @@ export const en = {
|
|
|
353
361
|
"local.dirMissing": "local sandbox directory does not exist: {{dir}}. Create it first, or point localSandbox({ dir }) at an existing directory.",
|
|
354
362
|
"local.dirNotWritable": "local sandbox directory is not writable: {{dir}} ({{message}})",
|
|
355
363
|
"local.notARepo": "the current directory (and its parents) is not inside a git repository, so localSandbox() has no deterministic root to run in. cd into the repository you want to evaluate, or pass an explicit directory: localSandbox({ dir: \"/path/to/repo\" }).",
|
|
356
|
-
"local.
|
|
364
|
+
"local.userUnsupported": 'the local sandbox provider does not support execution identity overrides, got { user: "{{user}}" } — niceeval does not escalate or switch identity on your machine. Use a container provider (docker / e2b / vercel) for steps that need a different user.',
|
|
357
365
|
"report.assertionThreshold": " (got {{score}} < {{threshold}})",
|
|
358
366
|
"report.error": "error",
|
|
359
367
|
"report.errored": "errored",
|
|
@@ -443,4 +451,5 @@ export const en = {
|
|
|
443
451
|
"vercel.fileNotFound": "File not found: {{path}}",
|
|
444
452
|
"vercel.rotateFailed": "[VercelSandbox] session rotate failed ({{seconds}}s): {{error}}",
|
|
445
453
|
"vercel.rotated": "[VercelSandbox] session rotated after {{seconds}}s -> {{sessionId}}",
|
|
454
|
+
"vercel.userUnsupported": 'the Vercel Sandbox provider only supports { user: "root" } (mapped to sudo: true) at the command level, got { user: "{{user}}" }. Use a container provider (docker / e2b) for other identities.',
|
|
446
455
|
};
|
package/dist/i18n/zh-CN.d.ts
CHANGED
|
@@ -146,7 +146,7 @@ export declare const zhCN: {
|
|
|
146
146
|
readonly "cli.experimentGroup": "路径";
|
|
147
147
|
readonly "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n";
|
|
148
148
|
readonly "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n";
|
|
149
|
-
readonly "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(
|
|
149
|
+
readonly "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(要求 coding agent 写 eval 前先读随包文档)。\n";
|
|
150
150
|
readonly "cli.init.esmHint": "提示:项目 package.json 没有 \"type\": \"module\"。niceeval 两种形态都能跑,但 CommonJS 形态下 config / eval 文件用不了顶层 await——建议在 package.json 加上 \"type\": \"module\"。\n";
|
|
151
151
|
readonly "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n";
|
|
152
152
|
readonly "cli.list.header": "发现 {{count}} 个 eval:\n";
|
|
@@ -162,6 +162,7 @@ export declare const zhCN: {
|
|
|
162
162
|
readonly "cli.view.noResults": "{{root}} 下没有结果。先 `niceeval exp` 跑一轮,再 `niceeval view`。\n";
|
|
163
163
|
readonly "cli.view.incompatibleForeign": "{{dir}}: 由 {{name}} {{version}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n请用写出它的那个工具查看这份报告。\n";
|
|
164
164
|
readonly "cli.view.urls": "niceeval view 可在浏览器打开:\n{{urls}}\n";
|
|
165
|
+
readonly "codex.envPathManaged": string;
|
|
165
166
|
readonly "context.capabilityMissing": string;
|
|
166
167
|
readonly "context.skipEmpty": "skip() 需要一个非空理由。";
|
|
167
168
|
readonly "define.agentNameRequired": "defineDirectAgent 需要 name。";
|
|
@@ -197,11 +198,16 @@ export declare const zhCN: {
|
|
|
197
198
|
readonly "feedback.human.counts": "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 通过 {{passed}} · 失败 {{failed}} · 出错 {{errored}} · 跳过 {{skipped}}";
|
|
198
199
|
readonly "feedback.human.diffHint": "Diff: niceeval show {{locator}} --diff";
|
|
199
200
|
readonly "feedback.human.evalHint": "Eval: niceeval show {{locator}} --source";
|
|
201
|
+
readonly "feedback.human.exampleLocator": "例如 {{locator}}";
|
|
202
|
+
readonly "feedback.human.failureFacts": "facts ×{{count}}";
|
|
200
203
|
readonly "feedback.human.failuresHeader": "FAILURES";
|
|
204
|
+
readonly "feedback.human.failuresSoFar": "已发生 {{count}} 条";
|
|
205
|
+
readonly "feedback.human.failuresTotalKinds": "共 {{total}} 条 · {{kinds}} 种形态";
|
|
201
206
|
readonly "feedback.human.heartbeat": "已运行 {{elapsed}} · {{counts}}";
|
|
202
207
|
readonly "feedback.human.inspect": "Inspect: niceeval show {{locator}}";
|
|
203
208
|
readonly "feedback.human.keptSandboxesHeader": "KEPT SANDBOXES";
|
|
204
209
|
readonly "feedback.human.moreActive": "… 还有 {{count}} 项运行中";
|
|
210
|
+
readonly "feedback.human.moreFailureKinds": "还有 {{count}} 种形态 — niceeval view";
|
|
205
211
|
readonly "feedback.human.nextHeader": "NEXT";
|
|
206
212
|
readonly "feedback.human.plan": "{{total}} 个 attempt · {{evals}} 个 eval × {{configs}} 个配置 · 并发 {{concurrency}}";
|
|
207
213
|
readonly "feedback.human.planHeader": "PLAN";
|
|
@@ -217,6 +223,7 @@ export declare const zhCN: {
|
|
|
217
223
|
readonly "feedback.human.summaryAllReusedLine": "{{passed}} 通过 · {{failed}} 失败 · {{errored}} 出错 (全部 {{reused}} 条复用)";
|
|
218
224
|
readonly "feedback.human.suppressedFailures": "… 还有 {{count}} 条失败被折叠";
|
|
219
225
|
readonly "feedback.human.trace": "Trace: niceeval show {{locator}} --execution";
|
|
226
|
+
readonly "feedback.human.warningsHeader": "WARNINGS";
|
|
220
227
|
readonly "feedback.phase.agentSetup": "agent 预置";
|
|
221
228
|
readonly "feedback.phase.agentEnsure": "正在准备 Agent";
|
|
222
229
|
readonly "feedback.phase.stateLoad": "载入实验状态";
|
|
@@ -269,7 +276,7 @@ export declare const zhCN: {
|
|
|
269
276
|
readonly "local.dirMissing": "本地沙箱目录不存在:{{dir}}。请先创建它,或让 localSandbox({ dir }) 指向一个已存在的目录。";
|
|
270
277
|
readonly "local.dirNotWritable": "本地沙箱目录不可写:{{dir}}({{message}})";
|
|
271
278
|
readonly "local.notARepo": "当前目录(及其上级目录)都不在任何 git 仓库内,localSandbox() 没有确定的 workdir 可用。cd 进要跑的仓库再执行,或显式传 localSandbox({ dir: \"/path/to/repo\" })。";
|
|
272
|
-
readonly "local.
|
|
279
|
+
readonly "local.userUnsupported": "本地 provider 不支持切换执行身份,收到 { user: \"{{user}}\" }——niceeval 不会在你的机器上提权或换身份。需要别的身份请改用容器 provider(docker / e2b / vercel)。";
|
|
273
280
|
readonly "report.assertionThreshold": " (得分 {{score}} < {{threshold}})";
|
|
274
281
|
readonly "report.error": "错误";
|
|
275
282
|
readonly "report.errored": "错误";
|
|
@@ -354,6 +361,7 @@ export declare const zhCN: {
|
|
|
354
361
|
readonly "vercel.fileNotFound": "File not found: {{path}}";
|
|
355
362
|
readonly "vercel.rotateFailed": "[VercelSandbox] session rotate failed ({{seconds}}s): {{error}}";
|
|
356
363
|
readonly "vercel.rotated": "[VercelSandbox] session rotated after {{seconds}}s → {{sessionId}}";
|
|
364
|
+
readonly "vercel.userUnsupported": "Vercel Sandbox provider 的命令级 user 只认 { user: \"root\" }(映射 sudo: true),收到 { user: \"{{user}}\" }。需要别的身份请改用容器 provider(docker / e2b)。";
|
|
357
365
|
};
|
|
358
366
|
export type MessageKey = keyof typeof zhCN;
|
|
359
367
|
export type Messages = globalThis.Record<MessageKey, string>;
|
package/dist/i18n/zh-CN.js
CHANGED
|
@@ -223,7 +223,7 @@ export const zhCN = {
|
|
|
223
223
|
"cli.experimentGroup": "路径",
|
|
224
224
|
"cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
|
|
225
225
|
"cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
|
|
226
|
-
"cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(
|
|
226
|
+
"cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(要求 coding agent 写 eval 前先读随包文档)。\n",
|
|
227
227
|
"cli.init.esmHint": "提示:项目 package.json 没有 \"type\": \"module\"。niceeval 两种形态都能跑,但 CommonJS 形态下 config / eval 文件用不了顶层 await——建议在 package.json 加上 \"type\": \"module\"。\n",
|
|
228
228
|
"cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
|
|
229
229
|
"cli.list.header": "发现 {{count}} 个 eval:\n",
|
|
@@ -240,6 +240,8 @@ export const zhCN = {
|
|
|
240
240
|
"cli.view.noResults": "{{root}} 下没有结果。先 `niceeval exp` 跑一轮,再 `niceeval view`。\n",
|
|
241
241
|
"cli.view.incompatibleForeign": "{{dir}}: 由 {{name}} {{version}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n请用写出它的那个工具查看这份报告。\n",
|
|
242
242
|
"cli.view.urls": "niceeval view 可在浏览器打开:\n{{urls}}\n",
|
|
243
|
+
"codex.envPathManaged": "codexAgent config.env.PATH 不支持:PATH 是 Sandbox 受管变量,静默丢弃或覆盖它会让 hooks 与子进程读到错误的可执行文件,且零报错。" +
|
|
244
|
+
"改用 Sandbox factory 的 pathPrepend 选项前置目录(见 docs/feature/sandbox/library.md)。",
|
|
243
245
|
"context.capabilityMissing": "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +
|
|
244
246
|
" 文档:node_modules/niceeval/docs-site/zh/tutorials/sandbox-agent.mdx",
|
|
245
247
|
"context.skipEmpty": "skip() 需要一个非空理由。",
|
|
@@ -276,11 +278,16 @@ export const zhCN = {
|
|
|
276
278
|
"feedback.human.counts": "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 通过 {{passed}} · 失败 {{failed}} · 出错 {{errored}} · 跳过 {{skipped}}",
|
|
277
279
|
"feedback.human.diffHint": "Diff: niceeval show {{locator}} --diff",
|
|
278
280
|
"feedback.human.evalHint": "Eval: niceeval show {{locator}} --source",
|
|
281
|
+
"feedback.human.exampleLocator": "例如 {{locator}}",
|
|
282
|
+
"feedback.human.failureFacts": "facts ×{{count}}",
|
|
279
283
|
"feedback.human.failuresHeader": "FAILURES",
|
|
284
|
+
"feedback.human.failuresSoFar": "已发生 {{count}} 条",
|
|
285
|
+
"feedback.human.failuresTotalKinds": "共 {{total}} 条 · {{kinds}} 种形态",
|
|
280
286
|
"feedback.human.heartbeat": "已运行 {{elapsed}} · {{counts}}",
|
|
281
287
|
"feedback.human.inspect": "Inspect: niceeval show {{locator}}",
|
|
282
288
|
"feedback.human.keptSandboxesHeader": "KEPT SANDBOXES",
|
|
283
289
|
"feedback.human.moreActive": "… 还有 {{count}} 项运行中",
|
|
290
|
+
"feedback.human.moreFailureKinds": "还有 {{count}} 种形态 — niceeval view",
|
|
284
291
|
"feedback.human.nextHeader": "NEXT",
|
|
285
292
|
"feedback.human.plan": "{{total}} 个 attempt · {{evals}} 个 eval × {{configs}} 个配置 · 并发 {{concurrency}}",
|
|
286
293
|
"feedback.human.planHeader": "PLAN",
|
|
@@ -296,6 +303,7 @@ export const zhCN = {
|
|
|
296
303
|
"feedback.human.summaryAllReusedLine": "{{passed}} 通过 · {{failed}} 失败 · {{errored}} 出错 (全部 {{reused}} 条复用)",
|
|
297
304
|
"feedback.human.suppressedFailures": "… 还有 {{count}} 条失败被折叠",
|
|
298
305
|
"feedback.human.trace": "Trace: niceeval show {{locator}} --execution",
|
|
306
|
+
"feedback.human.warningsHeader": "WARNINGS",
|
|
299
307
|
"feedback.phase.agentSetup": "agent 预置",
|
|
300
308
|
"feedback.phase.agentEnsure": "正在准备 Agent",
|
|
301
309
|
"feedback.phase.stateLoad": "载入实验状态",
|
|
@@ -348,7 +356,7 @@ export const zhCN = {
|
|
|
348
356
|
"local.dirMissing": "本地沙箱目录不存在:{{dir}}。请先创建它,或让 localSandbox({ dir }) 指向一个已存在的目录。",
|
|
349
357
|
"local.dirNotWritable": "本地沙箱目录不可写:{{dir}}({{message}})",
|
|
350
358
|
"local.notARepo": "当前目录(及其上级目录)都不在任何 git 仓库内,localSandbox() 没有确定的 workdir 可用。cd 进要跑的仓库再执行,或显式传 localSandbox({ dir: \"/path/to/repo\" })。",
|
|
351
|
-
"local.
|
|
359
|
+
"local.userUnsupported": "本地 provider 不支持切换执行身份,收到 { user: \"{{user}}\" }——niceeval 不会在你的机器上提权或换身份。需要别的身份请改用容器 provider(docker / e2b / vercel)。",
|
|
352
360
|
"report.assertionThreshold": " (得分 {{score}} < {{threshold}})",
|
|
353
361
|
"report.error": "错误",
|
|
354
362
|
"report.errored": "错误",
|
|
@@ -446,4 +454,5 @@ export const zhCN = {
|
|
|
446
454
|
"vercel.fileNotFound": "File not found: {{path}}",
|
|
447
455
|
"vercel.rotateFailed": "[VercelSandbox] session rotate failed ({{seconds}}s): {{error}}",
|
|
448
456
|
"vercel.rotated": "[VercelSandbox] session rotated after {{seconds}}s → {{sessionId}}",
|
|
457
|
+
"vercel.userUnsupported": "Vercel Sandbox provider 的命令级 user 只认 { user: \"root\" }(映射 sudo: true),收到 { user: \"{{user}}\" }。需要别的身份请改用容器 provider(docker / e2b)。",
|
|
449
458
|
};
|
|
@@ -58,7 +58,7 @@ export interface AttemptEvidence {
|
|
|
58
58
|
diff: DiffData | null;
|
|
59
59
|
/** OTel spans(`--timing` 把 turn 节点下的 agent/model/tool spans 挂回时间树);没有 trace 就是 null。 */
|
|
60
60
|
trace: import("../types.ts").TraceSpan[] | null;
|
|
61
|
-
/**
|
|
61
|
+
/** Sandbox 命令(成功与非零退出都记)的 stdout/stderr 证据(`--execution` 的独立命令卡);原样透传 attempt.commands()。 */
|
|
62
62
|
commands: readonly CommandExitEvidence[] | null;
|
|
63
63
|
artifactPaths: EvidencePaths;
|
|
64
64
|
capabilities: AttemptEvidenceCapabilities;
|
package/dist/record/open.js
CHANGED
|
@@ -429,8 +429,33 @@ async function readSnapshotDir(dir, meta, state) {
|
|
|
429
429
|
ev.attempts.sort((a, b) => a.result.attempt - b.result.attempt || (a.result.startedAt ?? "").localeCompare(b.result.startedAt ?? ""));
|
|
430
430
|
}
|
|
431
431
|
run.attempts = run.evals.flatMap((ev) => ev.attempts);
|
|
432
|
+
// 回退推导:run.json 缺 configHash(旧存量落盘,exp 写入面在这条修法之前从不写它)时,
|
|
433
|
+
// 若该快照全部 attempt 的 result.configHash 一致,取之为 Run 的 configHash——让存量记录
|
|
434
|
+
// 不重跑即痊愈。任一 attempt 缺失或值不一致就如实保持 undefined,不猜一个可能错的值
|
|
435
|
+
// (见 docs/feature/record/library.md「configHash:配置身份只算一次」)。
|
|
436
|
+
if (run.configHash === undefined) {
|
|
437
|
+
const derived = deriveConfigHashFromAttempts(run.attempts);
|
|
438
|
+
if (derived !== undefined)
|
|
439
|
+
run.configHash = derived;
|
|
440
|
+
}
|
|
432
441
|
return run;
|
|
433
442
|
}
|
|
443
|
+
/** `readSnapshotDir` 的 configHash 回退推导:全部 attempt 都带且相等才取用,否则 undefined。 */
|
|
444
|
+
function deriveConfigHashFromAttempts(attempts) {
|
|
445
|
+
if (attempts.length === 0)
|
|
446
|
+
return undefined;
|
|
447
|
+
let hash;
|
|
448
|
+
for (const attempt of attempts) {
|
|
449
|
+
const value = attempt.result.configHash;
|
|
450
|
+
if (value === undefined)
|
|
451
|
+
return undefined;
|
|
452
|
+
if (hash === undefined)
|
|
453
|
+
hash = value;
|
|
454
|
+
else if (hash !== value)
|
|
455
|
+
return undefined;
|
|
456
|
+
}
|
|
457
|
+
return hash;
|
|
458
|
+
}
|
|
434
459
|
function makeAttempt(run, snapshotDir, attemptDir, record) {
|
|
435
460
|
// 候选 artifact 目录:本 attempt 目录为主;--resume 携带条目的 artifact 留在原快照里,
|
|
436
461
|
// artifactBase(相对结果根 = 快照目录的上两级)指向那里,作为回退。
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { ReactNode } from "react";
|
|
2
|
+
import type { SeriesVariant } from "../presentation.ts";
|
|
3
|
+
/** 槽位色板下标 1..6 → CSS 变量,原样交给 fill / stroke / color。 */
|
|
4
|
+
export declare function seriesColorVar(colorIndex: number): string;
|
|
5
|
+
/** SVG pattern id:niceeval-series-pat-v{2|3|4}-c{1..6}。variant 1 是实心,无 pattern。 */
|
|
6
|
+
export declare function seriesPatternId(colorIndex: number, variant: SeriesVariant): string;
|
|
7
|
+
export declare function seriesFill(colorIndex: number, variant: SeriesVariant): string;
|
|
8
|
+
/** 线型变体 → strokeDasharray;variant 1 为空串(实线)。 */
|
|
9
|
+
export declare function seriesStrokeDasharray(variant: SeriesVariant): string;
|
|
10
|
+
export interface SeriesMarkerShape {
|
|
11
|
+
readonly path: string;
|
|
12
|
+
readonly viewBox: string;
|
|
13
|
+
}
|
|
14
|
+
/** 四种 marker 形状(viewBox 0 0 12 12),变体按 docs 槽序表 1–4。 */
|
|
15
|
+
export declare function seriesMarkerShape(variant: SeriesVariant): SeriesMarkerShape;
|
|
16
|
+
export declare function seriesMarker(colorIndex: number, variant: SeriesVariant): {
|
|
17
|
+
readonly path: string;
|
|
18
|
+
readonly viewBox: string;
|
|
19
|
+
readonly fill: string;
|
|
20
|
+
readonly stroke: string;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* 从 presentation.fill 反推 HTML 柱需要的系列 class。
|
|
24
|
+
* SVG 用 url(#pattern)/var();HTML 柱不能引用 SVG pattern,改挂 series-cN + fill-vN 类,
|
|
25
|
+
* 由 styles.css 用 repeating-linear-gradient 画等效图案,颜色仍走 --series 令牌。
|
|
26
|
+
*/
|
|
27
|
+
export declare function seriesClassesFromFill(fill: string): string;
|
|
28
|
+
/** 从 color 呈现或 series 的 stroke/fill 反推 series-cN 类(图例色点回落)。 */
|
|
29
|
+
export declare function seriesClassFromColorVar(color: string): string;
|
|
30
|
+
/**
|
|
31
|
+
* 页内注入一次的 SVG pattern defs。
|
|
32
|
+
* 18 个 pattern(3 非实心变体 × 6 色);id 与 seriesFill() 产出的 url(#…) 对齐。
|
|
33
|
+
* 子元素直接写 var(--niceeval-color-series-N),不走 currentColor
|
|
34
|
+
* (SVG pattern 内部 currentColor 取自 pattern 自身,引用者传不进来)。
|
|
35
|
+
*/
|
|
36
|
+
export declare function SeriesPatternDefs(): ReactNode;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
/** 槽位色板下标 1..6 → CSS 变量,原样交给 fill / stroke / color。 */
|
|
3
|
+
export function seriesColorVar(colorIndex) {
|
|
4
|
+
if (!Number.isInteger(colorIndex) || colorIndex < 1 || colorIndex > 6) {
|
|
5
|
+
throw new Error(`colorIndex must be an integer in [1, 6], got ${colorIndex}.`);
|
|
6
|
+
}
|
|
7
|
+
return `var(--niceeval-color-series-${colorIndex})`;
|
|
8
|
+
}
|
|
9
|
+
/** SVG pattern id:niceeval-series-pat-v{2|3|4}-c{1..6}。variant 1 是实心,无 pattern。 */
|
|
10
|
+
export function seriesPatternId(colorIndex, variant) {
|
|
11
|
+
if (variant === 1) {
|
|
12
|
+
throw new Error("variant 1 is solid fill; it has no pattern id.");
|
|
13
|
+
}
|
|
14
|
+
return `niceeval-series-pat-v${variant}-c${colorIndex}`;
|
|
15
|
+
}
|
|
16
|
+
export function seriesFill(colorIndex, variant) {
|
|
17
|
+
if (variant === 1)
|
|
18
|
+
return seriesColorVar(colorIndex);
|
|
19
|
+
return `url(#${seriesPatternId(colorIndex, variant)})`;
|
|
20
|
+
}
|
|
21
|
+
/** 线型变体 → strokeDasharray;variant 1 为空串(实线)。 */
|
|
22
|
+
export function seriesStrokeDasharray(variant) {
|
|
23
|
+
switch (variant) {
|
|
24
|
+
case 1:
|
|
25
|
+
return "";
|
|
26
|
+
case 2:
|
|
27
|
+
return "6 4";
|
|
28
|
+
case 3:
|
|
29
|
+
return "2 3";
|
|
30
|
+
case 4:
|
|
31
|
+
return "8 3 2 3";
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** 四种 marker 形状(viewBox 0 0 12 12),变体按 docs 槽序表 1–4。 */
|
|
35
|
+
export function seriesMarkerShape(variant) {
|
|
36
|
+
switch (variant) {
|
|
37
|
+
case 1:
|
|
38
|
+
// 圆
|
|
39
|
+
return { path: "M6 1.5a4.5 4.5 0 1 0 0.01 0z", viewBox: "0 0 12 12" };
|
|
40
|
+
case 2:
|
|
41
|
+
// 方
|
|
42
|
+
return { path: "M2.5 2.5h7v7h-7z", viewBox: "0 0 12 12" };
|
|
43
|
+
case 3:
|
|
44
|
+
// 菱
|
|
45
|
+
return { path: "M6 1.5 10.5 6 6 10.5 1.5 6z", viewBox: "0 0 12 12" };
|
|
46
|
+
case 4:
|
|
47
|
+
// 三角
|
|
48
|
+
return { path: "M6 1.5 10.5 10.5 1.5 10.5z", viewBox: "0 0 12 12" };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export function seriesMarker(colorIndex, variant) {
|
|
52
|
+
const shape = seriesMarkerShape(variant);
|
|
53
|
+
const color = seriesColorVar(colorIndex);
|
|
54
|
+
return {
|
|
55
|
+
path: shape.path,
|
|
56
|
+
viewBox: shape.viewBox,
|
|
57
|
+
fill: color,
|
|
58
|
+
stroke: color,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* 从 presentation.fill 反推 HTML 柱需要的系列 class。
|
|
63
|
+
* SVG 用 url(#pattern)/var();HTML 柱不能引用 SVG pattern,改挂 series-cN + fill-vN 类,
|
|
64
|
+
* 由 styles.css 用 repeating-linear-gradient 画等效图案,颜色仍走 --series 令牌。
|
|
65
|
+
*/
|
|
66
|
+
export function seriesClassesFromFill(fill) {
|
|
67
|
+
const pattern = /^url\(#niceeval-series-pat-v([2-4])-c([1-6])\)$/.exec(fill);
|
|
68
|
+
if (pattern) {
|
|
69
|
+
const variant = pattern[1];
|
|
70
|
+
const colorIndex = Number(pattern[2]);
|
|
71
|
+
return `niceeval-series-c${colorIndex - 1} niceeval-series-fill-v${variant}`;
|
|
72
|
+
}
|
|
73
|
+
const solid = /^var\(--niceeval-color-series-([1-6])\)$/.exec(fill);
|
|
74
|
+
if (solid) {
|
|
75
|
+
return `niceeval-series-c${Number(solid[1]) - 1}`;
|
|
76
|
+
}
|
|
77
|
+
return "niceeval-series-none";
|
|
78
|
+
}
|
|
79
|
+
/** 从 color 呈现或 series 的 stroke/fill 反推 series-cN 类(图例色点回落)。 */
|
|
80
|
+
export function seriesClassFromColorVar(color) {
|
|
81
|
+
const solid = /^var\(--niceeval-color-series-([1-6])\)$/.exec(color);
|
|
82
|
+
if (solid)
|
|
83
|
+
return `niceeval-series-c${Number(solid[1]) - 1}`;
|
|
84
|
+
return "niceeval-series-none";
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* 页内注入一次的 SVG pattern defs。
|
|
88
|
+
* 18 个 pattern(3 非实心变体 × 6 色);id 与 seriesFill() 产出的 url(#…) 对齐。
|
|
89
|
+
* 子元素直接写 var(--niceeval-color-series-N),不走 currentColor
|
|
90
|
+
* (SVG pattern 内部 currentColor 取自 pattern 自身,引用者传不进来)。
|
|
91
|
+
*/
|
|
92
|
+
export function SeriesPatternDefs() {
|
|
93
|
+
const patterns = [];
|
|
94
|
+
for (let colorIndex = 1; colorIndex <= 6; colorIndex++) {
|
|
95
|
+
const color = seriesColorVar(colorIndex);
|
|
96
|
+
// v2:对角斜线
|
|
97
|
+
patterns.push(_jsxs("pattern", { id: seriesPatternId(colorIndex, 2), patternUnits: "userSpaceOnUse", width: "6", height: "6", children: [_jsx("rect", { width: "6", height: "6", fill: color, fillOpacity: 0.22 }), _jsx("path", { d: "M-1 1l2-2M0 6l6-6M5 7l2-2", stroke: color, strokeWidth: 1.4, fill: "none" })] }, `v2-c${colorIndex}`));
|
|
98
|
+
// v3:水平条纹
|
|
99
|
+
patterns.push(_jsxs("pattern", { id: seriesPatternId(colorIndex, 3), patternUnits: "userSpaceOnUse", width: "6", height: "6", children: [_jsx("rect", { width: "6", height: "6", fill: color, fillOpacity: 0.18 }), _jsx("path", { d: "M0 1.5h6M0 4.5h6", stroke: color, strokeWidth: 1.5, fill: "none" })] }, `v3-c${colorIndex}`));
|
|
100
|
+
// v4:点阵
|
|
101
|
+
patterns.push(_jsxs("pattern", { id: seriesPatternId(colorIndex, 4), patternUnits: "userSpaceOnUse", width: "6", height: "6", children: [_jsx("rect", { width: "6", height: "6", fill: color, fillOpacity: 0.16 }), _jsx("circle", { cx: "2", cy: "2", r: "1.15", fill: color }), _jsx("circle", { cx: "5", cy: "5", r: "1.15", fill: color })] }, `v4-c${colorIndex}`));
|
|
102
|
+
}
|
|
103
|
+
return (_jsx("svg", { className: "niceeval-series-defs", width: 0, height: 0, "aria-hidden": "true", focusable: "false", style: { position: "absolute", width: 0, height: 0, overflow: "hidden" }, children: _jsx("defs", { children: patterns }) }));
|
|
104
|
+
}
|
|
@@ -4,6 +4,7 @@ import { Callouts, Chart, Col, Conversation, CopyBlock, DiffView, Grid, Scatter,
|
|
|
4
4
|
import { HeroCard } from "../components/site-components/index.js";
|
|
5
5
|
import { AttemptSummary } from "../components/attempt-detail/index.js";
|
|
6
6
|
import { experimentListContent } from "../components/entity-lists/content.js";
|
|
7
|
+
import { ExperimentTableView } from "../components/entity-lists/index.js";
|
|
7
8
|
import { attemptAssertionsContent, attemptConversationContent, attemptDiffContent, embedConversationInSource, executionEvidenceUnavailableCallouts, attemptFixPromptContent, attemptNoticesContent, projectedSourceContent, attemptTimelineContent, } from "../components/attempt-detail/content.js";
|
|
8
9
|
import { stabilityMatrixContent } from "../slices/content.js";
|
|
9
10
|
import { formatInstant, formatReportDateTimeRange, } from "../model/format.js";
|
|
@@ -34,6 +35,7 @@ function summaryView(result, locale) {
|
|
|
34
35
|
}
|
|
35
36
|
export const StandardOverviewResultView = defineComponent(async ({ result }, ctx) => {
|
|
36
37
|
const table = experimentListContent(result.experiments);
|
|
38
|
+
const freshTable = result.freshExperiments ? experimentListContent(result.freshExperiments) : null;
|
|
37
39
|
const hasPassRate = table.columns.some((column) => column.key === "passRate");
|
|
38
40
|
const hasTotalScore = table.columns.some((column) => column.key === "totalScore");
|
|
39
41
|
const defaultSort = hasPassRate === hasTotalScore
|
|
@@ -41,7 +43,7 @@ export const StandardOverviewResultView = defineComponent(async ({ result }, ctx
|
|
|
41
43
|
: hasPassRate
|
|
42
44
|
? "passRate"
|
|
43
45
|
: "totalScore";
|
|
44
|
-
return (_jsxs(Col, { children: [_jsx(HeroCard, { title: ctx.report.title, data: result.hero }), _jsx(Callouts, { items: result.notices }), _jsx(Callouts, { items: result.diagnostics }), _jsx(CopyBlock, { content: result.fixPrompt }), summaryView(result, DEFAULT_REPORT_LOCALE), _jsx(Col, { children: result.charts.map((chart) => chart.y === "passRate" ? (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "passRate", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y)) : (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "totalScore", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y))) }), _jsx(TableContentView, { data: table, sort: defaultSort, searchable: true })] }));
|
|
46
|
+
return (_jsxs(Col, { children: [_jsx(HeroCard, { title: ctx.report.title, data: result.hero }), _jsx(Callouts, { items: result.notices }), _jsx(Callouts, { items: result.diagnostics }), _jsx(CopyBlock, { content: result.fixPrompt }), summaryView(result, DEFAULT_REPORT_LOCALE), _jsx(Col, { children: result.charts.map((chart) => chart.y === "passRate" ? (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "passRate", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y)) : (_jsx(Scatter, { points: chart.points, x: "costUSD", y: "totalScore", point: "experiment", series: "series", connect: chart.connect, pointTarget: experimentTarget, legend: true }, chart.y))) }), freshTable === null ? (_jsx(TableContentView, { data: table, sort: defaultSort, searchable: true })) : (_jsx(ExperimentTableView, { fullContent: table, freshContent: freshTable, sort: defaultSort, searchable: true }))] }));
|
|
45
47
|
});
|
|
46
48
|
StandardOverviewResultView.displayName = "StandardOverviewResultView";
|
|
47
49
|
function stabilityPresentation(result) {
|
|
@@ -169,6 +171,14 @@ const TaskUsageKpi = defineComponent({
|
|
|
169
171
|
},
|
|
170
172
|
});
|
|
171
173
|
TaskUsageKpi.displayName = "TaskUsageKpi";
|
|
174
|
+
/** attempt 级 `ctx.fact()` 运行事实的完整键值表(docs/feature/record/architecture.md#facts运行事实);
|
|
175
|
+
* 没有 facts 时零输出,不摆空表。 */
|
|
176
|
+
const TaskFactsResultView = defineComponent(({ data }) => {
|
|
177
|
+
if (data === null || data.facts.length === 0)
|
|
178
|
+
return null;
|
|
179
|
+
return (_jsx(Grid, { className: "niceeval-facts-table", children: data.facts.map(({ key, value }) => (_jsx(TaskUsageKpi, { label: key, value: String(value) }, key))) }));
|
|
180
|
+
});
|
|
181
|
+
TaskFactsResultView.displayName = "TaskFactsResultView";
|
|
172
182
|
export const AttemptDetailsResultView = defineComponent(({ result }) => {
|
|
173
183
|
const notices = attemptNoticesContent(result.error, result.diagnostics) ?? [];
|
|
174
184
|
const source = projectedSourceContent(result.source.source, result.source.locator);
|
|
@@ -184,6 +194,6 @@ export const AttemptDetailsResultView = defineComponent(({ result }) => {
|
|
|
184
194
|
const conversation = attemptConversationContent(result.conversation.conversation);
|
|
185
195
|
const embedded = embedConversationInSource(source, conversation);
|
|
186
196
|
const files = attemptDiffContent(result.diff);
|
|
187
|
-
return (_jsxs(Col, { children: [_jsx(AttemptSummary, { data: result.summary }), _jsx(Callouts, { items: notices }), embedded.source !== null ? (_jsx(SourceView, { data: embedded.source })) : assertions !== null && assertions.rows.length > 0 ? (_jsx(TableContentView, { data: assertions })) : null, _jsx(CopyBlock, { content: attemptFixPromptContent(result.fixPrompt) }), _jsx(Waterfall, { nodes: timeline ?? [], title: { en: "Execution timeline", "zh-CN": "执行时间轴" } }), _jsx(TaskUsageResultView, { data: result.usage }), conversation === null ? (_jsx(Callouts, { items: executionEvidenceUnavailableCallouts })) : embedded.conversation !== null ? (_jsx(Conversation, { data: embedded.conversation })) : (null), _jsx(DiffView, { files: files })] }));
|
|
197
|
+
return (_jsxs(Col, { children: [_jsx(AttemptSummary, { data: result.summary }), _jsx(Callouts, { items: notices }), embedded.source !== null ? (_jsx(SourceView, { data: embedded.source })) : assertions !== null && assertions.rows.length > 0 ? (_jsx(TableContentView, { data: assertions })) : null, _jsx(CopyBlock, { content: attemptFixPromptContent(result.fixPrompt) }), _jsx(Waterfall, { nodes: timeline ?? [], title: { en: "Execution timeline", "zh-CN": "执行时间轴" } }), _jsx(TaskUsageResultView, { data: result.usage }), _jsx(TaskFactsResultView, { data: result.facts }), conversation === null ? (_jsx(Callouts, { items: executionEvidenceUnavailableCallouts })) : embedded.conversation !== null ? (_jsx(Conversation, { data: embedded.conversation })) : (null), _jsx(DiffView, { files: files })] }));
|
|
188
198
|
});
|
|
189
199
|
AttemptDetailsResultView.displayName = "AttemptDetailsResultView";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AttemptEvidence } from "../../../record/attempt-evidence.ts";
|
|
2
|
-
import type { AttemptAssertionsData, AttemptCommandEvidenceData, AttemptConversationData, AttemptDiagnosticsData, AttemptDiffData, AttemptErrorData, AttemptFixPromptData, AttemptSummaryData, AttemptTimelineData, AttemptTraceData, UsageTableData } from "../../model/types.ts";
|
|
2
|
+
import type { AttemptAssertionsData, AttemptCommandEvidenceData, AttemptConversationData, AttemptDiagnosticsData, AttemptDiffData, AttemptErrorData, AttemptFactsData, AttemptFixPromptData, AttemptSummaryData, AttemptTimelineData, AttemptTraceData, UsageTableData } from "../../model/types.ts";
|
|
3
3
|
export declare function attemptSummaryData(evidence: AttemptEvidence): AttemptSummaryData;
|
|
4
4
|
export declare function attemptErrorData(evidence: AttemptEvidence): AttemptErrorData | null;
|
|
5
5
|
export declare function attemptAssertionsData(evidence: AttemptEvidence): AttemptAssertionsData | null;
|
|
@@ -35,6 +35,13 @@ export declare function attemptDiagnosticsData(evidence: AttemptEvidence): Attem
|
|
|
35
35
|
* 没有任何用量事实可摆,与其余叶子同一条"没有 usage 时零输出"规则。
|
|
36
36
|
*/
|
|
37
37
|
export declare function usageTableData(evidence: AttemptEvidence): UsageTableData | null;
|
|
38
|
+
/**
|
|
39
|
+
* attempt 级 `ctx.fact()` 运行事实的完整键值表;`AttemptRecord.facts` 缺失或为空对象时返回
|
|
40
|
+
* null,不渲染空表(与其余叶子同一条「没有证据时零输出」规则)。
|
|
41
|
+
* `facts` 落盘就是 `Record<string, string | number | boolean>`,JS 对象的字符串键天然保留
|
|
42
|
+
* 写入顺序,这里按该顺序投影成数组,不重新排序。
|
|
43
|
+
*/
|
|
44
|
+
export declare function attemptFactsData(evidence: AttemptEvidence): AttemptFactsData | null;
|
|
38
45
|
export declare function attemptTraceData(evidence: AttemptEvidence): AttemptTraceData | null;
|
|
39
46
|
/**
|
|
40
47
|
* `null` 与空清单是两件事:`null` = 这次 attempt 没有 diff 证据(direct agent、发布未带 diff),
|
|
@@ -57,7 +57,9 @@ function looksLikeTruncatedCommandTail(message, commands) {
|
|
|
57
57
|
}
|
|
58
58
|
/** 展示层唯一的命令分类规则;Record 只保存 checked 与 exitCode。 */
|
|
59
59
|
function commandClassification(command) {
|
|
60
|
-
|
|
60
|
+
if (command.exitCode === 0)
|
|
61
|
+
return "succeeded";
|
|
62
|
+
return command.checked ? "failed" : "observed";
|
|
61
63
|
}
|
|
62
64
|
export function attemptErrorData(evidence) {
|
|
63
65
|
const err = evidence.result.error;
|
|
@@ -394,6 +396,22 @@ export function usageTableData(evidence) {
|
|
|
394
396
|
...(estimatedCostUSD !== null ? { estimatedCostUSD } : {}),
|
|
395
397
|
};
|
|
396
398
|
}
|
|
399
|
+
// ───────────────────────── AttemptFacts ─────────────────────────
|
|
400
|
+
/**
|
|
401
|
+
* attempt 级 `ctx.fact()` 运行事实的完整键值表;`AttemptRecord.facts` 缺失或为空对象时返回
|
|
402
|
+
* null,不渲染空表(与其余叶子同一条「没有证据时零输出」规则)。
|
|
403
|
+
* `facts` 落盘就是 `Record<string, string | number | boolean>`,JS 对象的字符串键天然保留
|
|
404
|
+
* 写入顺序,这里按该顺序投影成数组,不重新排序。
|
|
405
|
+
*/
|
|
406
|
+
export function attemptFactsData(evidence) {
|
|
407
|
+
const facts = evidence.result.facts;
|
|
408
|
+
if (!facts)
|
|
409
|
+
return null;
|
|
410
|
+
const entries = Object.entries(facts);
|
|
411
|
+
if (entries.length === 0)
|
|
412
|
+
return null;
|
|
413
|
+
return { facts: entries.map(([key, value]) => ({ key, value })) };
|
|
414
|
+
}
|
|
397
415
|
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
398
416
|
export function attemptTraceData(evidence) {
|
|
399
417
|
const spans = evidence.trace;
|
|
@@ -6,7 +6,7 @@ import { Callouts, CommandEvidence, Col, Conversation, CopyBlock, DiffView, Grid
|
|
|
6
6
|
import { formatDurationMs, formatInstant, formatPoints, formatUSD } from "../../model/format.js";
|
|
7
7
|
import { localeText } from "../../model/locale.js";
|
|
8
8
|
import { cx } from "../shared.js";
|
|
9
|
-
import { toAttemptAssertions, toAttemptFixPrompt, toAttemptNotices, toAttemptSource, toAttemptSummary, toAttemptUsage, toCommandEvidence, toConversationTurns, toDiffFiles, toTimelineNodes, } from "../../model/conversions.js";
|
|
9
|
+
import { toAttemptAssertions, toAttemptFacts, toAttemptFixPrompt, toAttemptNotices, toAttemptSource, toAttemptSummary, toAttemptUsage, toCommandEvidence, toConversationTurns, toDiffFiles, toTimelineNodes, } from "../../model/conversions.js";
|
|
10
10
|
import { embedConversationInSource, executionEvidenceUnavailableCallouts } from "./content.js";
|
|
11
11
|
export { validateAssertionsData, validateCommandEvidenceData, validateConversationData, validateDiagnosticsData, validateDiffData, validateErrorData, validateFixPromptData, validateSummaryData, validateTimelineData, validateTraceData, validateUsageData, } from "./validate.js";
|
|
12
12
|
/**
|
|
@@ -80,6 +80,27 @@ const AttemptUsage = defineComponent({
|
|
|
80
80
|
},
|
|
81
81
|
});
|
|
82
82
|
AttemptUsage.displayName = "AttemptUsage";
|
|
83
|
+
/** attempt 级 `ctx.fact()` 运行事实的完整键值表(docs/feature/record/architecture.md#facts运行事实);
|
|
84
|
+
* 没有 facts 时零输出,不摆空表。 */
|
|
85
|
+
const AttemptFacts = defineComponent({
|
|
86
|
+
dimensions: () => ({}),
|
|
87
|
+
web(props) {
|
|
88
|
+
const d = props.data;
|
|
89
|
+
if (d === null || d === undefined || d.facts.length === 0)
|
|
90
|
+
return null;
|
|
91
|
+
return (_jsx(Grid, { className: cx("niceeval-facts-table", props.className), children: d.facts.map(({ key, value }) => (_jsx(Kpi, { label: key, value: String(value) }, key))) }));
|
|
92
|
+
},
|
|
93
|
+
text(props) {
|
|
94
|
+
const d = props.data;
|
|
95
|
+
if (d === null || d === undefined || d.facts.length === 0)
|
|
96
|
+
return "";
|
|
97
|
+
const lines = ["facts:"];
|
|
98
|
+
for (const { key, value } of d.facts)
|
|
99
|
+
lines.push(` ${key} ${value}`);
|
|
100
|
+
return lines.join("\n");
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
AttemptFacts.displayName = "AttemptFacts";
|
|
83
104
|
function evidenceOf(props, ctx) {
|
|
84
105
|
if (props.attempt !== undefined)
|
|
85
106
|
return props.attempt;
|
|
@@ -113,7 +134,7 @@ AttemptAssessment.displayName = "AttemptAssessment";
|
|
|
113
134
|
export const AttemptDetails = defineComponent(async (props, ctx) => {
|
|
114
135
|
const evidence = evidenceOf(props, ctx);
|
|
115
136
|
const hasSource = evidence.capabilities.source;
|
|
116
|
-
const [notices, source, assertions, summary, fixPrompt, timeline, usage, commandEvidence, conversation, diff] = await Promise.all([
|
|
137
|
+
const [notices, source, assertions, summary, fixPrompt, timeline, usage, facts, commandEvidence, conversation, diff] = await Promise.all([
|
|
117
138
|
toAttemptNotices(evidence),
|
|
118
139
|
hasSource ? toAttemptSource(evidence) : Promise.resolve(null),
|
|
119
140
|
hasSource ? Promise.resolve(null) : toAttemptAssertions(evidence),
|
|
@@ -121,12 +142,13 @@ export const AttemptDetails = defineComponent(async (props, ctx) => {
|
|
|
121
142
|
toAttemptFixPrompt(evidence),
|
|
122
143
|
toTimelineNodes(evidence),
|
|
123
144
|
toAttemptUsage(evidence),
|
|
145
|
+
toAttemptFacts(evidence),
|
|
124
146
|
toCommandEvidence(evidence),
|
|
125
147
|
toConversationTurns(evidence),
|
|
126
148
|
toDiffFiles(evidence),
|
|
127
149
|
]);
|
|
128
150
|
const embedded = embedConversationInSource(source, conversation);
|
|
129
151
|
const commandSections = commandEvidenceSections(commandEvidence);
|
|
130
|
-
return (_jsxs(Col, { className: props.className, children: [_jsx(AttemptSummary, { data: summary }), _jsxs(Col, { children: [_jsx(Callouts, { items: notices }), _jsx(CommandEvidence, { data: commandSections.before }), embedded.source !== null ? (_jsx(SourceView, { data: embedded.source })) : assertions !== null && assertions.rows.length > 0 ? (_jsx(TableContentView, { data: assertions })) : null] }), _jsx(CopyBlock, { content: fixPrompt }), _jsx(Waterfall, { nodes: timeline, title: { en: "Execution timeline", "zh-CN": "执行时间轴" } }), _jsx(AttemptUsage, { data: usage }), conversation === null ? (_jsx(Callouts, { items: executionEvidenceUnavailableCallouts })) : embedded.conversation !== null ? (_jsx(Conversation, { data: embedded.conversation })) : (null), _jsx(CommandEvidence, { data: commandSections.after }), _jsx(DiffView, { files: diff })] }));
|
|
152
|
+
return (_jsxs(Col, { className: props.className, children: [_jsx(AttemptSummary, { data: summary }), _jsxs(Col, { children: [_jsx(Callouts, { items: notices }), _jsx(CommandEvidence, { data: commandSections.before }), embedded.source !== null ? (_jsx(SourceView, { data: embedded.source })) : assertions !== null && assertions.rows.length > 0 ? (_jsx(TableContentView, { data: assertions })) : null] }), _jsx(CopyBlock, { content: fixPrompt }), _jsx(Waterfall, { nodes: timeline, title: { en: "Execution timeline", "zh-CN": "执行时间轴" } }), _jsx(AttemptUsage, { data: usage }), _jsx(AttemptFacts, { data: facts }), conversation === null ? (_jsx(Callouts, { items: executionEvidenceUnavailableCallouts })) : embedded.conversation !== null ? (_jsx(Conversation, { data: embedded.conversation })) : (null), _jsx(CommandEvidence, { data: commandSections.after }), _jsx(DiffView, { files: diff })] }));
|
|
131
153
|
});
|
|
132
154
|
AttemptDetails.displayName = "AttemptDetails";
|
|
@@ -283,8 +283,9 @@ function commandEvidenceProblem(value, path) {
|
|
|
283
283
|
return `"${path}.display" must be a string`;
|
|
284
284
|
if (typeof value.exitCode !== "number")
|
|
285
285
|
return `"${path}.exitCode" must be a number`;
|
|
286
|
-
if (value.classification !== "observed" && value.classification !== "failed")
|
|
287
|
-
return `"${path}.classification" must be observed | failed`;
|
|
286
|
+
if (value.classification !== "succeeded" && value.classification !== "observed" && value.classification !== "failed") {
|
|
287
|
+
return `"${path}.classification" must be succeeded | observed | failed`;
|
|
288
|
+
}
|
|
288
289
|
if (value.durationMs !== undefined && typeof value.durationMs !== "number")
|
|
289
290
|
return `"${path}.durationMs" must be a number`;
|
|
290
291
|
if (typeof value.stdout !== "string")
|
|
@@ -11,6 +11,12 @@ export declare function failureSummaryOf(result: EvalResult): {
|
|
|
11
11
|
summary: string | null;
|
|
12
12
|
more: number;
|
|
13
13
|
};
|
|
14
|
+
/**
|
|
15
|
+
* 「只看新执行」开关在场的判据(docs/feature/reports/components/summaries/experiment-table.md
|
|
16
|
+
* 「只看新执行」):Sample 里既没有历史执行也没有过期结论时不画开关——一个永远不改变行集的
|
|
17
|
+
* 控件只会让人怀疑自己看漏了什么。`ExperimentTable` 与内建默认报告共用同一条判据。
|
|
18
|
+
*/
|
|
19
|
+
export declare function hasHistoricalOrStale(items: readonly ExperimentListItem[]): boolean;
|
|
14
20
|
/** 已选出的 AttemptHandle[] → 列表行;顺序保持传入顺序(不再次按 Sample 去重)。 */
|
|
15
21
|
export declare function attemptRowsOf(attempts: readonly AttemptHandle[]): Promise<AttemptListItem[]>;
|
|
16
22
|
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Sample 展平顺序(不重排)。 */
|