niceeval 0.5.4 → 0.6.0

package/src/i18n/en.ts

@@ -8,10 +8,22 @@ export const en = {
   "agent.diagnose.noTranscript": "transcript was not generated",
   "agent.diagnose.outputTail": "output tail: {{tail}}",
   "agent.diagnose.zeroEvents": "transcript exists but contains 0 events",
-  "bub.homeDetectFailed": "Failed to detect sandbox $HOME (empty output from `printf $HOME`). Refusing to fall back to a backend-specific path; check the sandbox backend.",
+  "bub.homeDetectFailed": "Failed to detect sandbox $HOME (empty output from `printf $HOME`). Refusing to fall back to a provider-specific path; check the sandbox provider.",
+  "bub.checkpointCaptureFailed": "bub checkpoint cache backfill failed (this sandbox is unaffected; later sandboxes will reinstall): {{error}}",
+  "bub.checkpointRestoreFailed": "bub checkpoint restore failed, falling back to a full install: {{error}}",
   "bub.installFailed": "bub install failed after {{attempts}} attempts:\n{{tail}}",
   "bub.setupNotRun": "bub adapter setup() has not run in this sandbox (missing home/workspace info). The runner must call setup before send.",
   "checkpoint.emptyTar": "checkpoint: tar is empty (paths: {{paths}})",
+  "skill.localMissing": "Local skill path \"{{path}}\" does not exist (resolved to {{resolved}}). Paths are resolved from the project root you run niceeval in.",
+  "skill.localDirNoSkillFile": "Local skill directory \"{{path}}\" has no SKILL.md. A directory-shaped skill must contain SKILL.md at its root.",
+  "skill.localUnsupportedShape": "Local skill path \"{{path}}\" has an unsupported shape. Accepted: a directory containing SKILL.md, or a single .md file.",
+  "skill.repoCloneFailed": "Could not fetch repo skill {{source}} (ref: {{ref}}):\n{{tail}}",
+  "skill.repoNoSkills": "Repo skill {{source}} contains no SKILL.md.",
+  "skill.repoAmbiguous": "Repo skill {{source}} contains multiple skills; select which ones to enable with `skills: [...]`. Available: {{available}}.",
+  "skill.repoUnknownSkill": "Repo skill {{source}} (ref: {{ref}}) has no skill named \"{{skill}}\". Available: {{available}}.",
+  "skill.copyFailed": "Could not install skill \"{{name}}\" into {{dest}}:\n{{tail}}",
+  "plugin.marketplaceFailed": "Could not connect {{agent}} marketplace \"{{name}}\" (source: {{source}}, ref: {{ref}}):\n{{tail}}",
+  "plugin.installFailed": "Could not install {{agent}} plugin \"{{name}}\" (marketplace: {{marketplace}}):\n{{tail}}",
   "cli.all": "(all)",
   "cli.browserOpenFailed": "Could not open the browser automatically. Open manually: {{url}}\n",
   "cli.clean.done": "Deleted .niceeval/ historical run artifacts.\n",
@@ -40,15 +52,23 @@ export const en = {
     "niceeval — agent-native evals\n\n" +
     "Usage:\n" +
     "  niceeval exp [group|experiment] [eval-id-prefix…]   run experiments\n" +
-    "  niceeval show [eval-id-prefix…]                     read results in the terminal\n" +
-    "      bare: current verdicts per experiment (composed across runs);\n" +
+    "  niceeval show [eval-id-prefix… | @<locator>]        read results in the terminal\n" +
+    "      bare: current verdicts per experiment (composed across runs), each row\n" +
+    "        with a compact attempt index (locator + evidence capability letters\n" +
+    "        [E,X,⏱,D] + failure reason)\n" +
     "      a single eval id: attempts + assertion details\n" +
-    "      --transcript / --trace / --diff[=file]   evidence slices of one eval\n" +
+    "      @<locator>  exactly one attempt: no flag -> compact overview;\n" +
+    "        with a flag -> that evidence slice\n" +
+    "      --eval        the Eval source captured when this attempt ran,\n" +
+    "        assertions mapped back to source lines\n" +
+    "      --execution   this attempt's execution event stream (messages/thinking/\n" +
+    "        Skill loads/tool calls); OTel adds timing to the same node when present\n" +
+    "      --diff[=file] sandbox workspace file-change summary; =file expands one file\n" +
     "      --history   cross-run timeline (mutually exclusive with --report)\n" +
     "      --run <dir>   pin a results dir    --experiment <id>   one experiment\n" +
-    "      --attempt <n>   pick an attempt    --report <file>   custom report\n" +
+    "      --report <file>   custom report\n" +
     "  niceeval list                                       list discovered evals\n" +
-    "  niceeval view [eval-id-prefix…|summary.json] [--out dir] [--port n] [--no-open]\n" +
+    "  niceeval view [eval-id-prefix…|snapshot.json] [--out dir] [--port n] [--no-open]\n" +
     "      report slot + evidence rooms; --report <file> swaps in your report\n" +
     "      (same file as show); --run <dir> pins a results dir;\n" +
     "      --experiment <id> one experiment\n" +
@@ -70,10 +90,9 @@ export const en = {
   "cli.show.historyReportConflict":
     "`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
   "cli.show.evidenceNeedsEval":
-    "--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
-  "cli.show.attemptNeedsEval":
-    "--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
-  "cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
+    "--eval / --execution / --diff show one attempt's evidence, but the selection matched {{matched}} evals. Pick an attempt locator from the index below:\n{{index}}\n",
+  "cli.show.locatorMalformed": "{{message}}\n",
+  "cli.show.locatorNotFound": "{{message}}\n",
   "cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
   "cli.eval.noMatchHintExperiment": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
   "cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
@@ -88,7 +107,7 @@ export const en = {
   "cli.noAgent": "No agent specified (use --agent <name>).\n",
   "cli.none": "(none)",
   "cli.pressCtrlC": "Press Ctrl+C to exit.\n",
-  "cli.resultsPath": "Structured results: {{path}} (each result's artifactsDir holds that attempt's events.json / trace.json / diff.json)\n",
+  "cli.resultsPath": "Structured results: {{path}} (snapshot.json + per-attempt result.json / events.json / trace.json / diff.json)\n",
   "cli.run.experimentRequired":
     "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
     "  Docs: node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
@@ -190,18 +209,18 @@ export const en = {
   "runner.startSandboxTeardown": "sandbox teardown (environment provisioning hooks)...",
   "runner.timeout": "attempt timed out ({{timeoutMs}}ms)\nRecent progress:\n{{recentLogs}}",
   "runner.traceSelected": " -> kept {{count}} semantic spans",
-  "runner.resumeCarry": "  · reusing {{carried}} passing results from last run, re-running {{retry}} evals\n",
+  "runner.resumeCarry": "  · reusing {{carried}} settled results from last run, re-running {{retry}} evals\n",
   "runner.resumeCarryDetail": "      carried [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
-  "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
+  "sandbox.providerNotImplemented": "{{provider}} sandbox provider is not implemented; use docker, vercel, or e2b",
   "sandbox.missingSpec":
     "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
-    "  Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
+    "  Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-providers.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
   "sandbox.forceCleanup": "  · [sandbox] force-cleaning {{count}} sandboxes...\n",
-  "sandbox.stopFailed": "  · [sandbox] failed to stop sandbox {{id}} (ignored; backend TTL should clean it up): {{message}}\n",
+  "sandbox.stopFailed": "  · [sandbox] failed to stop sandbox {{id}} (ignored; provider TTL should clean it up): {{message}}\n",
   "sandbox.stopTimeout": "stop timed out ({{timeoutMs}}ms)",
   "scoring.evalError": "evaluation error: {{error}}",
   "session.fileFallback": "[file]",

package/src/i18n/zh-CN.ts

@@ -6,13 +6,25 @@ export const zhCN = {
   "agent.diagnose.noTranscript": "transcript 未生成",
   "agent.diagnose.outputTail": "输出末尾:{{tail}}",
   "agent.diagnose.zeroEvents": "transcript 存在但 0 事件",
-  "bub.homeDetectFailed": "无法探测沙箱 $HOME(printf $HOME 输出为空)。不兜底到后端专属固定路径,请检查沙箱后端。",
+  "bub.homeDetectFailed": "无法探测沙箱 $HOME(printf $HOME 输出为空)。不兜底到 provider 专属固定路径,请检查沙箱 provider。",
+  "bub.checkpointCaptureFailed": "bub checkpoint 缓存回填失败(本沙箱不受影响,后续沙箱会重新安装):{{error}}",
+  "bub.checkpointRestoreFailed": "bub checkpoint 还原失败,回退到全量安装:{{error}}",
   "bub.installFailed": "bub 安装失败(重试 {{attempts}} 次):\n{{tail}}",
   "bub.setupNotRun": "bub adapter 的 setup() 尚未在该沙箱运行(缺 home/workspace 信息);运行器应先调 setup 再 send。",
   "checkpoint.emptyTar": "checkpoint: tar 为空(paths: {{paths}})",
+  "skill.localMissing": "本地 skill 路径不存在:\"{{path}}\"(解析到 {{resolved}})。path 相对跑 niceeval 的项目根解析。",
+  "skill.localDirNoSkillFile": "本地 skill 目录 \"{{path}}\" 里没有 SKILL.md。目录形态的 skill 必须在根下带一个 SKILL.md。",
+  "skill.localUnsupportedShape": "本地 skill 路径 \"{{path}}\" 形态不支持。只接受:含 SKILL.md 的目录,或单个 .md 文件。",
+  "skill.repoCloneFailed": "repo skill 拉取失败:{{source}}(ref: {{ref}})\n{{tail}}",
+  "skill.repoNoSkills": "repo skill {{source}} 里没找到任何 SKILL.md。",
+  "skill.repoAmbiguous": "repo skill {{source}} 里有多个 skill,必须用 `skills: [...]` 明确选择要启用哪些。可选:{{available}}。",
+  "skill.repoUnknownSkill": "repo skill {{source}}(ref: {{ref}})里没有名为 \"{{skill}}\" 的 skill。可选:{{available}}。",
+  "skill.copyFailed": "skill \"{{name}}\" 装进 {{dest}} 失败:\n{{tail}}",
+  "plugin.marketplaceFailed": "{{agent}} marketplace \"{{name}}\" 连接失败(source: {{source}}, ref: {{ref}}):\n{{tail}}",
+  "plugin.installFailed": "{{agent}} plugin \"{{name}}\"(marketplace: {{marketplace}})安装失败:\n{{tail}}",
   "cli.all": "(全部)",
   "cli.browserOpenFailed": "无法自动打开浏览器,请手动访问:{{url}}\n",
-  "cli.clean.done": "已删除 .niceeval/ 历史运行工件。\n",
+  "cli.clean.done": "已删除 .niceeval/ 历史运行 artifact。\n",
   "cli.config.missing":
     "找不到 niceeval.config.ts。\n" +
     "修法:\n" +
@@ -28,7 +40,7 @@ export const zhCN = {
   "runner.budgetUnenforceable":
     "{{budgetKey}} 的 budget:连续多个 attempt 完成后都拿不到成本数据(agent 不上报用量且模型不在价格表)——该 agent 的 budget 无法执行,取消护栏继续跑。\n",
   "judge.modelMissing":
-    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定评判模型(没有内置默认模型)。\n" +
+    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定裁判模型(没有内置默认模型)。\n" +
     "  文档:node_modules/niceeval/docs-site/zh/guides/scoring-guide.mdx",
   "loaders.yamlMissing":
     "loadYaml(\"{{path}}\") 需要 YAML 解析器:请先 `pnpm add yaml`(或改用 loadJson + JSON 数据集)。",
@@ -38,19 +50,24 @@ export const zhCN = {
     "niceeval — agent-native evals\n\n" +
     "用法:\n" +
     "  niceeval exp [组|实验] [eval-id 前缀…]   跑实验\n" +
-    "  niceeval show [eval-id 前缀…]            终端读结果\n" +
-    "      裸跑:每个 experiment 的现刻判决(跨 run 合成)\n" +
+    "  niceeval show [eval-id 前缀… | @<locator>]   终端读结果\n" +
+    "      裸跑:每个 experiment 的现刻判定(跨 run 合成),每行带紧凑 attempt 索引\n" +
+    "        (locator + 证据 capability 字母 [E,X,⏱,D] + 失败原因)\n" +
     "      单个 eval id:attempt 与断言明细\n" +
-    "      --transcript / --trace / --diff[=文件]   单 eval 的证据切面\n" +
+    "      @<locator>  精确一个 attempt:无 flag → 紧凑全景;带 flag → 对应证据切面\n" +
+    "      --eval        该 attempt 运行时保存的 Eval 源码,断言标回源码行\n" +
+    "      --execution   该 attempt 的执行事件流(消息/thinking/Skill/工具调用),\n" +
+    "        有 OTel 时同一节点补时间\n" +
+    "      --diff[=文件] sandbox 工作区文件改动摘要;=文件 展开单个文件\n" +
     "      --history   跨 run 时间轴(与 --report 互斥)\n" +
     "      --run <目录> 钉死结果目录   --experiment <id> 只看该实验\n" +
-    "      --attempt <n> 指定 attempt   --report <文件> 自定义报告\n" +
+    "      --report <文件> 自定义报告\n" +
     "  niceeval list                            列出发现到的 eval\n" +
-    "  niceeval view [eval-id 前缀…|summary.json] [--out 目录] [--port n] [--no-open]\n" +
+    "  niceeval view [eval-id 前缀…|snapshot.json] [--out 目录] [--port n] [--no-open]\n" +
     "      报告槽 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
     "      --run <目录> 钉死结果目录   --experiment <id> 只看该实验\n" +
-    "      --out <目录> 静态导出:index.html 连同查看器工件,可直接静态托管\n" +
-    "  niceeval clean                           删除 .niceeval/ 历史工件\n" +
+    "      --out <目录> 静态导出:index.html 连同查看器 artifact,可直接静态托管\n" +
+    "  niceeval clean                           删除 .niceeval/ 历史 artifact\n" +
     "  niceeval init                            脚手架 config + evals/\n\n" +
     "标志:\n" +
     "  --runs n  --max-concurrency n  --timeout ms  --budget usd  --tag t\n" +
@@ -67,10 +84,9 @@ export const zhCN = {
   "cli.show.historyReportConflict":
     "`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
   "cli.show.evidenceNeedsEval":
-    "--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
-  "cli.show.attemptNeedsEval":
-    "--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
-  "cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
+    "--eval / --execution / --diff show one attempt's evidence, but the selection matched {{matched}} evals. Pick an attempt locator from the index below:\n{{index}}\n",
+  "cli.show.locatorMalformed": "{{message}}\n",
+  "cli.show.locatorNotFound": "{{message}}\n",
   "cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
   "cli.eval.noMatchHintExperiment": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
   "cli.eval.noMatchKnown": "已发现 {{count}} 个 eval:{{evals}}\n",
@@ -85,14 +101,14 @@ export const zhCN = {
   "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
   "cli.none": "(无)",
   "cli.pressCtrlC": "按 Ctrl+C 退出。\n",
-  "cli.resultsPath": "结构化结果:{{path}}(每条结果的 artifactsDir 下是该次 attempt 的 events.json / trace.json / diff.json)\n",
+  "cli.resultsPath": "结构化结果:{{path}}(snapshot.json + 每 attempt 的 result.json / events.json / trace.json / diff.json)\n",
   "cli.run.experimentRequired":
     "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n" +
     "  文档:node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
   "cli.run.experimentRequiredHint": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "已发现实验:{{experiments}}\n",
   "cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
-  "cli.view.exportedDir": "已导出静态查看站:{{out}}(整个目录可直接静态托管;本地打开 {{out}}/index.html 需经 http 服务,file:// 下工件 fetch 不可用)\n",
+  "cli.view.exportedDir": "已导出静态查看站:{{out}}(整个目录可直接静态托管;本地打开 {{out}}/index.html 需经 http 服务,file:// 下 artifact fetch 不可用)\n",
   "cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
   "cli.view.noResults": "{{root}} 下没有结果。先 `niceeval exp` 跑一轮,再 `niceeval view`。\n",
   "cli.view.incompatibleForeign": "{{dir}}: 由 {{name}} {{version}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n请用写出它的那个工具查看这份报告。\n",
@@ -187,18 +203,18 @@ export const zhCN = {
   "runner.startSandboxTeardown": "sandbox teardown(环境预置钩子)…",
   "runner.timeout": "attempt 超时({{timeoutMs}}ms)\n最近进度:\n{{recentLogs}}",
   "runner.traceSelected": " → 留 {{count}}(按语义)",
-  "runner.resumeCarry": "  · 复用上次 {{carried}} 个通过的结果,重跑 {{retry}} 个 eval\n",
+  "runner.resumeCarry": "  · 复用上次 {{carried}} 个已判定的结果,重跑 {{retry}} 个 eval\n",
   "runner.resumeCarryDetail": "      复用 [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "使用 remote agent(不创建沙箱)…",
-  "sandbox.backendNotImplemented": "{{backend}} sandbox backend not implemented; use docker, vercel, or e2b",
+  "sandbox.providerNotImplemented": "{{provider}} sandbox provider not implemented; use docker, vercel, or e2b",
   "sandbox.missingSpec":
-    "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n" +
-    "  文档:node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
+    "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认 provider——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/sandbox-providers.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
   "sandbox.forceCleanup": "  · [sandbox] 强制清理 {{count}} 个沙箱…\n",
-  "sandbox.stopFailed": "  · [sandbox] 停沙箱 {{id}} 失败(已忽略,靠后端过期兜底):{{message}}\n",
+  "sandbox.stopFailed": "  · [sandbox] 停沙箱 {{id}} 失败(已忽略,靠 provider 过期兜底):{{message}}\n",
   "sandbox.stopTimeout": "stop 超时({{timeoutMs}}ms)",
   "scoring.evalError": "评估出错: {{error}}",
   "session.fileFallback": "[file]",

package/src/index.ts

@@ -22,7 +22,7 @@ export type {
   ToolMatch,
   ValueAssertion,
   Severity,
-  ResultOutcome,
+  Verdict,
   EvalDef,
   ExperimentDef,
   Config,