niceeval 0.6.1 → 0.6.2

package/docs-site/zh/reference/define-eval.mdx

@@ -1,7 +1,7 @@
 ---
 title: "defineEval：声明、配置并运行 NiceEval eval"
 sidebarTitle: "defineEval"
-description: "defineEval 参考：选项、test context t、Turn 返回值、sandbox helper 和数据集数组导出。"
+description: "defineEval 参考：选项、test context t、Turn 返回值、sandbox helper，以及数组和 keyed record 数据集导出。"
 ---
 
 `defineEval` 是编写 eval 的主要入口。每个 eval 文件调用一次，传入描述和 `test(t)`，并默认导出结果。
@@ -84,15 +84,28 @@ metadata?: Record<string, unknown>;
 
 任意附加元数据,原样透传进 EvalResult,不参与调度或打分;供自定义 reporter 消费。
 
+#### `diff`
+
+```ts
+diff?: { include?: string[]; ignore?: string[] };
+```
+
+调整 agent diff 的归因排除清单(仅沙箱型;见 docs/feature/eval/README.md):两个数组都是
+gitignore 风格 glob(workdir 相对)。默认排除 .git/node_modules/构建产物/包管理器缓存;
+`ignore` 在默认清单上追加排除;`include` 优先级最高,把匹配路径显式加回。
+合成规则固定为「默认 ∪ ignore,再被 include 打洞」,清单在分类账锚点时冻结。
+
 #### `setup`
 
 ```ts
-setup?: (sandbox: Sandbox) => Promise<void | Cleanup> | void | Cleanup;
+setup?: (sandbox: Sandbox, ctx: SandboxHookContext) => Promise<void | Cleanup> | void | Cleanup;
 ```
 
 eval 级预置:拿到沙箱(已上传 workspace + git 基线 + 装好依赖前)。
 默认命令以非 root 跑(agent 的自然环境);装系统依赖时给 `runCommand` 传 `{ root: true }`
 (如 `runCommand("apt-get", ["install", …], { root: true })`),跨 provider 语义一致。
+第二个参数是绑定到 `eval.setup` 的窄上下文(`ctx.progress` / `ctx.diagnostic`,
+见 docs/feature/eval/README.md);可返回 cleanup 闭包,归因到 `eval.teardown`。
 
 #### `test`
 
@@ -115,10 +128,10 @@ eval 主体:拿到 TestContext,驱动对话 / 沙箱操作并就地断言。
 #### `send`
 
 ```ts
-send(text: string): Promise<TurnHandle>;
+send(input: SendInput): Promise<TurnHandle>;
 ```
 
-向默认会话发一条消息,返回该轮的 TurnHandle。事件同时累加进默认会话的累计事件流,供下面的作用域断言使用。
+向默认会话发一条消息(字符串或结构化消息),返回该轮的 TurnHandle。事件同时累加进默认会话的累计事件流,供下面的作用域断言使用。
 
 #### `sendFile`
 
@@ -219,13 +232,31 @@ readonly flags: Readonly<Record<string, unknown>>;
 
 本次 attempt 生效的实验 flags(experiment.flags 的只读视图;实验条件,非命令行开关)。
 
+#### `progress`
+
+```ts
+progress(update: ProgressUpdate): void;
+```
+
+作用域反馈:报告 eval 自己执行的长步骤(上传 fixture、跑构建……)。短命状态,scope 固定
+为 `eval.run`;只报告不断言(见 docs/feature/eval/library/context.md「向运行反馈长步骤」)。
+
+#### `diagnostic`
+
+```ts
+diagnostic(input: DiagnosticInput): void;
+```
+
+作用域反馈:报告运行结束后仍需保留的问题(永久事件,落 attempt diagnostics)。
+即使 level 为 "error" 也不自动改变 verdict——测试结论仍由断言决定。
+
 #### `log`
 
 ```ts
 log(msg: string): void;
 ```
 
-打一行调试日志;有 live 进度回调时走该回调,否则落到 stderr,不出现在最终结果里。
+`progress({ message: msg })` 的别名(调试日志),不出现在最终结果里。
 
 #### `skip`
 
@@ -380,10 +411,10 @@ eventOrder(types: StreamEvent["type"][]): AssertionHandle;
 #### `eventsSatisfy`
 
 ```ts
-eventsSatisfy(predicate: (events: readonly StreamEvent[]) => boolean, label?: string): AssertionHandle;
+eventsSatisfy(label: string, predicate: (events: readonly StreamEvent[]) => boolean): AssertionHandle;
 ```
 
-用自定义谓词对默认会话累计的整段事件流断言;失败时用 label 说明原因。
+用自定义谓词对默认会话累计的整段事件流断言;label 必填、进断言标题。
 
 #### `sandbox`
 
@@ -633,10 +664,10 @@ eventOrder(types: StreamEvent["type"][]): AssertionHandle;
 #### `eventsSatisfy`
 
 ```ts
-eventsSatisfy(predicate: (events: readonly StreamEvent[]) => boolean, label?: string): AssertionHandle;
+eventsSatisfy(label: string, predicate: (events: readonly StreamEvent[]) => boolean): AssertionHandle;
 ```
 
-用自定义谓词对本轮整段事件流断言;失败时用 label 说明原因。
+用自定义谓词对本轮整段事件流断言;label 必填、进断言标题(谓词不透明,解释责任在 label)。
 
 #### `maxTokens`
 
@@ -678,3 +709,5 @@ export default rows.map((row) =>
 ```
 
 数组导出会生成稳定 ID：`file/0000`、`file/0001` 等。
+
+已有稳定业务 key 时也可以默认导出 `Record<string, EvalDef>`。例如 key `15193` 在 `swelancer.eval.ts` 中生成 `swelancer/15193`。key 必须是非空单一路径片段，不能是 `.` / `..`，不能含 `/`、`\\` 或控制字符；发现顺序按 key 字典序固定。

package/docs-site/zh/reference/expect.mdx

@@ -136,7 +136,7 @@ export function makeAssertion(spec: {
 t.check(t.reply, includes("Paris"));
 t.check(t.reply, includes(/order #\d+/i));
 t.check(turn.data, equals({ intent: "refund" }));
-t.check(t.reply, matches(/#[A-Z0-9]{8}/));
+t.check(turn.data, matches(OrderSchema)); // Standard Schema / Zod 结构校验;正则匹配用 includes
 t.check(t.reply, similarity("The answer explains the refund window").atLeast(0.8));
 t.check(turn.data, satisfies((value) => Array.isArray(value)));
 ```
@@ -172,6 +172,22 @@ readonly severity: Severity;
 readonly threshold?: number;
 ```
 
+#### `isOptional`
+
+```ts
+readonly isOptional?: boolean;
+```
+
+`.optional()` 链过的标记:评不了只记 unavailable,不把 attempt 拖成 errored。
+
+#### `expected`
+
+```ts
+readonly expected?: string;
+```
+
+期望条件的有界文本描述(如 `contains "Brooklyn"`),失败时进 AssertionResult.expected。
+
 #### `score`
 
 ```ts
@@ -195,6 +211,15 @@ atLeast(threshold: number): ValueAssertion;
 转成软阈值断言:未达 threshold 时该条记为 failed,但默认不拖累整条 eval 的 verdict;
 `--strict` 运行下,软阈值失败也会把整条 eval 的 verdict 计为 failed。返回新实例,不改原对象。
 
+#### `optional`
+
+```ts
+optional(): ValueAssertion;
+```
+
+允许这条断言证据缺席:评不了时只记录 `outcome: "unavailable"`,不影响判定。
+与 severity 正交(severity 说影不影响质量判定,optional 说证据允不允许缺席)。返回新实例,不改原对象。
+
 {/* GENERATED:END value-assertion */}
 
 ## 自定义 matcher

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "niceeval",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "Agent-native eval tool — eval agents, services, functions, and coding-agent fixtures",
   "type": "module",
   "license": "MIT",
@@ -24,6 +24,10 @@
       "types": "./src/sandbox/index.ts",
       "import": "./src/sandbox/index.ts"
     },
+    "./sandbox/e2b-template": {
+      "types": "./src/sandbox/e2b-agent-template.ts",
+      "import": "./src/sandbox/e2b-agent-template.ts"
+    },
     "./adapter": {
       "types": "./src/agents/index.ts",
       "import": "./src/agents/index.ts"

package/src/agents/ai-sdk-otel.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
 import { describe, expect, it } from "vitest";
 
 import { aiSdkOtel } from "./ai-sdk-otel.ts";

package/src/agents/ai-sdk.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
 import { describe, expect, it } from "vitest";
 
 import { aiSdkAgent, fromAiSdk } from "./ai-sdk.ts";
@@ -457,6 +458,8 @@ function fakeCtx(opts: { id?: string } = {}): AgentContext {
     flags: {},
     sandbox: undefined as unknown as AgentContext["sandbox"],
     session,
+    progress: () => {},
+    diagnostic: () => {},
     log: () => {},
   };
 }

package/src/agents/ai-sdk.ts

@@ -13,6 +13,7 @@
 //     (不在 steps 里),这里挖出来补成 `action.result` —— 拒绝(execution-denied)映射成
 //     `status: "rejected"`,喂 `calledTool(..., { status: "rejected" })` 与 `noFailedActions()`。
 
+import { completeCoverage } from "../scoring/coverage.ts";
 import { randomUUID } from "node:crypto";
 
 import { defineAgent } from "../define.ts";
@@ -466,6 +467,8 @@ export function aiSdkAgent<M = unknown>(options: AiSdkAgentOptions<M>): Agent {
 
   return defineAgent({
     name: options.name ?? "ai-sdk",
+    // 官方 SDK 完整 steps/output:全通道 complete(见 adapters/architecture/evidence.md)。
+    coverage: completeCoverage,
     // tracing 开了才让运行器为这个 agent 起 OTLP 接收器(ctx.telemetry 才会出现);
     // AgentTracing 的其余字段(env/configure/scope)这里都用不上,空对象即可。
     tracing: options.tracing ? {} : undefined,

package/src/agents/bub-install-spec.test.ts

@@ -0,0 +1,34 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
+import { describe, expect, it } from "vitest";
+import { readFile } from "node:fs/promises";
+import {
+  BUB_INSTALL_MARKER,
+  DEFAULT_BUB_OTEL_PLUGIN,
+  DEFAULT_BUB_OVERRIDE,
+  bubInstallHash,
+  bubInstallSpec,
+} from "./bub-install-spec.ts";
+
+describe("Bub install specification", () => {
+  it("uses immutable default Git commits", () => {
+    expect(DEFAULT_BUB_OVERRIDE).toMatch(/@[0-9a-f]{40}$/);
+    expect(DEFAULT_BUB_OTEL_PLUGIN).toMatch(/@[0-9a-f]{40}#subdirectory=/);
+  });
+
+  it("fingerprints plugins independent of caller ordering", () => {
+    const packages = ["alpha==1", "beta==2"];
+    expect(bubInstallHash(packages)).toBe(bubInstallHash([" beta==2 ", "alpha==1", "alpha==1"]));
+    expect(bubInstallSpec(packages)).toContain("--with alpha==1 --with beta==2");
+    expect(BUB_INSTALL_MARKER).toBe(".local/share/niceeval/bub-install-hash");
+  });
+
+  it("keeps the non-TypeScript Docker recipe in sync", async () => {
+    const [dockerfile, override] = await Promise.all([
+      readFile(new URL("../../sandbox/docker/Dockerfile", import.meta.url), "utf8"),
+      readFile(new URL("../../sandbox/docker/bub-override.txt", import.meta.url), "utf8"),
+    ]);
+    expect(override.trim()).toBe(DEFAULT_BUB_OVERRIDE);
+    expect(dockerfile).toContain(DEFAULT_BUB_OTEL_PLUGIN);
+    expect(dockerfile).toContain(bubInstallHash([]));
+  });
+});

package/src/agents/bub-install-spec.ts

@@ -0,0 +1,32 @@
+import { createHash } from "node:crypto";
+
+export const DEFAULT_BUB_OVERRIDE =
+  "bub @ git+https://github.com/CorrectRoadH/bub.git@86fbd0febc1665353f5131173554e1f513e66b4c";
+
+export const DEFAULT_BUB_OTEL_PLUGIN =
+  "git+https://github.com/bubbuild/bub-contrib.git@add4a6a133c5658aec8f167ef50804d9ee55d22e#subdirectory=packages/bub-tapestore-otel";
+
+export const BUB_CHECKPOINT_SUBDIRS = [".local"] as const;
+export const BUB_INSTALL_MARKER = ".local/share/niceeval/bub-install-hash";
+
+export function normalizeBubPackages(packages: readonly string[]): string[] {
+  return [...new Set(packages.map((value) => value.trim()).filter(Boolean))].sort();
+}
+
+export function bubInstallSpec(
+  packages: readonly string[],
+  override = DEFAULT_BUB_OVERRIDE,
+  otelPlugin = DEFAULT_BUB_OTEL_PLUGIN,
+): string {
+  const normalized = normalizeBubPackages(packages);
+  const plugins = normalized.length ? ` --with ${normalized.join(" --with ")}` : "";
+  return `bub --override(${override}) --with ${otelPlugin}${plugins} --checkpoint(${BUB_CHECKPOINT_SUBDIRS.join(",")})`;
+}
+
+export function bubInstallHash(
+  packages: readonly string[],
+  override = DEFAULT_BUB_OVERRIDE,
+  otelPlugin = DEFAULT_BUB_OTEL_PLUGIN,
+): string {
+  return createHash("md5").update(bubInstallSpec(packages, override, otelPlugin)).digest("hex").slice(0, 12);
+}

package/src/agents/bub.ts

@@ -1,3 +1,4 @@
+import { completeCoverage } from "../scoring/coverage.ts";
 import { defineSandboxAgent } from "../define.ts";
 import { requireEnv, getEnv } from "../util.ts";
 import { shared } from "./shared.ts";
@@ -16,6 +17,14 @@ import { readFile, writeFile, mkdir } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join, dirname } from "node:path";
 import { t } from "../i18n/index.ts";
+import {
+  BUB_CHECKPOINT_SUBDIRS,
+  BUB_INSTALL_MARKER,
+  DEFAULT_BUB_OTEL_PLUGIN,
+  DEFAULT_BUB_OVERRIDE,
+  bubInstallHash,
+  normalizeBubPackages,
+} from "./bub-install-spec.ts";
 
 // ───────────────────────────────────────────────────────────────────────────
 // bub 的 agent adapter(沙箱型)。
@@ -64,50 +73,32 @@ const SKILL_DIR = ".agents/skills";
 // 可用 NICEEVAL_BUB_OVERRIDE / NICEEVAL_BUB_OTEL_PLUGIN 覆盖,不必改源码。
 const BUB_OVERRIDE =
   getEnv("NICEEVAL_BUB_OVERRIDE") ??
-  "bub @ git+https://github.com/CorrectRoadH/bub.git@fix/tape-assistant-text-with-tool-calls";
+  DEFAULT_BUB_OVERRIDE;
 const BUB_OVERRIDE_FILE = "/tmp/bub-override.txt";
 // otel 插件跟上游 main 走(bub-contrib#50 起从 bub.tape 导入,要求 bub ≥ 0.3.10dev,
 // 与上面的 override 分支兼容)。插件不发 PyPI,git 依赖是唯一安装方式。
 const OTEL_PLUGIN =
   getEnv("NICEEVAL_BUB_OTEL_PLUGIN") ??
-  "git+https://github.com/bubbuild/bub-contrib.git#subdirectory=packages/bub-tapestore-otel";
+  DEFAULT_BUB_OTEL_PLUGIN;
 
-// override 钉在 git ref 上时,镜像里烘焙的 bub 不可信:模板构建时间早于 ref 当前指向的
-// commit 的话,`command -v bub` 命中的就是修复前的旧构建 —— e2b 模板 fasteval-agents 上
-// 整轮 bub turn failed(send 后无 AI 回复)就是这么来的:override 分支修好了,但捷径
-// 让它从未被安装。所以 pinned 时绕开 PATH 捷径、恒走 uv 安装(有 checkpoint 缓存,
-// 不是每沙箱都全量装),且运行时钉死用 $HOME/.local/bin/bub —— 只改 ensureBub 不改这里
-// 的话,PATH 上的 /usr/local/bin/bub 仍会先于新装的被 command -v 找到,白装。
-const BUB_PINNED = BUB_OVERRIDE.includes("git+");
-// bub 二进制:pinned → 恒用 uv 装到 $HOME/.local/bin 的那个;非 pinned → 优先用镜像里
-// (预制模板)烘焙在 PATH 上的 bub(装到 /usr/local/bin,见 sandbox/docker/Dockerfile)。
-const BUB = BUB_PINNED ? "$HOME/.local/bin/bub" : "$(command -v bub || echo $HOME/.local/bin/bub)";
+// NiceEval 的预制配方与运行时安装都写到 $HOME/.local；显式使用该路径，避免 PATH 上
+// 另一个未知版本的 bub 抢先命中。
+const BUB = "$HOME/.local/bin/bub";
 
 // checkpoint 只打 $HOME/.local:uv 装的 python 工具链、bub 的 tool venv 和 bin shim 全在
 // 这里,restore 后即可运行。~/.cache/uv 是 wheel/构建缓存,只在「下一次安装」有用,而
 // restore 场景 bub 已经装好、不会再装——打进去只是把单次 HTTP 传输撑到 100MB+,在 e2b
 // 文件 API 上超时/连接重置概率明显偏高。子目录列表参与 INSTALL_HASH:改它会换缓存文件
 // 名,不会继续复用老的大 checkpoint。
-const CHECKPOINT_SUBDIRS = [".local"];
+const CHECKPOINT_SUBDIRS = BUB_CHECKPOINT_SUBDIRS;
 
 /** 规范化 python plugin:去空白、丢空串、去重 —— 安装命令与 checkpoint key 用同一份列表。 */
 function normalizePackages(plugins?: readonly PythonPluginSpec[]): string[] {
-  const seen = new Set<string>();
-  for (const p of plugins ?? []) {
-    const pkg = p.package.trim();
-    if (pkg) seen.add(pkg);
-  }
-  return [...seen].sort();
-}
-
-/** 安装环境的完整描述:bub 版本 + otel 插件 + python plugin 集合 + checkpoint 覆盖面。 */
-function installSpecOf(packages: readonly string[]): string {
-  const plugins = packages.length ? ` --with ${packages.join(" --with ")}` : "";
-  return `bub --override(${BUB_OVERRIDE}) --with ${OTEL_PLUGIN}${plugins} --checkpoint(${CHECKPOINT_SUBDIRS.join(",")})`;
+  return normalizeBubPackages((plugins ?? []).map((plugin) => plugin.package));
 }
 
 function installHashOf(packages: readonly string[]): string {
-  return createHash("md5").update(installSpecOf(packages)).digest("hex").slice(0, 12);
+  return bubInstallHash(packages, BUB_OVERRIDE, OTEL_PLUGIN);
 }
 
 function diskCachePath(home: string, installHash: string): string {
@@ -127,15 +118,16 @@ async function ensureBub(
   log: AgentContext["log"],
   packages: readonly string[],
 ): Promise<void> {
-  // 预制模板已把 bub 烘焙进镜像(PATH 上)→ 直接用,跳过 uv 安装 + checkpoint 全套。
-  // pinned(git ref override)时不走此捷径:烘焙的 bub 无法验证是不是 ref 当前指向的
-  // 构建,必须按 override 真装(见 BUB_PINNED 注释)。有 python plugin 时同理不能走捷径:
-  // 镜像里烘焙的 bub 环境没有这些包。
-  if (!BUB_PINNED && packages.length === 0 && (await sb.runShell("command -v bub >/dev/null 2>&1")).exitCode === 0) {
+  const installHash = installHashOf(packages);
+  const marker = `${home}/${BUB_INSTALL_MARKER}`;
+  // 只信任带完整安装规格指纹的预制环境。仅 command -v bub 无法证明版本、OTel 插件和
+  // 用户 pythonPlugins 一致；NiceEval 的 E2B Bub 配方和运行时安装都会写这个 marker。
+  if ((await sb.runShell(
+    `test -x ${BUB} && test "$(cat '${marker}' 2>/dev/null)" = '${installHash}'`,
+  )).exitCode === 0) {
     return;
   }
 
-  const installHash = installHashOf(packages);
   const cacheKey = `${home}::${installHash}`;
   const withPlugins = packages.map((p) => `--with '${p}'`).join(" ");
   const checkpointPaths = CHECKPOINT_SUBDIRS.map((d) => `${home}/${d}`);
@@ -190,6 +182,11 @@ async function ensureBub(
         }));
       }
     }
+    const markerDir = marker.slice(0, marker.lastIndexOf("/"));
+    const mark = await sb.runShell(`mkdir -p '${markerDir}' && printf '%s' '${installHash}' > '${marker}'`);
+    if (mark.exitCode !== 0) {
+      throw new Error(`Failed to write Bub installation marker: ${mark.stderr || mark.stdout}`);
+    }
     // 到这里 bub 已装进本沙箱,checkpoint 只是给后续沙箱的缓存回填:capture/下载失败
     // (大 buffer 在 e2b 文件 API 上的瞬态错误)降级为警告,绝不反过来杀掉已就绪的 attempt。
     try {
@@ -224,6 +221,8 @@ export function bubAgent(config?: BubConfig): Agent {
 
   return defineSandboxAgent({
     name: "bub",
+    // 官方 adapter:transcript 经生命周期 fixture 验证,全通道 complete。
+    coverage: completeCoverage,
     spanMapper: mapBubSpans,
 
     tracing: {

package/src/agents/claude-code.test.ts

@@ -1,19 +1,27 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
 // Claude Code native plugin 安装(installPlugins)的单测:单 plugin 的命令构造、同名
-// marketplace 的去重、ref 钉定时改走 clone+checkout+本地路径连接、resolvedVersion 取不到
-// 时优雅省略、marketplace/plugin 安装失败的报错。沙箱是内存 fake,风格与
-// src/agents/skills.test.ts 一致(记命令的 FakeSandbox + 按命令前缀打脚本),不另起一套。
+// marketplace 的去重、add 后的注册名回读校验、ref 钉定时改走 clone+checkout+本地路径连接、
+// resolvedVersion 取不到时优雅省略、marketplace/plugin 安装失败的报错;外加 settingsFile
+// (原生配置文件)的 setup 流。沙箱是内存 fake,风格与 src/agents/skills.test.ts 一致
+// (记命令的 FakeSandbox + 按命令前缀打脚本),不另起一套。
 // 定稿见 docs/feature/adapters/architecture/coding-agent-extensions.md。
 
-import { describe, expect, it } from "vitest";
-import { installPlugins, type ClaudeCodePluginSpec } from "./claude-code.ts";
-import type { CommandResult, Sandbox, SandboxFile } from "../types.ts";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { claudeCodeAgent, installPlugins, type ClaudeCodePluginSpec } from "./claude-code.ts";
+import type { AgentContext, AgentSetupManifest, CommandResult, Sandbox, SandboxFile } from "../types.ts";
 
-/** 内存沙箱:runShell 记命令(可按命令包含的子串打脚本化输出)。 */
+/** 内存沙箱:runShell 记命令(可按命令包含的子串打脚本化输出),uploadFile / writeFiles 记内容。 */
 class FakeSandbox implements Partial<Sandbox> {
   readonly workdir = "/workspace";
   readonly sandboxId = "fake";
   readonly otlpHost = null;
   readonly commands: string[] = [];
+  readonly uploads: { path: string; content: Buffer }[] = [];
+  readonly written: Record<string, string> = {};
   script: { match: string; result: (cmd: string) => Partial<CommandResult> }[] = [];
 
   async runShell(script: string): Promise<CommandResult> {
@@ -21,7 +29,12 @@ class FakeSandbox implements Partial<Sandbox> {
     const hit = this.script.find((s) => script.includes(s.match));
     return { stdout: "", stderr: "", exitCode: 0, ...hit?.result(script) };
   }
-  async writeFiles(): Promise<void> {}
+  async writeFiles(files: Record<string, string>): Promise<void> {
+    Object.assign(this.written, files);
+  }
+  async uploadFile(path: string, content: Buffer): Promise<void> {
+    this.uploads.push({ path, content });
+  }
   async uploadFiles(_files: SandboxFile[]): Promise<void> {}
   async fileExists(): Promise<boolean> {
     return false;
@@ -33,7 +46,11 @@ class FakeSandbox implements Partial<Sandbox> {
 
 const sb = (s?: FakeSandbox["script"]): FakeSandbox => {
   const box = new FakeSandbox();
-  if (s) box.script = s;
+  // add 后的注册名回读默认回到配置名(校验通过);mismatch 用例把自己的脚本放前面覆盖。
+  box.script = [
+    ...(s ?? []),
+    { match: "claude plugin marketplace list --json", result: () => ({ stdout: JSON.stringify([{ name: "acme" }]) }) },
+  ];
   return box;
 };
 const asSandbox = (box: FakeSandbox): Sandbox => box as unknown as Sandbox;
@@ -48,6 +65,7 @@ describe("claude-code installPlugins · 命令构造", () => {
 
     expect(box.commands).toEqual([
       "claude plugin marketplace add 'acme/claude-code-plugins'",
+      "claude plugin marketplace list --json",
       "claude plugin install 'safe-shell@acme'",
       "claude plugin list --json",
     ]);
@@ -150,3 +168,106 @@ describe("claude-code installPlugins · 失败语义", () => {
     ).rejects.toThrow(/safe-shell/);
   });
 });
+
+describe("claude-code installPlugins · marketplace 名回读校验", () => {
+  const plugins: ClaudeCodePluginSpec[] = [
+    { marketplace: { name: "acme", source: "acme/claude-code-plugins" }, name: "safe-shell" },
+  ];
+
+  it("add 静默注册成别的名字(仓库 manifest 的真实 name)→ 立刻抛带两个名字的错误,不走到 plugin install", async () => {
+    const box = sb([
+      {
+        match: "claude plugin marketplace list --json",
+        result: () => ({ stdout: JSON.stringify([{ name: "duyet-claude-plugins" }]) }),
+      },
+    ]);
+    const err = await installPlugins(asSandbox(box), plugins).catch((e: Error) => e);
+    expect(err).toBeInstanceOf(Error);
+    expect((err as Error).message).toContain("acme");
+    expect((err as Error).message).toContain("duyet-claude-plugins");
+    expect(box.commands.some((c) => c.startsWith("claude plugin install"))).toBe(false);
+  });
+
+  it("回读命令失败 → 按回读失败抛错,不静默放行", async () => {
+    const box = sb([
+      { match: "claude plugin marketplace list --json", result: () => ({ exitCode: 1, stderr: "boom" }) },
+    ]);
+    await expect(installPlugins(asSandbox(box), plugins)).rejects.toThrow(/marketplace/);
+  });
+
+  it("回读输出解析不出(未知形状)→ 同样按回读失败抛错", async () => {
+    const box = sb([
+      { match: "claude plugin marketplace list --json", result: () => ({ stdout: "not json" }) },
+    ]);
+    await expect(installPlugins(asSandbox(box), plugins)).rejects.toThrow(/acme/);
+  });
+});
+
+describe("claudeCodeAgent settingsFile · setup", () => {
+  let root: string;
+  let cwdBefore: string;
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), "niceeval-claude-settings-"));
+    cwdBefore = process.cwd();
+    process.chdir(root); // settingsFile 相对项目根 = process.cwd()(docs 定稿口径)
+  });
+
+  afterEach(async () => {
+    process.chdir(cwdBefore);
+    await rm(root, { recursive: true, force: true });
+  });
+
+  const ctx = {} as AgentContext; // claude 的 setup 不读 ctx
+
+  it("原始字节原样上传并 mv 成用户级 ~/.claude/settings.json;manifest 记项目相对路径 + SHA-256,不落正文", async () => {
+    const body = '{\n  "$schema": "https://json.schemastore.org/claude-code-settings.json",\n  "permissions": { "deny": ["WebSearch", "WebFetch"] }\n}\n';
+    await mkdir(join(root, "configs"), { recursive: true });
+    await writeFile(join(root, "configs/no-web.json"), body);
+
+    const box = sb();
+    await claudeCodeAgent({ apiKey: "k", settingsFile: "./configs/no-web.json" }).setup!(asSandbox(box), ctx);
+
+    // 原始字节走 uploadFile(不走会补换行的 heredoc),再 mv 到用户层落点
+    expect(box.uploads).toHaveLength(1);
+    expect(box.uploads[0]!.content.toString("utf8")).toBe(body);
+    const mv = box.commands.find((c) => c.includes("~/.claude/settings.json"))!;
+    expect(mv).toContain(`mv ${box.uploads[0]!.path} ~/.claude/settings.json`);
+
+    const manifest = JSON.parse(box.written["__niceeval__/agent-setup.json"]!) as AgentSetupManifest;
+    expect(manifest.nativeConfigFile).toEqual({
+      agent: "claude-code",
+      path: "configs/no-web.json",
+      sha256: createHash("sha256").update(body).digest("hex"),
+    });
+    expect(box.written["__niceeval__/agent-setup.json"]).not.toContain("permissions");
+  });
+
+  it("保留键(model / env)出现在文件里 → setup 报错点名冲突键,什么都不上传", async () => {
+    await writeFile(join(root, "bad.json"), '{ "model": "opus", "permissions": {} }');
+    const box = sb();
+    await expect(
+      claudeCodeAgent({ apiKey: "k", settingsFile: "bad.json" }).setup!(asSandbox(box), ctx),
+    ).rejects.toThrow(/model/);
+    expect(box.uploads).toHaveLength(0);
+    expect(box.written["__niceeval__/agent-setup.json"]).toBeUndefined();
+  });
+
+  it("路径不存在 / 逃出项目根 → setup 报错(attempt errored 的通道)", async () => {
+    const box = sb();
+    await expect(
+      claudeCodeAgent({ apiKey: "k", settingsFile: "configs/nope.json" }).setup!(asSandbox(box), ctx),
+    ).rejects.toThrow(/configs\/nope\.json/);
+    await expect(
+      claudeCodeAgent({ apiKey: "k", settingsFile: "../escape.json" }).setup!(asSandbox(box), ctx),
+    ).rejects.toThrow(/相对路径|relative/);
+  });
+
+  it("没配 settingsFile 时不碰 settings.json,也不为它写 manifest", async () => {
+    const box = sb();
+    await claudeCodeAgent({ apiKey: "k" }).setup!(asSandbox(box), ctx);
+    expect(box.uploads).toHaveLength(0);
+    expect(box.commands.some((c) => c.includes("settings.json"))).toBe(false);
+    expect(box.written["__niceeval__/agent-setup.json"]).toBeUndefined();
+  });
+});