niceeval 0.6.1 → 0.7.0

package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx}

@@ -36,7 +36,7 @@ pnpm exec niceeval init
 ```
 
 ### Adapter
-Adapter 是连接 Agent 与 [NiceEval](https://niceeval.com/) 的中间层。有三种不同程序的接入方式，从无侵入式到部分修改你的 Agent 代码，来解锁更有用的评估功能。 见[Tier](/zh/concepts/tier)。
+Adapter 是连接 Agent 与 [NiceEval](https://niceeval.com/) 的中间层。有三种不同程序的接入方式，从无侵入式到部分修改你的 Agent 代码，来解锁更有用的评估功能。 见[Tier](/zh/explanation/tier)。
 
 ```ts
 // agents/my-bot.ts
@@ -101,23 +101,11 @@ pnpm exec niceeval view         # 需要交互浏览时打开本地查看器
 
 到这里第一条 eval 已经跑通。
 
-这个最小 adapter 是**单轮**的——第二轮不会记得第一轮说了什么，因为还没告诉应用怎么接上历史。多轮会话和工具调用事件（解锁 `t.calledTool()`）、HITL、tracing 一样，都是之后给 adapter 加的可选增量：顺着[接入你的 Agent](/zh/guides/connect-your-agent)逐层往下走即可，已写的 eval 不用改。
+这个最小 adapter 是**单轮**的——第二轮不会记得第一轮说了什么，因为还没告诉应用怎么接上历史。多轮会话和工具调用事件（解锁 `t.calledTool()`）、HITL、tracing 一样，都是之后给 adapter 加的可选增量：顺着[接入你的 Agent](/zh/how-to/connect-your-agent)逐层往下走即可，已写的 eval 不用改。
 
-## 按场景走
+## 对照完整项目
 
-已经知道自己要评什么的话，直接从对应场景开始：
-
-<CardGroup cols={3}>
-  <Card title="如果你需要 eval 你的 Claude Code / Codex 插件" icon="plug" href="/zh/example/claude-code-codex-plugin">
-    适合插件、Hook、MCP server 和项目级 coding-agent 扩展。
-  </Card>
-  <Card title="如果你需要 eval 你的 Claude Code / Codex Skill" icon="wand-magic-sparkles" href="/zh/example/claude-code-codex-skill">
-    适合验证 Skill 是否被触发、是否按流程执行、是否真的提升任务成功率。
-  </Card>
-  <Card title="如果你需要 eval 你的 AI Agent 应用" icon="globe" href="/zh/example/ai-agent-application">
-    适合 AI SDK、LangGraph、Pi 或自研 agent 应用的完整示例。
-  </Card>
-</CardGroup>
+第一条 Eval 跑通后，再到 [Examples](/zh/examples) 按被测对象选择完整项目。那里提供可运行源码；通用操作步骤仍以 How-to Guides 为准。
 
 ## 放进 CI
 
@@ -135,5 +123,5 @@ jobs:
 ```
 
 <Tip>
-  接下来读 [编写 eval](/zh/guides/authoring) 和 [评分指南](/zh/guides/scoring-guide)，把示例替换成你的真实场景。
+  接下来读 [编写 eval](/zh/how-to/authoring) 和 [评分指南](/zh/how-to/scoring-guide)，把示例替换成你的真实场景。
 </Tip>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "niceeval",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "Agent-native eval tool — eval agents, services, functions, and coding-agent fixtures",
   "type": "module",
   "license": "MIT",
@@ -24,6 +24,10 @@
       "types": "./src/sandbox/index.ts",
       "import": "./src/sandbox/index.ts"
     },
+    "./sandbox/e2b-template": {
+      "types": "./src/sandbox/e2b-agent-template.ts",
+      "import": "./src/sandbox/e2b-agent-template.ts"
+    },
     "./adapter": {
       "types": "./src/agents/index.ts",
       "import": "./src/agents/index.ts"
@@ -56,6 +60,10 @@
       "types": "./dist/report/react/index.d.ts",
       "import": "./dist/report/react/index.js"
     },
+    "./report/built-in": {
+      "types": "./dist/report/built-in/index.d.ts",
+      "import": "./dist/report/built-in/index.js"
+    },
     "./report/react/styles.css": "./src/report/react/styles.css",
     "./report/react/enhance.js": "./src/report/react/enhance.js"
   },
@@ -165,7 +173,7 @@
     "build:report": "tsc -p tsconfig.report-build.json && node scripts/prune-report-dist.mjs",
     "site:dev": "cd site && node scripts/dev.mjs",
     "site:build": "cd site && next build",
-    "docs:dev": "cd docs-site && npx --yes mint@latest dev",
+    "docs:dev": "node scripts/docs-dev.mjs",
     "docs:validate": "cd docs-site && npx --yes mint@latest validate",
     "docs:links": "cd docs-site && npx --yes mint@latest broken-links --check-anchors --check-redirects"
   }

package/src/agents/ai-sdk-otel.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
 import { describe, expect, it } from "vitest";
 
 import { aiSdkOtel } from "./ai-sdk-otel.ts";

package/src/agents/ai-sdk.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
 import { describe, expect, it } from "vitest";
 
 import { aiSdkAgent, fromAiSdk } from "./ai-sdk.ts";
@@ -457,6 +458,8 @@ function fakeCtx(opts: { id?: string } = {}): AgentContext {
     flags: {},
     sandbox: undefined as unknown as AgentContext["sandbox"],
     session,
+    progress: () => {},
+    diagnostic: () => {},
     log: () => {},
   };
 }

package/src/agents/ai-sdk.ts

@@ -13,6 +13,7 @@
 //     (不在 steps 里),这里挖出来补成 `action.result` —— 拒绝(execution-denied)映射成
 //     `status: "rejected"`,喂 `calledTool(..., { status: "rejected" })` 与 `noFailedActions()`。
 
+import { completeCoverage } from "../scoring/coverage.ts";
 import { randomUUID } from "node:crypto";
 
 import { defineAgent } from "../define.ts";
@@ -466,6 +467,8 @@ export function aiSdkAgent<M = unknown>(options: AiSdkAgentOptions<M>): Agent {
 
   return defineAgent({
     name: options.name ?? "ai-sdk",
+    // 官方 SDK 完整 steps/output:全通道 complete(见 adapters/architecture/evidence.md)。
+    coverage: completeCoverage,
     // tracing 开了才让运行器为这个 agent 起 OTLP 接收器(ctx.telemetry 才会出现);
     // AgentTracing 的其余字段(env/configure/scope)这里都用不上,空对象即可。
     tracing: options.tracing ? {} : undefined,

package/src/agents/bub-install-spec.test.ts

@@ -0,0 +1,34 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
+import { describe, expect, it } from "vitest";
+import { readFile } from "node:fs/promises";
+import {
+  BUB_INSTALL_MARKER,
+  DEFAULT_BUB_OTEL_PLUGIN,
+  DEFAULT_BUB_OVERRIDE,
+  bubInstallHash,
+  bubInstallSpec,
+} from "./bub-install-spec.ts";
+
+describe("Bub install specification", () => {
+  it("uses immutable default Git commits", () => {
+    expect(DEFAULT_BUB_OVERRIDE).toMatch(/@[0-9a-f]{40}$/);
+    expect(DEFAULT_BUB_OTEL_PLUGIN).toMatch(/@[0-9a-f]{40}#subdirectory=/);
+  });
+
+  it("fingerprints plugins independent of caller ordering", () => {
+    const packages = ["alpha==1", "beta==2"];
+    expect(bubInstallHash(packages)).toBe(bubInstallHash([" beta==2 ", "alpha==1", "alpha==1"]));
+    expect(bubInstallSpec(packages)).toContain("--with alpha==1 --with beta==2");
+    expect(BUB_INSTALL_MARKER).toBe(".local/share/niceeval/bub-install-hash");
+  });
+
+  it("keeps the non-TypeScript Docker recipe in sync", async () => {
+    const [dockerfile, override] = await Promise.all([
+      readFile(new URL("../../sandbox/docker/Dockerfile", import.meta.url), "utf8"),
+      readFile(new URL("../../sandbox/docker/bub-override.txt", import.meta.url), "utf8"),
+    ]);
+    expect(override.trim()).toBe(DEFAULT_BUB_OVERRIDE);
+    expect(dockerfile).toContain(DEFAULT_BUB_OTEL_PLUGIN);
+    expect(dockerfile).toContain(bubInstallHash([]));
+  });
+});

package/src/agents/bub-install-spec.ts

@@ -0,0 +1,32 @@
+import { createHash } from "node:crypto";
+
+export const DEFAULT_BUB_OVERRIDE =
+  "bub @ git+https://github.com/CorrectRoadH/bub.git@86fbd0febc1665353f5131173554e1f513e66b4c";
+
+export const DEFAULT_BUB_OTEL_PLUGIN =
+  "git+https://github.com/bubbuild/bub-contrib.git@add4a6a133c5658aec8f167ef50804d9ee55d22e#subdirectory=packages/bub-tapestore-otel";
+
+export const BUB_CHECKPOINT_SUBDIRS = [".local"] as const;
+export const BUB_INSTALL_MARKER = ".local/share/niceeval/bub-install-hash";
+
+export function normalizeBubPackages(packages: readonly string[]): string[] {
+  return [...new Set(packages.map((value) => value.trim()).filter(Boolean))].sort();
+}
+
+export function bubInstallSpec(
+  packages: readonly string[],
+  override = DEFAULT_BUB_OVERRIDE,
+  otelPlugin = DEFAULT_BUB_OTEL_PLUGIN,
+): string {
+  const normalized = normalizeBubPackages(packages);
+  const plugins = normalized.length ? ` --with ${normalized.join(" --with ")}` : "";
+  return `bub --override(${override}) --with ${otelPlugin}${plugins} --checkpoint(${BUB_CHECKPOINT_SUBDIRS.join(",")})`;
+}
+
+export function bubInstallHash(
+  packages: readonly string[],
+  override = DEFAULT_BUB_OVERRIDE,
+  otelPlugin = DEFAULT_BUB_OTEL_PLUGIN,
+): string {
+  return createHash("md5").update(bubInstallSpec(packages, override, otelPlugin)).digest("hex").slice(0, 12);
+}

package/src/agents/bub.ts

@@ -1,3 +1,4 @@
+import { completeCoverage } from "../scoring/coverage.ts";
 import { defineSandboxAgent } from "../define.ts";
 import { requireEnv, getEnv } from "../util.ts";
 import { shared } from "./shared.ts";
@@ -16,6 +17,14 @@ import { readFile, writeFile, mkdir } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join, dirname } from "node:path";
 import { t } from "../i18n/index.ts";
+import {
+  BUB_CHECKPOINT_SUBDIRS,
+  BUB_INSTALL_MARKER,
+  DEFAULT_BUB_OTEL_PLUGIN,
+  DEFAULT_BUB_OVERRIDE,
+  bubInstallHash,
+  normalizeBubPackages,
+} from "./bub-install-spec.ts";
 
 // ───────────────────────────────────────────────────────────────────────────
 // bub 的 agent adapter(沙箱型)。
@@ -64,50 +73,32 @@ const SKILL_DIR = ".agents/skills";
 // 可用 NICEEVAL_BUB_OVERRIDE / NICEEVAL_BUB_OTEL_PLUGIN 覆盖,不必改源码。
 const BUB_OVERRIDE =
   getEnv("NICEEVAL_BUB_OVERRIDE") ??
-  "bub @ git+https://github.com/CorrectRoadH/bub.git@fix/tape-assistant-text-with-tool-calls";
+  DEFAULT_BUB_OVERRIDE;
 const BUB_OVERRIDE_FILE = "/tmp/bub-override.txt";
 // otel 插件跟上游 main 走(bub-contrib#50 起从 bub.tape 导入,要求 bub ≥ 0.3.10dev,
 // 与上面的 override 分支兼容)。插件不发 PyPI,git 依赖是唯一安装方式。
 const OTEL_PLUGIN =
   getEnv("NICEEVAL_BUB_OTEL_PLUGIN") ??
-  "git+https://github.com/bubbuild/bub-contrib.git#subdirectory=packages/bub-tapestore-otel";
+  DEFAULT_BUB_OTEL_PLUGIN;
 
-// override 钉在 git ref 上时,镜像里烘焙的 bub 不可信:模板构建时间早于 ref 当前指向的
-// commit 的话,`command -v bub` 命中的就是修复前的旧构建 —— e2b 模板 fasteval-agents 上
-// 整轮 bub turn failed(send 后无 AI 回复)就是这么来的:override 分支修好了,但捷径
-// 让它从未被安装。所以 pinned 时绕开 PATH 捷径、恒走 uv 安装(有 checkpoint 缓存,
-// 不是每沙箱都全量装),且运行时钉死用 $HOME/.local/bin/bub —— 只改 ensureBub 不改这里
-// 的话,PATH 上的 /usr/local/bin/bub 仍会先于新装的被 command -v 找到,白装。
-const BUB_PINNED = BUB_OVERRIDE.includes("git+");
-// bub 二进制:pinned → 恒用 uv 装到 $HOME/.local/bin 的那个;非 pinned → 优先用镜像里
-// (预制模板)烘焙在 PATH 上的 bub(装到 /usr/local/bin,见 sandbox/docker/Dockerfile)。
-const BUB = BUB_PINNED ? "$HOME/.local/bin/bub" : "$(command -v bub || echo $HOME/.local/bin/bub)";
+// NiceEval 的预制配方与运行时安装都写到 $HOME/.local；显式使用该路径，避免 PATH 上
+// 另一个未知版本的 bub 抢先命中。
+const BUB = "$HOME/.local/bin/bub";
 
 // checkpoint 只打 $HOME/.local:uv 装的 python 工具链、bub 的 tool venv 和 bin shim 全在
 // 这里,restore 后即可运行。~/.cache/uv 是 wheel/构建缓存,只在「下一次安装」有用,而
 // restore 场景 bub 已经装好、不会再装——打进去只是把单次 HTTP 传输撑到 100MB+,在 e2b
 // 文件 API 上超时/连接重置概率明显偏高。子目录列表参与 INSTALL_HASH:改它会换缓存文件
 // 名,不会继续复用老的大 checkpoint。
-const CHECKPOINT_SUBDIRS = [".local"];
+const CHECKPOINT_SUBDIRS = BUB_CHECKPOINT_SUBDIRS;
 
 /** 规范化 python plugin:去空白、丢空串、去重 —— 安装命令与 checkpoint key 用同一份列表。 */
 function normalizePackages(plugins?: readonly PythonPluginSpec[]): string[] {
-  const seen = new Set<string>();
-  for (const p of plugins ?? []) {
-    const pkg = p.package.trim();
-    if (pkg) seen.add(pkg);
-  }
-  return [...seen].sort();
-}
-
-/** 安装环境的完整描述:bub 版本 + otel 插件 + python plugin 集合 + checkpoint 覆盖面。 */
-function installSpecOf(packages: readonly string[]): string {
-  const plugins = packages.length ? ` --with ${packages.join(" --with ")}` : "";
-  return `bub --override(${BUB_OVERRIDE}) --with ${OTEL_PLUGIN}${plugins} --checkpoint(${CHECKPOINT_SUBDIRS.join(",")})`;
+  return normalizeBubPackages((plugins ?? []).map((plugin) => plugin.package));
 }
 
 function installHashOf(packages: readonly string[]): string {
-  return createHash("md5").update(installSpecOf(packages)).digest("hex").slice(0, 12);
+  return bubInstallHash(packages, BUB_OVERRIDE, OTEL_PLUGIN);
 }
 
 function diskCachePath(home: string, installHash: string): string {
@@ -127,15 +118,16 @@ async function ensureBub(
   log: AgentContext["log"],
   packages: readonly string[],
 ): Promise<void> {
-  // 预制模板已把 bub 烘焙进镜像(PATH 上)→ 直接用,跳过 uv 安装 + checkpoint 全套。
-  // pinned(git ref override)时不走此捷径:烘焙的 bub 无法验证是不是 ref 当前指向的
-  // 构建,必须按 override 真装(见 BUB_PINNED 注释)。有 python plugin 时同理不能走捷径:
-  // 镜像里烘焙的 bub 环境没有这些包。
-  if (!BUB_PINNED && packages.length === 0 && (await sb.runShell("command -v bub >/dev/null 2>&1")).exitCode === 0) {
+  const installHash = installHashOf(packages);
+  const marker = `${home}/${BUB_INSTALL_MARKER}`;
+  // 只信任带完整安装规格指纹的预制环境。仅 command -v bub 无法证明版本、OTel 插件和
+  // 用户 pythonPlugins 一致；NiceEval 的 E2B Bub 配方和运行时安装都会写这个 marker。
+  if ((await sb.runShell(
+    `test -x ${BUB} && test "$(cat '${marker}' 2>/dev/null)" = '${installHash}'`,
+  )).exitCode === 0) {
     return;
   }
 
-  const installHash = installHashOf(packages);
   const cacheKey = `${home}::${installHash}`;
   const withPlugins = packages.map((p) => `--with '${p}'`).join(" ");
   const checkpointPaths = CHECKPOINT_SUBDIRS.map((d) => `${home}/${d}`);
@@ -190,6 +182,11 @@ async function ensureBub(
         }));
       }
     }
+    const markerDir = marker.slice(0, marker.lastIndexOf("/"));
+    const mark = await sb.runShell(`mkdir -p '${markerDir}' && printf '%s' '${installHash}' > '${marker}'`);
+    if (mark.exitCode !== 0) {
+      throw new Error(`Failed to write Bub installation marker: ${mark.stderr || mark.stdout}`);
+    }
     // 到这里 bub 已装进本沙箱,checkpoint 只是给后续沙箱的缓存回填:capture/下载失败
     // (大 buffer 在 e2b 文件 API 上的瞬态错误)降级为警告,绝不反过来杀掉已就绪的 attempt。
     try {
@@ -224,6 +221,8 @@ export function bubAgent(config?: BubConfig): Agent {
 
   return defineSandboxAgent({
     name: "bub",
+    // 官方 adapter:transcript 经生命周期 fixture 验证,全通道 complete。
+    coverage: completeCoverage,
     spanMapper: mapBubSpans,
 
     tracing: {

package/src/agents/claude-code.test.ts

@@ -1,19 +1,27 @@
+// cases: docs/engineering/unit-tests/adapters/cases.md
 // Claude Code native plugin 安装(installPlugins)的单测:单 plugin 的命令构造、同名
-// marketplace 的去重、ref 钉定时改走 clone+checkout+本地路径连接、resolvedVersion 取不到
-// 时优雅省略、marketplace/plugin 安装失败的报错。沙箱是内存 fake,风格与
-// src/agents/skills.test.ts 一致(记命令的 FakeSandbox + 按命令前缀打脚本),不另起一套。
+// marketplace 的去重、add 后的注册名回读校验、ref 钉定时改走 clone+checkout+本地路径连接、
+// resolvedVersion 取不到时优雅省略、marketplace/plugin 安装失败的报错;外加 settingsFile
+// (原生配置文件)的 setup 流。沙箱是内存 fake,风格与 src/agents/skills.test.ts 一致
+// (记命令的 FakeSandbox + 按命令前缀打脚本),不另起一套。
 // 定稿见 docs/feature/adapters/architecture/coding-agent-extensions.md。
 
-import { describe, expect, it } from "vitest";
-import { installPlugins, type ClaudeCodePluginSpec } from "./claude-code.ts";
-import type { CommandResult, Sandbox, SandboxFile } from "../types.ts";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { claudeCodeAgent, installPlugins, type ClaudeCodePluginSpec } from "./claude-code.ts";
+import type { AgentContext, AgentSetupManifest, CommandResult, Sandbox, SandboxFile } from "../types.ts";
 
-/** 内存沙箱:runShell 记命令(可按命令包含的子串打脚本化输出)。 */
+/** 内存沙箱:runShell 记命令(可按命令包含的子串打脚本化输出),uploadFile / writeFiles 记内容。 */
 class FakeSandbox implements Partial<Sandbox> {
   readonly workdir = "/workspace";
   readonly sandboxId = "fake";
   readonly otlpHost = null;
   readonly commands: string[] = [];
+  readonly uploads: { path: string; content: Buffer }[] = [];
+  readonly written: Record<string, string> = {};
   script: { match: string; result: (cmd: string) => Partial<CommandResult> }[] = [];
 
   async runShell(script: string): Promise<CommandResult> {
@@ -21,7 +29,12 @@ class FakeSandbox implements Partial<Sandbox> {
     const hit = this.script.find((s) => script.includes(s.match));
     return { stdout: "", stderr: "", exitCode: 0, ...hit?.result(script) };
   }
-  async writeFiles(): Promise<void> {}
+  async writeFiles(files: Record<string, string>): Promise<void> {
+    Object.assign(this.written, files);
+  }
+  async uploadFile(path: string, content: Buffer): Promise<void> {
+    this.uploads.push({ path, content });
+  }
   async uploadFiles(_files: SandboxFile[]): Promise<void> {}
   async fileExists(): Promise<boolean> {
     return false;
@@ -33,7 +46,11 @@ class FakeSandbox implements Partial<Sandbox> {
 
 const sb = (s?: FakeSandbox["script"]): FakeSandbox => {
   const box = new FakeSandbox();
-  if (s) box.script = s;
+  // add 后的注册名回读默认回到配置名(校验通过);mismatch 用例把自己的脚本放前面覆盖。
+  box.script = [
+    ...(s ?? []),
+    { match: "claude plugin marketplace list --json", result: () => ({ stdout: JSON.stringify([{ name: "acme" }]) }) },
+  ];
   return box;
 };
 const asSandbox = (box: FakeSandbox): Sandbox => box as unknown as Sandbox;
@@ -48,6 +65,7 @@ describe("claude-code installPlugins · 命令构造", () => {
 
     expect(box.commands).toEqual([
       "claude plugin marketplace add 'acme/claude-code-plugins'",
+      "claude plugin marketplace list --json",
       "claude plugin install 'safe-shell@acme'",
       "claude plugin list --json",
     ]);
@@ -150,3 +168,106 @@ describe("claude-code installPlugins · 失败语义", () => {
     ).rejects.toThrow(/safe-shell/);
   });
 });
+
+describe("claude-code installPlugins · marketplace 名回读校验", () => {
+  const plugins: ClaudeCodePluginSpec[] = [
+    { marketplace: { name: "acme", source: "acme/claude-code-plugins" }, name: "safe-shell" },
+  ];
+
+  it("add 静默注册成别的名字(仓库 manifest 的真实 name)→ 立刻抛带两个名字的错误,不走到 plugin install", async () => {
+    const box = sb([
+      {
+        match: "claude plugin marketplace list --json",
+        result: () => ({ stdout: JSON.stringify([{ name: "duyet-claude-plugins" }]) }),
+      },
+    ]);
+    const err = await installPlugins(asSandbox(box), plugins).catch((e: Error) => e);
+    expect(err).toBeInstanceOf(Error);
+    expect((err as Error).message).toContain("acme");
+    expect((err as Error).message).toContain("duyet-claude-plugins");
+    expect(box.commands.some((c) => c.startsWith("claude plugin install"))).toBe(false);
+  });
+
+  it("回读命令失败 → 按回读失败抛错,不静默放行", async () => {
+    const box = sb([
+      { match: "claude plugin marketplace list --json", result: () => ({ exitCode: 1, stderr: "boom" }) },
+    ]);
+    await expect(installPlugins(asSandbox(box), plugins)).rejects.toThrow(/marketplace/);
+  });
+
+  it("回读输出解析不出(未知形状)→ 同样按回读失败抛错", async () => {
+    const box = sb([
+      { match: "claude plugin marketplace list --json", result: () => ({ stdout: "not json" }) },
+    ]);
+    await expect(installPlugins(asSandbox(box), plugins)).rejects.toThrow(/acme/);
+  });
+});
+
+describe("claudeCodeAgent settingsFile · setup", () => {
+  let root: string;
+  let cwdBefore: string;
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), "niceeval-claude-settings-"));
+    cwdBefore = process.cwd();
+    process.chdir(root); // settingsFile 相对项目根 = process.cwd()(docs 定稿口径)
+  });
+
+  afterEach(async () => {
+    process.chdir(cwdBefore);
+    await rm(root, { recursive: true, force: true });
+  });
+
+  const ctx = {} as AgentContext; // claude 的 setup 不读 ctx
+
+  it("原始字节原样上传并 mv 成用户级 ~/.claude/settings.json;manifest 记项目相对路径 + SHA-256,不落正文", async () => {
+    const body = '{\n  "$schema": "https://json.schemastore.org/claude-code-settings.json",\n  "permissions": { "deny": ["WebSearch", "WebFetch"] }\n}\n';
+    await mkdir(join(root, "configs"), { recursive: true });
+    await writeFile(join(root, "configs/no-web.json"), body);
+
+    const box = sb();
+    await claudeCodeAgent({ apiKey: "k", settingsFile: "./configs/no-web.json" }).setup!(asSandbox(box), ctx);
+
+    // 原始字节走 uploadFile(不走会补换行的 heredoc),再 mv 到用户层落点
+    expect(box.uploads).toHaveLength(1);
+    expect(box.uploads[0]!.content.toString("utf8")).toBe(body);
+    const mv = box.commands.find((c) => c.includes("~/.claude/settings.json"))!;
+    expect(mv).toContain(`mv ${box.uploads[0]!.path} ~/.claude/settings.json`);
+
+    const manifest = JSON.parse(box.written["__niceeval__/agent-setup.json"]!) as AgentSetupManifest;
+    expect(manifest.nativeConfigFile).toEqual({
+      agent: "claude-code",
+      path: "configs/no-web.json",
+      sha256: createHash("sha256").update(body).digest("hex"),
+    });
+    expect(box.written["__niceeval__/agent-setup.json"]).not.toContain("permissions");
+  });
+
+  it("保留键(model / env)出现在文件里 → setup 报错点名冲突键,什么都不上传", async () => {
+    await writeFile(join(root, "bad.json"), '{ "model": "opus", "permissions": {} }');
+    const box = sb();
+    await expect(
+      claudeCodeAgent({ apiKey: "k", settingsFile: "bad.json" }).setup!(asSandbox(box), ctx),
+    ).rejects.toThrow(/model/);
+    expect(box.uploads).toHaveLength(0);
+    expect(box.written["__niceeval__/agent-setup.json"]).toBeUndefined();
+  });
+
+  it("路径不存在 / 逃出项目根 → setup 报错(attempt errored 的通道)", async () => {
+    const box = sb();
+    await expect(
+      claudeCodeAgent({ apiKey: "k", settingsFile: "configs/nope.json" }).setup!(asSandbox(box), ctx),
+    ).rejects.toThrow(/configs\/nope\.json/);
+    await expect(
+      claudeCodeAgent({ apiKey: "k", settingsFile: "../escape.json" }).setup!(asSandbox(box), ctx),
+    ).rejects.toThrow(/相对路径|relative/);
+  });
+
+  it("没配 settingsFile 时不碰 settings.json,也不为它写 manifest", async () => {
+    const box = sb();
+    await claudeCodeAgent({ apiKey: "k" }).setup!(asSandbox(box), ctx);
+    expect(box.uploads).toHaveLength(0);
+    expect(box.commands.some((c) => c.includes("settings.json"))).toBe(false);
+    expect(box.written["__niceeval__/agent-setup.json"]).toBeUndefined();
+  });
+});

package/src/agents/claude-code.ts

@@ -1,10 +1,19 @@
+import { completeCoverage } from "../scoring/coverage.ts";
 import { defineSandboxAgent } from "../define.ts";
 import { requireEnv, getEnv } from "../util.ts";
 import { shared } from "./shared.ts";
 import { cloneRepo, installSkills } from "./skills.ts";
 import { writeAgentSetupManifest } from "./manifest.ts";
+import { verifyMarketplaceName } from "./marketplace.ts";
+import {
+  assertJsonNativeConfig,
+  loadNativeConfigFile,
+  uploadNativeConfigFile,
+  type LoadedNativeConfig,
+} from "./native-config.ts";
 import { mapClaudeCodeSpans } from "../o11y/otlp/mappers/claude-code.ts";
 import { t } from "../i18n/index.ts";
+import { DEFAULT_CLAUDE_CODE_CLI_VERSION } from "./coding-cli-versions.ts";
 import type { Agent, AgentSetupManifest, McpServer, Sandbox, SkillSpec } from "../types.ts";
 
 // ───────────────────────────────────────────────────────────────────────────
@@ -18,6 +27,15 @@ import type { Agent, AgentSetupManifest, McpServer, Sandbox, SkillSpec } from ".
 /** Claude Code 的 skill 目录(project 级):CLI 原生扫描它,不需要额外的发现指引。 */
 const SKILL_DIR = ".claude/skills";
 
+/** 沙箱里用户级 settings 的落点:`settingsFile` 的原始字节原样替换这份原本为空的用户层。 */
+const SETTINGS_PATH = "~/.claude/settings.json";
+
+/**
+ * `settingsFile` 的保留键:`model` 归 experiment(经 `--model` flag),`env` 归 Adapter
+ * (鉴权与 OTel 导出经进程环境变量注入)。清单定稿见 docs/feature/adapters/sdk/claude-code/README.md。
+ */
+const RESERVED_SETTINGS_KEYS = ["model", "env"] as const;
+
 /**
  * Claude Code 的原生 Plugin —— **只属于 Claude Code**,不能传给 Codex(Codex 有自己的
  * {@link import("./codex.ts").CodexPluginSpec})。每一项同时声明 Marketplace 连接和其中的 Plugin 名:
@@ -61,6 +79,15 @@ export interface ClaudeCodeConfig {
   skills?: SkillSpec[];
   /** Claude Code 原生 Plugin(先连 Marketplace,再从中装指定 Plugin)。 */
   plugins?: ClaudeCodePluginSpec[];
+  /**
+   * 一份完整的 Claude Code `settings.json`(官方格式)在本地项目里的路径 —— 相对运行
+   * niceeval 的项目根(含 `niceeval.config.ts` 的目录)解析,不是 Sandbox 内路径;只接受
+   * 项目根内的相对路径,包含 `..` 的路径、绝对路径、`~` 路径和解析后逃出项目根的符号链接
+   * 都在 setup 阶段报错。原始字节原样上传为沙箱里原本为空的用户级 `~/.claude/settings.json`
+   * (不继承宿主机配置、不拼接、不重新序列化);保留键 `model` 与 `env` 出现在文件里
+   * setup 报错。manifest 只记项目相对路径与字节 SHA-256,不落正文。
+   */
+  settingsFile?: string;
 }
 
 export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
@@ -69,6 +96,8 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
 
   return defineSandboxAgent({
     name: "claude-code",
+    // 官方 adapter:transcript 经生命周期 fixture 验证,全通道 complete。
+    coverage: completeCoverage,
     spanMapper: mapClaudeCodeSpans,
 
     // claude CLI 原生 OTLP trace spans(beta):interaction / llm_request / tool 层级,
@@ -88,7 +117,27 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
 
     async setup(sb) {
       // 预制模板已把 claude 烘焙进镜像(PATH 上)就跳过安装;否则 npm 全局装。
-      await sb.runShell("command -v claude >/dev/null 2>&1 || npm install -g @anthropic-ai/claude-code");
+      await sb.runShell(
+        `command -v claude >/dev/null 2>&1 || npm install -g @anthropic-ai/claude-code@${DEFAULT_CLAUDE_CODE_CLI_VERSION}`,
+      );
+
+      // 原生配置文件最先落(安装顺序契约的第 1 步):本地读原始字节 → 验 JSON 语法与保留键
+      // → 原样替换沙箱里原本为空的用户级 settings.json。字节 SHA-256 进 manifest 与安装
+      // checkpoint key(见 native-config.ts 的 nativeConfigCheckpointItem)。
+      let settings: LoadedNativeConfig | undefined;
+      if (config?.settingsFile !== undefined) {
+        settings = await loadNativeConfigFile({
+          agent: "claude-code",
+          field: "settingsFile",
+          path: config.settingsFile,
+        });
+        assertJsonNativeConfig(settings, {
+          agent: "claude-code",
+          field: "settingsFile",
+          reservedKeys: RESERVED_SETTINGS_KEYS,
+        });
+        await uploadNativeConfigFile(sb, settings, SETTINGS_PATH);
+      }
 
       if (config?.mcpServers?.length) {
         const servers: Record<string, object> = {};
@@ -119,8 +168,17 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
           ...(s.args?.length ? { args: [...s.args] } : {}),
         }));
       }
+      if (settings) {
+        // 只记来源路径与字节哈希,不落正文(任意官方配置都可能带敏感字符串)。
+        manifest.nativeConfigFile = { agent: "claude-code", path: settings.path, sha256: settings.sha256 };
+      }
       // 什么都没装就不写 manifest:空 artifact 不落文件(同 results 的落盘规则)。
-      if (manifest.skills.length || manifest.nativePlugins?.length || manifest.mcpServers?.length) {
+      if (
+        manifest.skills.length ||
+        manifest.nativePlugins?.length ||
+        manifest.mcpServers?.length ||
+        manifest.nativeConfigFile
+      ) {
         await writeAgentSetupManifest(sb, manifest);
       }
     },
@@ -134,7 +192,12 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
       if (ctx.session.id) args.push("--resume", ctx.session.id);
       args.push(input.text);
 
-      const env: Record<string, string> = { ANTHROPIC_API_KEY: getApiKey(), ...ctx.telemetry?.env };
+      const env: Record<string, string> = {
+        ANTHROPIC_API_KEY: getApiKey(),
+        // Eval runs must not silently change CLI version after the sandbox artifact was built.
+        DISABLE_AUTOUPDATER: "1",
+        ...ctx.telemetry?.env,
+      };
       const baseUrl = getBaseUrl();
       if (baseUrl) env["ANTHROPIC_BASE_URL"] = baseUrl;
 
@@ -153,7 +216,8 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
 }
 
 /**
- * 先按 `marketplace.name` 建立 Marketplace 连接(同名只连一次),再从该连接装指定 Plugin。
+ * 先按 `marketplace.name` 建立 Marketplace 连接(同名只连一次,add 后回读注册列表校验
+ * 名字真的注册上了),再从该连接装指定 Plugin。
  * `claude plugin marketplace add` 没有钉 ref 的入口,所以要钉 ref 时先自己按 ref clone 下来,
  * 再以本地路径连接(CLI 支持 path 形态的 marketplace 源)—— 「来源必须可复现」不因 CLI 少个
  * flag 就打折。
@@ -183,6 +247,14 @@ export async function installPlugins(
           }),
         );
       }
+      // add 静默按目标仓库 manifest 的 name 注册,错名会拖到 plugin install 才炸;
+      // 回读注册列表立刻校验(契约与真机复现见 marketplace.ts 顶部说明)。
+      await verifyMarketplaceName(sb, {
+        agent: "claude-code",
+        listCommand: "claude plugin marketplace list --json",
+        marketplace,
+        knownNames: connected,
+      });
       connected.add(marketplace.name);
     }