niceeval 0.8.1 → 0.9.1

package/src/runner/ledger.test.ts

@@ -1,4 +1,4 @@
-// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+// cases: docs/engineering/unit-tests/sandbox/cases.md
 // 变更分类账的集成测试:用宿主 shell 扮演沙箱(真实 git),验证
 // - .git 不在 workdir 内(agent 看不到分类账;eval 自己 git init 不冲突)
 // - eval 归因(send 前写入)不进 agent diff;send 窗口内写入逐窗口归因
@@ -153,6 +153,10 @@ describe("createChangeLedger", () => {
     await mkdir(join(workdir, "node_modules"), { recursive: true });
     await writeFile(join(workdir, "node_modules", "dep.js"), "excluded\n");
     await writeFile(join(workdir, "node_modules", "keep.js"), "included back\n");
+    await mkdir(join(workdir, "packages/app/node_modules/dep"), { recursive: true });
+    await writeFile(join(workdir, "packages/app/node_modules/dep/index.js"), "nested dependency\n");
+    await mkdir(join(workdir, "packages/app/__pycache__"), { recursive: true });
+    await writeFile(join(workdir, "packages/app/__pycache__/mod.pyc"), "nested cache\n");
     await mkdir(join(workdir, "secret"), { recursive: true });
     await writeFile(join(workdir, "secret", "token.txt"), "excluded via ignore\n");
     // Python 工具链目录不依赖项目 .gitignore:任意 *venv*/ 名字都由 runner 私有清单排除。
@@ -167,10 +171,35 @@ describe("createChangeLedger", () => {
     expect(paths).toContain("output.txt");
     expect(paths).toContain("node_modules/keep.js");
     expect(paths).not.toContain("node_modules/dep.js");
+    expect(paths).not.toContain("packages/app/node_modules/dep/index.js");
+    expect(paths).not.toContain("packages/app/__pycache__/mod.pyc");
     expect(paths).not.toContain("secret/token.txt");
     expect(paths.some((path) => path.includes("venv"))).toBe(false);
   });
 
+  // bug: memory/ledger-gitignore-pathspec-and-gitlinks.md
+  it("未排除的 nested repo 明确失败；整目录 ignore 后允许作为无关环境存在", async () => {
+    const first = await makeDirs();
+    const checkout = join(first.workdir, "checkout");
+    await mkdir(checkout, { recursive: true });
+    await execAsync("git init -q && git config user.email t@t && git config user.name t", { cwd: checkout });
+    await writeFile(join(checkout, "app.py"), "print('hello')\n");
+    await execAsync("git add app.py && git commit -qm baseline", { cwd: checkout });
+
+    await expect(createChangeLedger(hostSandbox(first.workdir, first.ledgerDir))).rejects.toThrow(
+      /nested Git repository checkout.*sandbox\.workdir root.*diff.*ignore/,
+    );
+
+    const second = await makeDirs();
+    const ignoredCheckout = join(second.workdir, "checkout");
+    await mkdir(ignoredCheckout, { recursive: true });
+    await execAsync("git init -q && git config user.email t@t && git config user.name t", { cwd: ignoredCheckout });
+    await writeFile(join(ignoredCheckout, "app.py"), "print('ignored')\n");
+    await execAsync("git add app.py && git commit -qm baseline", { cwd: ignoredCheckout });
+
+    await expect(createChangeLedger(hostSandbox(second.workdir, second.ledgerDir), { ignore: ["checkout/"] })).resolves.toBeDefined();
+  });
+
   it("整相导出只用一条 shell 命令 + 一次文件下载,不随文件数与窗口数增长", async () => {
     const { workdir, ledgerDir } = await makeDirs();
     const counters = { shells: [] as string[], downloads: [] as string[] };

package/src/runner/ledger.ts

@@ -123,6 +123,23 @@ function shellQuote(s: string): string {
   return `'${s.replaceAll("'", `'\\''`)}'`;
 }
 
+/**
+ * 把仓库根语义的 gitignore 风格规则编译成 ledger pathspec。
+ * 无斜杠的名字匹配任意深度；有斜杠的规则相对 workdir 根；目录本身与后代一起处理。
+ */
+function gitignorePathspecs(pattern: string, exclude: boolean): string[] {
+  let normalized = pattern;
+  while (normalized.startsWith("./")) normalized = normalized.slice(2);
+  if (normalized.startsWith("/")) normalized = normalized.slice(1);
+  normalized = normalized.replace(/\/+$/, "");
+  if (!normalized) return [];
+
+  const glob = normalized.includes("/") ? normalized : `**/${normalized}`;
+  const globs = exclude && !glob.endsWith("/**") ? [glob, `${glob}/**`] : [glob];
+  const magic = exclude ? ":(glob,exclude)" : ":(glob)";
+  return [...new Set(globs)].map((value) => `${magic}${value}`);
+}
+
 /** 打分类账锚点(workspace.baseline 阶段,环境层钩子之后):git init + 冻结排除清单 + 首笔 commit。 */
 export async function createChangeLedger(sandbox: Sandbox, opts?: LedgerOptions): Promise<ChangeLedger> {
   const excludes = [...DEFAULT_EXCLUDES, ...(opts?.ignore ?? [])];
@@ -131,11 +148,15 @@ export async function createChangeLedger(sandbox: Sandbox, opts?: LedgerOptions)
 
   // add -A -f:绕过项目自己的 .gitignore(项目 ignore 的文件照常记录);排除靠 pathspec
   // (runner 私有清单,agent / fixture 写 .gitignore 影响不了它);include 用第二次 add 打洞加回。
-  const excludeSpecs = excludes.map((e) => shellQuote(`:(exclude)${e}`)).join(" ");
+  const excludeSpecs = excludes.flatMap((pattern) => gitignorePathspecs(pattern, true)).map(shellQuote).join(" ");
   // include 打洞:路径此刻可能还不存在(如 agent 之后才写),unmatched pathspec 不算错。
+  const includeSpecs = includes.flatMap((pattern) => gitignorePathspecs(pattern, false)).map(shellQuote).join(" ");
   const includeAdd =
-    includes.length > 0 ? ` && { git add -A -f -- ${includes.map(shellQuote).join(" ")} 2>/dev/null || true; }` : "";
-  const addAll = `git add -A -f -- . ${excludeSpecs}${includeAdd}`;
+    includeSpecs.length > 0 ? ` && { git -c advice.addEmbeddedRepo=false add -A -f -- ${includeSpecs} 2>/dev/null || true; }` : "";
+  const rejectGitlinks =
+    " && nested=$(git ls-files --stage | awk '$1 == \"160000\" { sub(/^[^\\t]*\\t/, \"\"); print; exit }')" +
+    ' && if [ -n "$nested" ]; then printf \'%s\\n\' "niceeval ledger cannot track nested Git repository $nested as file-level evidence; move the checkout to sandbox.workdir root, or add the whole path to defineEval({ diff: { ignore: [...] } }) when it is intentionally out of scope" >&2; exit 2; fi';
+  const addAll = `git -c advice.addEmbeddedRepo=false add -A -f -- . ${excludeSpecs}${includeAdd}${rejectGitlinks}`;
 
   const anchor = await sandbox.runShell(`git init -q "${LEDGER_GIT_DIR}" && ${addAll} && git commit -q --allow-empty -m "anchor"`, {
     env,
@@ -170,7 +191,8 @@ async function exportAgentWindows(sandbox: Sandbox, env: Record<string, string>)
 
 function ensureCommandSucceeded(result: { exitCode: number; stderr: string }, operation: string): void {
   if (result.exitCode === 0) return;
-  const detail = result.stderr.trim().split("\n")[0];
+  // git 可能先输出 advisory warning，再输出 niceeval 的可操作诊断；最后一行最接近失败根因。
+  const detail = result.stderr.trim().split("\n").at(-1);
   throw new Error(`${operation} failed (exit ${result.exitCode})${detail ? `: ${detail}` : ""}`);
 }
 

package/src/runner/run.ts

@@ -21,6 +21,7 @@ import {
 import { failureDetailFromResult } from "./feedback/failure.ts";
 import { encodeAttemptLocator, type AttemptLocator } from "../results/locator.ts";
 import { runWho } from "./types.ts";
+import { prepareRunSandboxes, sandboxForEval } from "./sandbox-selection.ts";
 import type { Agent, EvalResult, JudgeConfig, Reporter, ReporterRegistration, RunShape, RunSummary } from "../types.ts";
 import type { AgentRun, Attempt, LifecyclePhase, AttemptRef, RunOptions } from "./types.ts";
 
@@ -69,6 +70,8 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   const snapshotStartedAt = startedAt;
   const t0 = Date.now();
 
+  prepareRunSandboxes(opts.evals, opts.agentRuns, opts.config.sandbox);
+
   // 按 sourcePath 缓存文件内容,fingerprint 与 judge 预检共用:
   // 矩阵大时(实验 × eval)规划阶段不做串行重复文件读。
   const sourceCache = new Map<string, Promise<string>>();
@@ -88,7 +91,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   // (cli.ts 在 --force 时不传 priorResults,也不算 carryPlan)。
   // carryPlan 优先用调用方(cli.ts,为了 live 表格)已经算好的那份,不重算一遍。
   const { plannedFingerprints, priorRunKeys, carriedResults } =
-    opts.carryPlan ?? (await planCarry(opts.evals, opts.agentRuns, opts.priorResults));
+    opts.carryPlan ?? (await planCarry(opts.evals, opts.agentRuns, opts.priorResults, opts.config.sandbox));
 
   // 携入覆盖计数:priorRunKeys 只回答「这个 (experimentId, evalId) 组合有没有可携入的终态
   // 结果」,不回答「携入了几条」。runs 被调大(或实验改成更大的 runs)时,上次可能只留下比
@@ -131,6 +134,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
           attempt: i,
           key,
           fingerprint: plannedFingerprints.get(cacheKey(run, evalDef.id)) ?? "",
+          sandboxSpec: sandboxForEval(run, evalDef, opts.config.sandbox),
           // locator 在构造 fresh attempt plan 时即算好并作为身份贯穿执行、留存登记与落盘
           // (不是完成后写回,见 docs/cli.md);裸 run(无 experimentId)不产出。
           locator: run.experimentId

package/src/runner/sandbox-selection.test.ts

@@ -0,0 +1,131 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+
+import { afterEach, describe, expect, it } from "vitest";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { defineEval, e2bSandbox, vercelSandbox } from "../define.ts";
+import type { Agent, DiscoveredEval } from "../types.ts";
+import type { AgentRun } from "./types.ts";
+import { computeFingerprint } from "./fingerprint.ts";
+import {
+  prepareRunSandboxes,
+  resolvedSandboxRecommendedConcurrency,
+  sandboxForEval,
+  sandboxProjection,
+} from "./sandbox-selection.ts";
+
+const roots: string[] = [];
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
+});
+
+function agent(kind: "sandbox" | "remote"): Agent {
+  return { name: `${kind}-agent`, kind } as Agent;
+}
+
+async function evalDef(id: string, environment?: string): Promise<DiscoveredEval> {
+  const root = await mkdtemp(join(tmpdir(), "niceeval-sandbox-selection-"));
+  roots.push(root);
+  const sourcePath = join(root, "case.eval.ts");
+  await writeFile(sourcePath, "export default { test() {} };\n");
+  return {
+    id,
+    environment,
+    baseDir: root,
+    sourcePath,
+    source: { path: "evals/case.eval.ts", content: "export default { test() {} };\n", sha256: "source" },
+    test() {},
+  };
+}
+
+function run(overrides: Partial<AgentRun> = {}): AgentRun {
+  return {
+    agent: agent("sandbox"),
+    flags: {},
+    runs: 1,
+    earlyExit: true,
+    evalFilter: () => true,
+    experimentId: "profiles/run",
+    ...overrides,
+  };
+}
+
+describe("eval-level sandbox selection", () => {
+  it("environments 查表:profile 换预制产物,未声明的 eval 用基础产物且不进 sandboxByEval", async () => {
+    const py39 = await evalDef("astropy/old", "python-3.9-astropy-4.2");
+    const node18 = await evalDef("legacy/node", "node-18-legacy");
+    const plain = await evalDef("weather/basic");
+    const selected = run({
+      sandbox: e2bSandbox({
+        template: "niceeval-agents",
+        environments: {
+          "python-3.9-astropy-4.2": { template: "niceeval-py39-astropy42" },
+          "node-18-legacy": { template: "niceeval-node18" },
+        },
+      }),
+    });
+
+    prepareRunSandboxes([py39, node18, plain], [selected]);
+    expect(sandboxForEval(selected, py39)).toMatchObject({ provider: "e2b", template: "niceeval-py39-astropy42" });
+    expect(sandboxForEval(selected, node18)).toMatchObject({ provider: "e2b", template: "niceeval-node18" });
+    expect(sandboxForEval(selected, plain)).toMatchObject({ provider: "e2b", template: "niceeval-agents" });
+
+    const projection = sandboxProjection(selected);
+    expect(projection.sandbox).toMatchObject({ provider: "e2b", params: { template: "niceeval-agents" } });
+    expect(projection.sandboxByEval).toMatchObject({
+      "astropy/old": { provider: "e2b", params: { template: "niceeval-py39-astropy42" } },
+      "legacy/node": { provider: "e2b", params: { template: "niceeval-node18" } },
+    });
+    expect(projection.sandboxByEval).not.toHaveProperty("weather/basic");
+
+    const [oldFingerprint, nodeFingerprint, plainFingerprint] = await Promise.all([
+      computeFingerprint(py39, selected),
+      computeFingerprint(node18, selected),
+      computeFingerprint(plain, selected),
+    ]);
+    expect(oldFingerprint).not.toBe(nodeFingerprint);
+    expect(oldFingerprint).not.toBe(plainFingerprint);
+  });
+
+  it("选中 eval 的 profile 缺表项在创建 sandbox 前穷举报错;defineEval 拒绝空 profile", async () => {
+    expect(() => defineEval({ environment: "  ", test() {} })).toThrow(/environment.*non-empty profile id/);
+
+    const missingA = await evalDef("astropy/old", "python-3.9-astropy-4.2");
+    const missingB = await evalDef("legacy/node", "node-18-legacy");
+    const bare = run({ sandbox: e2bSandbox({ template: "niceeval-agents" }) });
+    let thrown: Error | undefined;
+    try {
+      prepareRunSandboxes([missingA, missingB], [bare]);
+    } catch (error) {
+      thrown = error as Error;
+    }
+    expect(thrown?.message).toMatch(/profiles\/run/);
+    expect(thrown?.message).toMatch(/astropy\/old → "python-3\.9-astropy-4\.2"/);
+    expect(thrown?.message).toMatch(/legacy\/node → "node-18-legacy"/);
+    expect(thrown?.message).toMatch(/environments/);
+  });
+
+  it("provider 推荐并发取所有解析结果的最小值;remote agent 零查表", async () => {
+    const item = await evalDef("astropy/old", "python-3.9-astropy-4.2");
+    const plain = await evalDef("weather/basic");
+    const e2bRun = run({
+      sandbox: e2bSandbox({
+        template: "niceeval-agents",
+        environments: { "python-3.9-astropy-4.2": { template: "niceeval-py39-astropy42" } },
+      }),
+    });
+    const vercelRun = run({ experimentId: "profiles/vercel", sandbox: vercelSandbox({ snapshotId: "snap_base" }) });
+    expect(resolvedSandboxRecommendedConcurrency([item, plain], [e2bRun])).toBe(20);
+    expect(resolvedSandboxRecommendedConcurrency([plain], [e2bRun, vercelRun])).toBe(1);
+
+    const remote = run({
+      agent: agent("remote"),
+      sandbox: e2bSandbox({ template: "niceeval-agents" }),
+    });
+    expect(() => prepareRunSandboxes([item], [remote])).not.toThrow();
+    expect(resolvedSandboxRecommendedConcurrency([item], [remote])).toBe(10);
+    expect(sandboxProjection(remote)).toEqual({});
+    expect(remote.resolvedSandboxes).toBeUndefined();
+  });
+});

package/src/runner/sandbox-selection.ts

@@ -0,0 +1,110 @@
+// ExperimentDef.sandbox 的规划期解析:spec 携带 environments 表时,按每条选中 eval 的
+// `environment` profile 查表派生该 eval 的具体 spec;缺表项在创建任何沙箱、计算 carry 或
+// 选择全局并发之前一次性穷举报错。指纹、并发预算、attempt 创建与结果审计全部消费这同一份
+// 解析结果(见 docs/feature/experiments/library.md「不同 eval 起自不同预制环境」)。
+
+import { sandboxRecommendedConcurrency, sandboxRunInfo } from "../sandbox/resolve.ts";
+import type { DiscoveredEval, SandboxOption, SandboxRunInfo } from "../types.ts";
+import type { AgentRun } from "./types.ts";
+
+/** environments 表是内置 provider spec 的数据字段;这里只做查表,不认 provider 名。 */
+function specEnvironments(spec: SandboxOption): Readonly<Record<string, Record<string, unknown>>> | undefined {
+  const environments = (spec as { environments?: unknown }).environments;
+  if (typeof environments !== "object" || environments === null) return undefined;
+  return environments as Readonly<Record<string, Record<string, unknown>>>;
+}
+
+/** 按 profile 派生该 eval 的具体 spec(浅覆盖预制产物槽位,hooks 与其余参数共享);缺表项返回 undefined。 */
+function deriveSpec(spec: SandboxOption, profile: string): SandboxOption | undefined {
+  const override = specEnvironments(spec)?.[profile];
+  if (override === undefined) return undefined;
+  return { ...spec, ...override } as SandboxOption;
+}
+
+function missingEnvironmentsError(run: AgentRun, missing: ReadonlyArray<readonly [string, string]>): Error {
+  const entries = missing.map(([id, profile]) => `  ${id} → ${JSON.stringify(profile)}`).join("\n");
+  return new Error(
+    `sandbox spec for experiment ${JSON.stringify(run.experimentId ?? run.agent.name)} has no environments entry for:\n${entries}\n` +
+      `add the missing profile(s) to the spec's environments table — dockerSandbox({ environments: { "<profile>": { image } } }), ` +
+      `e2bSandbox({ environments: { "<profile>": { template } } }), vercelSandbox({ environments: { "<profile>": { snapshotId } } }) — ` +
+      `or fix the eval's environment declaration`,
+  );
+}
+
+/** 该 eval 实际起步的 spec:未声明 environment 用基础 spec;声明了则查表派生并缓存。 */
+export function sandboxForEval(run: AgentRun, evalDef: DiscoveredEval, fallback?: SandboxOption): SandboxOption | undefined {
+  if (run.agent.kind !== "sandbox") return undefined;
+  const spec = run.sandbox ?? fallback;
+  if (spec === undefined || evalDef.environment === undefined) return spec;
+
+  const cached = run.resolvedSandboxes?.get(evalDef.id);
+  if (cached !== undefined) return cached;
+
+  const derived = deriveSpec(spec, evalDef.environment);
+  if (derived === undefined) throw missingEnvironmentsError(run, [[evalDef.id, evalDef.environment]]);
+  const cache = run.resolvedSandboxes ?? new Map<string, SandboxOption>();
+  cache.set(evalDef.id, derived);
+  run.resolvedSandboxes = cache;
+  return derived;
+}
+
+/** 在 dry-run / carry / concurrency / attempt 展开之前一次性查表;全部缺项一次穷举,不等到花费发生后才出现。 */
+export function prepareRunSandboxes(evals: DiscoveredEval[], runs: AgentRun[], fallback?: SandboxOption): void {
+  for (const run of runs) {
+    if (run.agent.kind !== "sandbox") continue;
+    const spec = run.sandbox ?? fallback;
+    if (spec === undefined) continue; // 缺 spec 的错误由既有 resolveSandbox 路径按原文案报
+    const missing: Array<readonly [string, string]> = [];
+    for (const evalDef of evals) {
+      if (!run.evalFilter(evalDef.id) || evalDef.environment === undefined) continue;
+      if (run.resolvedSandboxes?.has(evalDef.id)) continue;
+      const derived = deriveSpec(spec, evalDef.environment);
+      if (derived === undefined) {
+        missing.push([evalDef.id, evalDef.environment]);
+        continue;
+      }
+      const cache = run.resolvedSandboxes ?? new Map<string, SandboxOption>();
+      cache.set(evalDef.id, derived);
+      run.resolvedSandboxes = cache;
+    }
+    if (missing.length > 0) throw missingEnvironmentsError(run, missing);
+  }
+}
+
+/** ExperimentRunInfo 的 sandbox 投影:顶层恒为基础 spec;sandboxByEval 只含声明了 environment 的选中 eval。 */
+export function sandboxProjection(run: AgentRun, fallback?: SandboxOption): {
+  sandbox?: SandboxRunInfo;
+  sandboxByEval?: Record<string, SandboxRunInfo>;
+} {
+  if (run.agent.kind !== "sandbox") return {};
+  const sandbox = sandboxRunInfo(run.sandbox ?? fallback);
+  const entries = [...(run.resolvedSandboxes ?? new Map<string, SandboxOption>()).entries()].sort(([a], [b]) =>
+    a.localeCompare(b),
+  );
+  const sandboxByEval: Record<string, SandboxRunInfo> = {};
+  for (const [evalId, derived] of entries) {
+    const info = sandboxRunInfo(derived);
+    if (info !== undefined) sandboxByEval[evalId] = info;
+  }
+  return {
+    ...(sandbox !== undefined ? { sandbox } : {}),
+    ...(entries.length > 0 ? { sandboxByEval } : {}),
+  };
+}
+
+export function resolvedSandboxRecommendedConcurrency(
+  evals: DiscoveredEval[],
+  runs: AgentRun[],
+  fallback?: SandboxOption,
+): number {
+  prepareRunSandboxes(evals, runs, fallback);
+  const recommendations: number[] = [];
+  for (const run of runs) {
+    if (run.agent.kind !== "sandbox") continue;
+    for (const evalDef of evals) {
+      if (!run.evalFilter(evalDef.id)) continue;
+      recommendations.push(sandboxRecommendedConcurrency(sandboxForEval(run, evalDef, fallback)));
+    }
+  }
+  return recommendations.length > 0 ? Math.min(...recommendations) : 10;
+}

package/src/runner/types.ts

@@ -31,7 +31,15 @@ export interface ExperimentRunInfo {
   /** evals 过滤器的指纹(数组内容 / 函数体哈希),供「配置没变」判断;与 selectedEvalIds 一起取代原过滤器。 */
   evalFilterFingerprint?: string;
   /** provider 名、provider 的公开参数投影与配置 fingerprint;参数只经投影落盘,token/凭据永不进来。 */
-  sandbox?: { provider: string; params?: Record<string, JsonValue>; fingerprint?: string };
+  sandbox?: SandboxRunInfo;
+  /** spec 携带 environments 表时:声明了 environment 的选中 eval 各自解析到的产物投影,按 eval id 留审计映射;其余 eval 以 `sandbox` 为准。 */
+  sandboxByEval?: Record<string, SandboxRunInfo>;
+}
+
+export interface SandboxRunInfo {
+  provider: string;
+  params?: Record<string, JsonValue>;
+  fingerprint?: string;
 }
 
 /**
@@ -312,6 +320,8 @@ export interface EvalDef {
   description?: string;
   /** 标签,供 CLI `--tag` 过滤和 view 分类;与 id 前缀过滤是两套独立的筛选维度。 */
   tags?: string[];
+  /** 这条 eval 需要的环境 profile id(provider-neutral,如 `"python-3.9-astropy-4.2"`);由 sandbox spec 的 `environments` 表翻译成该 provider 的预制产物。 */
+  environment?: string;
   /** 覆盖项目级 Config.judge,只对这一个 eval 生效(如换个更贵的评审模型)。 */
   judge?: JudgeConfig;
   /** 覆盖 / 追加项目级 Config.reporters,只对这一个 eval 生效。 */
@@ -379,7 +389,10 @@ export interface ExperimentDef {
   evals?: "*" | string[] | ((id: string) => boolean);
   /** 覆盖项目级 / CLI 的单次 attempt 超时(毫秒),只对这个实验生效。 */
   timeoutMs?: number;
-  /** 覆盖项目级 Config.sandbox,只对这个实验生效。 */
+  /**
+   * 覆盖项目级 Config.sandbox,只对这个实验生效。固定 SandboxSpec 对全部选中 eval 复用;
+   * spec 可携带 `environments` 表,按 eval 的 `environment` profile 换预制产物。
+   */
   sandbox?: SandboxOption;
   /**
    * 本实验的花费上限(USD)。调度器按「已完成 attempt 的实测花费」累计,到顶后跳过这个实验
@@ -479,6 +492,8 @@ export interface AgentRun {
   runs: number;
   earlyExit: boolean;
   sandbox?: SandboxOption;
+  /** environments 查表的规划期缓存(只含声明了 environment 的 selected eval);每条只派生一次。 */
+  resolvedSandboxes?: Map<string, SandboxOption>;
   timeoutMs?: number;
   budget?: number;
   evalFilter: (id: string) => boolean;
@@ -541,6 +556,8 @@ export interface Attempt {
   /** agent+model+evalId,用于首过即停。 */
   key: string;
   fingerprint: string;
+  /** 规划期按 eval 的 environment 查表派生的具体 spec；attempt 生命周期不再重新查表。 */
+  sandboxSpec?: SandboxOption;
   /**
    * 构造 fresh attempt plan 时即算好的 Attempt 定位符(不是完成后写回):由 invocation 的
    * snapshotStartedAt 与 attempt 身份派生,贯穿执行、留存登记与落盘——登记项、run 收尾反馈与

package/src/sandbox/types.ts

@@ -126,18 +126,24 @@ export interface DockerSandboxSpec extends SandboxHooks<DockerSandboxSpec> {
   readonly provider: "docker";
   /** 覆盖默认镜像;默认按 runtime 选 `node:*-slim`。预制模板:传烘焙好 agent CLI 的镜像名。 */
   readonly image?: string;
+  /** 按 eval 的 `environment` profile 覆盖预制镜像:键为 profile id,值为该 profile 起步的镜像。未声明 environment 的 eval 用 `image`。 */
+  readonly environments?: Readonly<Record<string, { readonly image: string }>>;
   readonly runtime?: SandboxRuntime;
 }
 export interface VercelSandboxSpec extends SandboxHooks<VercelSandboxSpec> {
   readonly provider: "vercel";
   /** 从已有快照起 microVM。预制模板:烘焙好 agent CLI 的 snapshotId。 */
   readonly snapshotId?: string;
+  /** 按 eval 的 `environment` profile 覆盖预制快照:键为 profile id,值为该 profile 起步的 snapshotId。未声明 environment 的 eval 用 `snapshotId`。 */
+  readonly environments?: Readonly<Record<string, { readonly snapshotId: string }>>;
   readonly runtime?: SandboxRuntime;
 }
 export interface E2BSandboxSpec extends SandboxHooks<E2BSandboxSpec> {
   readonly provider: "e2b";
   /** e2b 模板名/ID。预制模板:烘焙好 agent CLI 的模板(如 `"niceeval-agents"`)。省略用 e2b 默认 `"base"`。 */
   readonly template?: string;
+  /** 按 eval 的 `environment` profile 覆盖预制模板:键为 profile id,值为该 profile 起步的模板。未声明 environment 的 eval 用 `template`。 */
+  readonly environments?: Readonly<Record<string, { readonly template: string }>>;
   /** 仅作记录;e2b 的 node 版本由模板决定,不在创建时选。 */
   readonly runtime?: SandboxRuntime;
 }

package/src/shared/aggregate.ts

@@ -47,7 +47,7 @@ export function experimentGroupOf(experimentId: string): string | undefined {
 /**
  * eval id 前缀过滤,同 CLI 位置参数语义(docs/feature/reports/show.md「打开与收窄」):
  * eval 位置参数是收窄过滤,按**裸前缀宽松匹配**——"algebra" 命中 "algebra"、"algebra/..."
- * 也命中 "algebra2",多命中正是它的用途(与 `--experiment` 的按路径段匹配有意不同)。
+ * 也命中 "algebra2",多命中正是它的用途(与 `--exp` 的按路径段匹配有意不同)。
  */
 export function evalPrefixPredicate(evals?: string | string[]): (id: string) => boolean {
   if (evals === undefined) return () => true;

package/src/show/index.ts

@@ -7,9 +7,9 @@
 //   @<locator>        精确 attempt:无证据 flag → 紧凑全景;带 flag → 对应证据切面
 //   --source / --execution / --diff[=路径]   证据切面(宿主本体):出现即走证据室,不渲染报告槽
 //   --history        执行时间轴(逐 experimentId + evalId 分节),与 --report 互斥
-//   --report <文件>  整槽换成用户报告;位置前缀 / --results / --experiment 先收窄 Scope 再注入
+//   --report <文件>  整槽换成用户报告;位置前缀 / --results / --exp 先收窄 Scope 再注入
 //   --page <id>      多页报告选页;未命中列出可用页 id 按用法错误退出
-//   --results <目录>  结果根换成该目录;--experiment 让 Scope 只留该实验
+//   --results <目录>  结果根换成该目录;--exp 让 Scope 只留该实验
 //
 // 数据全部走 niceeval/results 的读取面(openResults + 合成 Scope + loadAttemptEvidence),
 // 不自己爬目录;证据可用性只由 loadAttemptEvidence 在单 Attempt 页面计算。
@@ -38,10 +38,8 @@ import { attemptHistory } from "./compose.ts";
 import {
   HostReportError,
   loadHostReport,
-  localizeText,
   reportMetaFor,
   renderHostPageText,
-  resolveReportTitle,
   type HostCommandContext,
 } from "./report-host.ts";
 import {
@@ -55,7 +53,7 @@ import {
   evalDetailText,
   evalSourceText,
   executionText,
-  pageIndexText,
+  otherPagesText,
   timingText,
   pickDetailAttempt,
   skippedRunsText,
@@ -324,6 +322,9 @@ async function show(
   };
   const sourceLabel = flags.report ?? "the built-in report";
 
+  // 初始页 = --page 指定的页,缺省第一页(docs/feature/reports/show/reports.md Case 2);
+  // 本地宿主只 resolve 被打开的这一页——其余页只留 id / title,不触发取数(见 shell.md
+  // 「行为约束」「本地宿主只 resolve 被打开的页」)。
   let page = report.pages[0];
   if (flags.page !== undefined) {
     const hit = report.pages.find((p) => p.id === flags.page);
@@ -334,12 +335,6 @@ async function show(
       );
     }
     page = hit;
-  } else if (report.pages.length > 1) {
-    // 多页未选页:只输出页索引与可复制的单页命令,不倾倒页内容(与可比组索引同一模式)。
-    // 标题行走标题回退链(终点是内置文案「Eval 运行结果 / Eval Results」,恒有值)。
-    const title = localizeText(resolveReportTitle(report.title, selection.snapshots), locale) ?? "Eval Results";
-    io.out(pageIndexText({ report, title, command: commandContext, locale }) + "\n");
-    return;
   }
 
   // attemptCommand 留给渲染管线的默认值:AttemptLocator 已经是可直接 `niceeval show @<locator>`
@@ -354,5 +349,17 @@ async function show(
       commandContext: { ...commandContext, ...(flags.page !== undefined ? { page: flags.page } : {}) },
     },
   );
-  io.out(text + "\n");
+
+  // 页数大于一时尾部附「其余页」索引(只列未渲染的页,不倾倒内容);单页定义没有这段。
+  const remaining = report.pages.filter((p) => p.id !== page.id);
+  if (remaining.length === 0) {
+    io.out(text + "\n");
+    return;
+  }
+  const tail = otherPagesText({
+    otherPages: remaining.map((p) => ({ id: p.id, title: p.title })),
+    command: commandContext,
+    locale,
+  });
+  io.out(`${text}\n\n${tail}\n`);
 }

package/src/show/render.ts

@@ -113,7 +113,7 @@ export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHa
 /**
  * 详情块 / eval-id 前缀证据切面默认挑最新一次失败的 attempt;没有失败挑最新一次。
  * 精确选某一次 attempt 走 `@<locator>`(`resolveLocator`),不再有数字 `--attempt`——
- * --experiment 已在 Selection 合成时收窄。
+ * --exp 已在 Selection 合成时收窄。
  */
 export function pickDetailAttempt(attempts: AttemptHandle[]): AttemptHandle | undefined {
   if (attempts.length === 0) return undefined;
@@ -418,29 +418,29 @@ export function attemptHistoryText(opts: {
   return `${head}\n\n${indentBlock(table, "  ")}`;
 }
 
-// ───────────────────────── --report 页索引 ─────────────────────────
+// ───────────────────────── --report 其余页索引 ─────────────────────────
 
 /**
- * 多页报告的页索引(docs/feature/reports/show/reports.md Case 2):标题行 + 每页一行
- * (id / 本 locale 页名 / 可复制的 `--page` 命令)。索引命令携带完整上下文
- * (--results / --report / 位置参数),复制即可精确复现下一层视图。
+ * 渲染初始页之后追加的「其余页」索引(docs/feature/reports/show/reports.md Case 2):
+ * 只列未渲染的页 —— 每行 id / 本 locale 页名 / 可复制的 `--page` 命令,索引命令携带完整上下文
+ * (--results / --report / 位置参数),复制即可精确复现下一层视图。调用方只在页数大于一时
+ * 拼接这段(单页定义没有「其余页」段);`otherPages` 不含被渲染的那一页。
  */
-export function pageIndexText(opts: {
-  report: HostReport;
-  title: string;
+export function otherPagesText(opts: {
+  otherPages: { id: string; title: HostReport["pages"][number]["title"] }[];
   command: HostCommandContext;
   locale: string;
 }): string {
-  const { report, title, command, locale } = opts;
-  const head = `${title} · ${locale === "zh-CN" ? `${report.pages.length} 页` : `${report.pages.length} pages`}`;
+  const { otherPages, command, locale } = opts;
+  const head = locale === "zh-CN" ? "其余页：" : "Other pages:";
   const table = renderAlignedRows(
-    report.pages.map((page) => [
+    otherPages.map((page) => [
       page.id,
       localizeText(page.title, locale) ?? page.id,
       showCommand({ ...command, page: page.id }),
     ]),
   );
-  return `${head}\n\n${indentBlock(table, "  ")}`;
+  return `${head}\n${indentBlock(table, "  ")}`;
 }
 
 // ───────────────────────── 截断预算(--eval / --execution / 全景共用) ─────────────────────────