niceeval 0.9.0 → 0.10.0

package/src/runner/run.ts

@@ -8,13 +8,15 @@ import { probeJudge } from "../scoring/judge.ts";
 import { t } from "../i18n/index.ts";
 import { cacheKey, planCarry } from "./fingerprint.ts";
 import { OtelReceiverPool } from "../o11y/otlp/turn-otel.ts";
-import { runAttemptEffect } from "./attempt.ts";
+import { errorFromThrown, experimentRunInfo, runAttemptEffect } from "./attempt.ts";
 import { runReporter, emitReporterEvent, scopeReporter, summarize } from "./report.ts";
 import {
   reportActivity,
   reportAttemptLifecycle,
   reportBudgetExhausted,
   reportDiagnostic,
+  reportExperimentHook,
+  reportExperimentProgress,
   reportFailure,
   reportInterrupted,
 } from "./feedback/sink.ts";
@@ -22,8 +24,11 @@ import { failureDetailFromResult } from "./feedback/failure.ts";
 import { encodeAttemptLocator, type AttemptLocator } from "../results/locator.ts";
 import { runWho } from "./types.ts";
 import { prepareRunSandboxes, sandboxForEval } from "./sandbox-selection.ts";
+import { selectedEvalsForRun } from "./eval-selection.ts";
+import { registerExperimentTeardown, unregisterExperimentTeardown } from "./experiment-cleanup-registry.ts";
+import { withCleanupTimeout } from "./cleanup-timeout.ts";
 import type { Agent, EvalResult, JudgeConfig, Reporter, ReporterRegistration, RunShape, RunSummary } from "../types.ts";
-import type { AgentRun, Attempt, LifecyclePhase, AttemptRef, RunOptions } from "./types.ts";
+import type { AgentRun, Attempt, ExperimentHookContext, LifecyclePhase, AttemptRef, RunOptions } from "./types.ts";
 
 /** 反馈层的 attempt 身份 + 展示 label,两个 sink.ts lifecycle 调用点共用,避免各自手写
  *  同一组字段(见 memory 的 live-who-key-mismatch-freezes-rows —— 手写副本漏改是真实事故源)。 */
@@ -113,10 +118,10 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   // 首个通过会 abort 同 key 其余 attempt,且它们的结果被下面的去重检查丢弃,不会重复计入。
   const attempts: Attempt[] = [];
   for (const run of opts.agentRuns) {
-    const evals = opts.evals.filter((e) => run.evalFilter(e.id));
-    // 解析后实际选中的 eval id 全集(evals 过滤器的求值结果)进 ExperimentRunInfo.selectedEvalIds,
-    // 与 evalFilterFingerprint 一起取代过滤器本身落盘(见 docs/feature/results/architecture.md)。
-    run.selectedEvalIds = evals.map((e) => e.id);
+    // selectedEvalIds 已由 CLI 在构造 AgentRun 时对候选 eval 各求值一次算好(见
+    // eval-selection.ts 的 resolveExperimentEvals());这里只按 resolved id 取 eval,
+    // 不重新调用用户谓词(见 docs/feature/results/architecture.md「selectedEvalIds」)。
+    const evals = selectedEvalsForRun(opts.evals, run);
     for (let i = 0; i < run.runs; i++) {
       for (const evalDef of evals) {
         const carryKey = `${run.experimentId ?? ""}|${evalDef.id}`;
@@ -150,6 +155,30 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     }
   }
 
+  // 派发顺序:瓶颈优先(docs/runner.md「派发顺序:瓶颈优先,追求最小总墙钟时间」)。
+  // 单次 attempt 耗时未知且假设同批内大致均匀,轮次数(该 run 的 attempt 数 / 有效并发宽度,
+  // 向上取整)就是耗时的代理指标——把 identical-machine 调度的 LPT 规则推广到 moldable job
+  // 场景:轮次多的 run 是关键路径瓶颈,让它先抢到并发位,总时长才接近瓶颈自身的串行耗时,
+  // 而不是「瓶颈耗时 + 排在它前面的其它 run 先跑完的耗时」。只在建 attempt 列表时算一次,
+  // 不随 earlyExit / fail-fast / budget 实际提前收尾而重算(动态调整不值得为一个尽力而为的
+  // 启发式引入)。只重排派发顺序,不改两级信号量本身;结果仍按发现顺序输出(见下方 sort)。
+  {
+    const byRun = new Map<AgentRun, Attempt[]>();
+    for (const a of attempts) {
+      let group = byRun.get(a.run);
+      if (!group) byRun.set(a.run, (group = []));
+      group.push(a);
+    }
+    const rounds = (run: AgentRun, count: number): number => {
+      const width = Math.min(run.maxConcurrency ?? opts.maxConcurrency, opts.maxConcurrency);
+      return Math.ceil(count / width);
+    };
+    const groups = [...byRun.entries()];
+    groups.sort((a, b) => rounds(b[0], b[1].length) - rounds(a[0], a[1].length));
+    attempts.length = 0;
+    for (const [, group] of groups) attempts.push(...group);
+  }
+
   // 预检 judge:验证 API key + 端点可达,避免跑完 agent 才发现 judge 不通。
   // 放在 attempts 展开之后,fail fast 只对会真正触发 judge 的运行生效
   // (目标收集逻辑见 judgeProbeTargets;全部结果携入、attempts 为空时也自然跳过)。
@@ -169,26 +198,14 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     }
   }
 
-  if (carriedResults.length > 0) {
-    const retryCount = new Set(attempts.map((a) => `${a.run.experimentId ?? ""}|${a.evalDef.id}`)).size;
-    reportActivity(t("runner.resumeCarry", { carried: carriedResults.length, retry: retryCount }).trimEnd());
-    // 按 experiment 分组列出被复用(跳过)的 eval:不列清单的话,用户只看到数量,
-    // 无法核对「跳过的是不是我以为已经过了的那些」。同一 key 多个 run 去重。
-    const carriedByExperiment = new Map<string, Set<string>>();
-    for (const r of carriedResults) {
-      const ids = carriedByExperiment.get(r.experimentId!) ?? new Set<string>();
-      ids.add(r.id);
-      carriedByExperiment.set(r.experimentId!, ids);
-    }
-    for (const [experiment, ids] of [...carriedByExperiment].sort(([a], [b]) => a.localeCompare(b))) {
-      reportActivity(t("runner.resumeCarryDetail", { experiment, evals: [...ids].sort().join(", ") }).trimEnd());
-    }
-  }
+  // 缓存携入只在 plan 的 Reuse 行给数量,不逐条铺 eval id 清单(见 cli.md「人在终端里怎么用」:
+  // 哪些 eval 复用、哪些重跑属于 --dry 与 niceeval view,不占 human 的 scrollback)。
 
   // onRunStart 报「本次实际要跑的 eval」(过滤 + 去重),不是发现到的全部 —— 否则计数误导。
   const runningIds = new Set(attempts.map((a) => a.evalDef.id));
   const runningEvals = [...runningIds].map((id) => ({ id }));
   const firstAgent = opts.agentRuns[0]?.agent;
+  const firstModel = opts.agentRuns[0]?.model;
   const shape: RunShape = {
     evals: runningEvals.length,
     configs: opts.agentRuns.length,
@@ -269,7 +286,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   // budget 护栏:只按「已完成 attempt 的实测花费」判断,不做预测性节流。之前的实现会按
   // 「平均成本 × 在飞数」预扣,快到顶就让还没起飞的 attempt 排队等——这在探测阶段(还没有任何
   // 成本样本时)等价于把同一 budgetKey 的并发摁到一个很小的数,且完全没有文档承诺过这个副作用
-  // (`docs-site/zh/how-to/write-experiment.mdx` 对 `budget` 的描述只有一句「这一格配置的预算
+  // (`docs-site/zh/tutorials/write-experiment.mdx` 对 `budget` 的描述只有一句「这一格配置的预算
   // 上限」)。新语义:已完成 attempt 的花费加总一旦到顶,就不再放新 attempt 起飞(已经在飞的
   // 照常跑完,不会被中途打断);到顶之前不做任何预测性限流,并发完全由 globalSem / runSem 决定。
   // 代价是「已花 + 在飞未结算」的总花费可能短暂超出 budget——这是有意识的取舍:budget 是防止
@@ -320,6 +337,150 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     opts = { ...opts, otelPool: new OtelReceiverPool(opts.config.telemetry?.port) };
   }
 
+  // 实验级生命周期(见 docs/feature/experiments/architecture.md「实验级生命周期」):
+  // setup 整场至多一次——第一个通过派发许可(preflight)的 attempt 触发,后续 attempt 等同一个
+  // memoized promise;等待发生在 gated 里、globalSem 之外,不占全局并发位。teardown 是
+  // ExperimentDef.teardown 字段,按 per-run 剩余 attempt 计数在最后一个 attempt 收尾后执行,
+  // 当且仅当 setup 时点走到过(triggered)——setup 抛错不豁免,未声明 setup 不影响触发;
+  // 计数递减挂在 Effect.ensuring 上,中断路径同样递减,teardown 因此必跑。
+  interface ExperimentLifecycle {
+    /** memoized 的 setup 执行;undefined = 还没有 attempt 触发过。未声明 setup 时为已决议 promise。 */
+    setupPromise?: Promise<void>;
+    /** setup 时点已走到(第一个通过派发许可的 attempt 已触发本实验生命周期;未声明 setup 也置位)。 */
+    triggered: boolean;
+    /** setup 抛过错(用独立布尔标记,不能拿 error 值判断——throw undefined 也是失败)。 */
+    setupFailed: boolean;
+    setupError?: unknown;
+    /** teardown 的 memoized 执行体:谁先到(计数归零 / 扫尾 / 强清 drain)谁启动,后到者等
+     *  同一个 promise——「恰好一次 + 在飞可等待」的全部机制。undefined = 还没人启动。 */
+    teardownPromise?: Promise<void>;
+    /** 本实验还没收尾的 attempt 数;归零即触发 teardown。 */
+    remaining: number;
+    /** 本实验的收尾时点已走到(计数归零 / run 收尾扫尾)。 */
+    tornDown: boolean;
+  }
+  const expLifecycles = new Map<AgentRun, ExperimentLifecycle>();
+  for (const a of attempts) {
+    if (!a.run.setup && !a.run.teardown) continue;
+    let lc = expLifecycles.get(a.run);
+    if (!lc)
+      expLifecycles.set(a.run, (lc = { triggered: false, setupFailed: false, remaining: 0, tornDown: false }));
+    lc.remaining += 1;
+  }
+  const makeExperimentHookContext = (run: AgentRun): ExperimentHookContext => {
+    const experimentId = run.experimentId ?? run.agent.name;
+    return {
+      experimentId: run.experimentId ?? "",
+      selectedEvalIds: run.selectedEvalIds,
+      signal: opts.signal,
+      // progress 是短命状态:只更新本实验运行级行的 detail,不属于任何 attempt 的 active 条目。
+      progress: (u) => {
+        const suffix = u.current !== undefined && u.total !== undefined ? ` (${u.current}/${u.total})` : "";
+        reportExperimentProgress({ experimentId, detail: `${u.message}${suffix}` });
+      },
+      // diagnostic 进运行级永久事件流;实验级钩子不属于任何单个 attempt,不落 result.json。
+      diagnostic: (input) =>
+        reportDiagnostic({
+          key: input.dedupeKey ?? `${input.code}:experiment:${experimentId}`,
+          severity: input.level,
+          message: input.message,
+          data: { experimentId, ...(input.data ?? {}) },
+        }),
+    };
+  };
+  const runExperimentTeardown = (run: AgentRun, lc: ExperimentLifecycle): Promise<void> => {
+    // memoized 一次性执行体(docs/cli.md「中断:三级响应」):正常路径(计数归零 / run 收尾
+    // 扫尾)、强清 drain、崩溃路径谁先到都启动同一个 promise,后到者等到同一个结果——
+    // 不双跑、也不空转;注册表条目在 settle 后注销,drain 因此能等待在飞中的 teardown。
+    lc.teardownPromise ??= (async () => {
+      const experimentId = run.experimentId ?? run.agent.name;
+      try {
+        // 触发规则(docs/feature/experiments/architecture.md):setup 时点没走到(一个 attempt
+        // 都没通过派发许可)则跳过;setup 抛错不豁免——半初始化现场同样要扫尾。
+        if (!lc.triggered || !run.teardown) return;
+        // 与 setup 串行:setup 仍在飞(极端时序:全部 attempt 在 setup 完成前被中断收尾)时等它
+        // settle 再收尾;setupPromise 自带 catch(失败收进 setupFailed),这里不会 reject。
+        await lc.setupPromise;
+        // 起止由 runner 发布,不依赖钩子自己调 progress(见 cli.md「实验级钩子的显示」)。
+        reportExperimentHook({ experimentId, hook: "teardown", status: "started" });
+        const startedAt = Date.now();
+        try {
+          // 有界执行(docs/cli.md 的有界性前提):挂起的 teardown 到点按失败收束,不能无限拖住
+          // 退出;超时后遗留的 promise 悬空,随进程退出消亡。
+          const ctx = makeExperimentHookContext(run);
+          await withCleanupTimeout(() => run.teardown!(ctx));
+          reportExperimentHook({ experimentId, hook: "teardown", status: "done", durationMs: Date.now() - startedAt });
+        } catch (e) {
+          reportExperimentHook({ experimentId, hook: "teardown", status: "failed", durationMs: Date.now() - startedAt });
+          // teardown 失败只作运行级诊断,不改任何已产出的 verdict(与 sandbox.teardown 的
+          // teardown-failed 同一语义);资源可能泄漏,所以要说出来。
+          reportDiagnostic({
+            key: `experiment-teardown-failed:${experimentId}`,
+            severity: "warning",
+            message: t("runner.experimentTeardownFailed", {
+              experimentId,
+              message: e instanceof Error ? e.message : String(e),
+            }).trimEnd(),
+            data: { experimentId },
+          });
+        }
+      } finally {
+        // settle 后才注销:drain 的「启动全部未启动 + 等待全部未 settle」依赖条目在飞期间仍可见。
+        unregisterExperimentTeardown(experimentId);
+      }
+    })();
+    return lc.teardownPromise;
+  };
+  const ensureExperimentSetup = (a: Attempt): Promise<void> => {
+    const lc = expLifecycles.get(a.run)!;
+    if (!lc.setupPromise) {
+      const run = a.run;
+      const experimentId = run.experimentId ?? run.agent.name;
+      lc.triggered = true;
+      // teardown 从触发时点起就静态可达:立即登记进宿主机侧兜底表,setup 挂起 / 抛错都不会
+      // 丢收尾——强清退出(二次中断/看门狗/崩溃路径)时 cli 由此排空未被运行路径消费的
+      // teardown(docs/cli.md「中断:三级响应」)。
+      if (run.teardown) registerExperimentTeardown(experimentId, () => runExperimentTeardown(run, lc));
+      if (!run.setup) {
+        lc.setupPromise = Promise.resolve();
+        return lc.setupPromise;
+      }
+      const ctx = makeExperimentHookContext(run);
+      lc.setupPromise = (async () => {
+        // 起止由 runner 发布(见 cli.md「实验级钩子的显示」):一个什么都不调的 setup 也必须
+        // 可见,不能让「0 running · N queued 长时间不动」看起来像调度卡死。
+        reportExperimentHook({ experimentId, hook: "setup", status: "started" });
+        const startedAt = Date.now();
+        try {
+          const returned = (await run.setup!(ctx)) as unknown;
+          if (typeof returned === "function") {
+            // 迁移护栏:tsx 用户没有类型检查,旧式「setup 返回 cleanup」会被静默忽略而泄漏资源。
+            // best-effort 执行一次返回的函数(把已起的资源收掉),然后按 setup 失败报清晰错误。
+            try {
+              await withCleanupTimeout(returned as () => unknown);
+            } catch {
+              // 迁移护栏里的旧式 cleanup 失败不再叠加报错,主错误(下一行)已指明修法。
+            }
+            throw new Error(
+              t("runner.setupReturnedCleanup", {
+                layer: `ExperimentDef.setup (${experimentId})`,
+                hint: "ExperimentDef.teardown",
+              }).trimEnd(),
+            );
+          }
+        } catch (e) {
+          reportExperimentHook({ experimentId, hook: "setup", status: "failed", durationMs: Date.now() - startedAt });
+          throw e;
+        }
+        reportExperimentHook({ experimentId, hook: "setup", status: "done", durationMs: Date.now() - startedAt });
+      })().catch((e) => {
+        lc.setupFailed = true;
+        lc.setupError = e;
+      });
+    }
+    return lc.setupPromise;
+  };
+
   // earlyExit:为每个 key 各建一个 AbortController。某 attempt 通过或 errored 时 abort 它,
   // 让并发进行中的同 key attempt 通过 signal 尽早退出,而不只是等排队的才能被跳过。
   const evalAbortControllers = new Map<string, AbortController>();
@@ -469,7 +630,29 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
               who: feedbackWho(a),
               phase: initialPhase,
             });
-            const result = yield* runAttemptEffect(a, opts, sandboxSem, attemptSignal);
+            // 实验级 setup 失败:不派发 agent,为这条 attempt 合成结构化 errored 结果
+            // (code experiment-setup-failed、phase experiment.setup),走与真实结果完全相同的
+            // 下游路径(locator / 反馈事件 / reporter / 落盘)——环境起不来是每条 eval 都没跑成
+            // 的事实,要逐条进报告,不是一条一次性日志(见 docs/feature/experiments/
+            // architecture.md「实验级生命周期」)。
+            const expLc = expLifecycles.get(a.run);
+            const result = expLc?.setupFailed
+              ? ({
+                  id: a.evalDef.id,
+                  description: a.evalDef.description,
+                  experimentId: a.run.experimentId,
+                  experiment: experimentRunInfo(a.run, opts.config.sandbox),
+                  agent: a.run.agent.name,
+                  model: a.run.model,
+                  verdict: "errored",
+                  fingerprint: a.fingerprint,
+                  attempt: a.attempt,
+                  startedAt: new Date().toISOString(),
+                  durationMs: 0,
+                  assertions: [],
+                  error: { ...errorFromThrown(expLc.setupError, "experiment.setup"), code: "experiment-setup-failed" },
+                } satisfies EvalResult)
+              : yield* runAttemptEffect(a, opts, sandboxSem, attemptSignal);
             // locator 在这里确定 —— 早于本 attempt 触发的任何 reporter 回调 / 事件
             // (onEvalComplete、eval:complete),所以每一个观察者看到的都已经是最终值,
             // 和落盘 result.json 完全一致(writer.ts 的 entry.locator ?? 兜底分支因此
@@ -530,9 +713,11 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
               // 并发情况:同 key 另一个 attempt 已通过后本 attempt 才完成(被 abort 后产出
               // errored),不计入结果。
               return;
-            } else if (result.verdict === "errored") {
+            } else if (result.verdict === "errored" && !expLc?.setupFailed) {
               // errored 不中止其余样本(基建可能自愈);只有同一错误 code 连续复现才判定为
               // 确定性错误,进 run 级 fail-fast 停止派发(不 abort 已在飞的 attempt)。
+              // 实验级 setup 失败的合成结果不进 fail-fast:它不派发 agent、零成本,
+              // 契约是本实验「所有」attempt 逐条记 errored 进报告,不该被止损机制截短。
               const code = result.error?.code ?? "unexpected-error";
               const prev = lastErrorCode.get(a.key);
               const streak = prev?.code === code ? prev.streak + 1 : 1;
@@ -576,10 +761,29 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
         const gated = Effect.gen(function* () {
           const proceed = yield* preflight;
           if (!proceed) return;
+          // 实验级 setup:第一个走到这里的 attempt 真正执行,其余等同一个 memoized promise
+          // (它从不 reject——失败收进 lc.setupFailed,由 body 合成 errored 结果);等待发生在
+          // globalSem 之外,慢启动的 setup 不占全局并发位。
+          if (a.run.setup || a.run.teardown) yield* Effect.promise(() => ensureExperimentSetup(a));
           yield* globalSem.withPermits(1)(body);
         });
         const runSem = runSems.get(a.run);
-        return runSem ? runSem.withPermits(1)(gated) : gated;
+        const withRunSem = runSem ? runSem.withPermits(1)(gated) : gated;
+        if (!a.run.setup && !a.run.teardown) return withRunSem;
+        // 实验级 teardown 计数:每个 attempt 收尾(含被 preflight 跳过、被中断的)都递减,
+        // 归零触发 ExperimentDef.teardown。ensuring 在中断路径同样执行,teardown 因此必跑。
+        return withRunSem.pipe(
+          Effect.ensuring(
+            Effect.promise(async () => {
+              const lc = expLifecycles.get(a.run)!;
+              lc.remaining -= 1;
+              if (lc.remaining === 0 && !lc.tornDown) {
+                lc.tornDown = true;
+                await runExperimentTeardown(a.run, lc);
+              }
+            }),
+          ),
+        );
       },
       { concurrency: "unbounded", discard: true },
     ).pipe(
@@ -596,15 +800,44 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     { signal: opts.signal },
   );
   await opts.otelPool?.close();
+
+  // 实验级 teardown 兜底扫尾:正常路径由 per-attempt ensuring 的计数归零触发(见上),但一次
+  // 真实批跑观察到过计数路径未触发的间歇现象(根因未定位,排查记录见 memory 的
+  // experiment-teardown-missed-once-in-batch)。走到这里时 forEach 的全部 fiber 连同 finalizer
+  // 都已结算,任何 tornDown 仍为 false 的实验都意味着泄漏;在此强制收尾并报警示诊断——
+  // 扫尾幂等(cleanup 消费一次性),宁可多一道兜底,不把宿主机资源(隧道/容器)留给用户手拆。
+  // 真·缺陷抛出前同样要扫(finalizer 语义,见 docs/feature/experiments/architecture.md
+  // 「实验级生命周期」);cli 的 main().catch() 只兜沙箱,不知道实验级 cleanup 的存在。
+  const sweepExperimentTeardowns = async (): Promise<void> => {
+    for (const [run, lc] of expLifecycles) {
+      if (lc.tornDown) continue;
+      lc.tornDown = true;
+      // 无事可扫:没触发过 / 没声明 teardown——静默跳过。
+      if (!lc.triggered || !run.teardown) continue;
+      // 只有扫尾是启动者时才报「late」诊断;已在飞的(如强清 drain 先到)只等 settle,不算漏。
+      if (!lc.teardownPromise) {
+        const experimentId = run.experimentId ?? run.agent.name;
+        reportDiagnostic({
+          key: `experiment-teardown-late:${experimentId}`,
+          severity: "warning",
+          message: t("runner.experimentTeardownLate", { experimentId }).trimEnd(),
+          data: { experimentId, remaining: lc.remaining },
+        });
+      }
+      await runExperimentTeardown(run, lc);
+    }
+  };
   if (Exit.isFailure(exit)) {
     // signal abort 或 cause 含中断 → 当作用户中断,走部分汇总;否则是真·缺陷,照常抛出。
     if (opts.signal?.aborted || Cause.isInterrupted(exit.cause)) {
       interrupted = true;
     } else {
+      await sweepExperimentTeardowns();
       throw Cause.squash(exit.cause);
     }
   }
   if (interrupted) reportInterrupted();
+  await sweepExperimentTeardowns();
 
   // 稳定排序:按发现顺序 + attempt;携带结果并入后一起排
   const order = new Map(opts.evals.map((e, i) => [e.id, i]));
@@ -616,7 +849,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
       a.attempt - b.attempt,
   );
 
-  const summary = summarize(allResults, firstAgent?.name ?? "", startedAt, Date.now() - t0, opts.config.name);
+  const summary = summarize(allResults, firstAgent?.name ?? "", startedAt, Date.now() - t0, opts.config.name, firstModel);
   await emitReporterEvent(reporters, { type: "run:summary", summary });
   for (const reg of reporters) {
     // required reporter(默认 artifacts、显式 --json/--junit)在这一步失败,不能中断其它

package/src/runner/sandbox-selection.test.ts

@@ -45,7 +45,7 @@ function run(overrides: Partial<AgentRun> = {}): AgentRun {
     flags: {},
     runs: 1,
     earlyExit: true,
-    evalFilter: () => true,
+    selectedEvalIds: [],
     experimentId: "profiles/run",
     ...overrides,
   };
@@ -64,6 +64,7 @@ describe("eval-level sandbox selection", () => {
           "node-18-legacy": { template: "niceeval-node18" },
         },
       }),
+      selectedEvalIds: ["astropy/old", "legacy/node", "weather/basic"],
     });
 
     prepareRunSandboxes([py39, node18, plain], [selected]);
@@ -93,7 +94,10 @@ describe("eval-level sandbox selection", () => {
 
     const missingA = await evalDef("astropy/old", "python-3.9-astropy-4.2");
     const missingB = await evalDef("legacy/node", "node-18-legacy");
-    const bare = run({ sandbox: e2bSandbox({ template: "niceeval-agents" }) });
+    const bare = run({
+      sandbox: e2bSandbox({ template: "niceeval-agents" }),
+      selectedEvalIds: ["astropy/old", "legacy/node"],
+    });
     let thrown: Error | undefined;
     try {
       prepareRunSandboxes([missingA, missingB], [bare]);
@@ -114,14 +118,20 @@ describe("eval-level sandbox selection", () => {
         template: "niceeval-agents",
         environments: { "python-3.9-astropy-4.2": { template: "niceeval-py39-astropy42" } },
       }),
+      selectedEvalIds: ["astropy/old", "weather/basic"],
+    });
+    const vercelRun = run({
+      experimentId: "profiles/vercel",
+      sandbox: vercelSandbox({ snapshotId: "snap_base" }),
+      selectedEvalIds: ["weather/basic"],
     });
-    const vercelRun = run({ experimentId: "profiles/vercel", sandbox: vercelSandbox({ snapshotId: "snap_base" }) });
     expect(resolvedSandboxRecommendedConcurrency([item, plain], [e2bRun])).toBe(20);
     expect(resolvedSandboxRecommendedConcurrency([plain], [e2bRun, vercelRun])).toBe(1);
 
     const remote = run({
       agent: agent("remote"),
       sandbox: e2bSandbox({ template: "niceeval-agents" }),
+      selectedEvalIds: ["astropy/old"],
     });
     expect(() => prepareRunSandboxes([item], [remote])).not.toThrow();
     expect(resolvedSandboxRecommendedConcurrency([item], [remote])).toBe(10);

package/src/runner/sandbox-selection.ts

@@ -54,9 +54,10 @@ export function prepareRunSandboxes(evals: DiscoveredEval[], runs: AgentRun[], f
     if (run.agent.kind !== "sandbox") continue;
     const spec = run.sandbox ?? fallback;
     if (spec === undefined) continue; // 缺 spec 的错误由既有 resolveSandbox 路径按原文案报
+    const selectedIds = new Set(run.selectedEvalIds);
     const missing: Array<readonly [string, string]> = [];
     for (const evalDef of evals) {
-      if (!run.evalFilter(evalDef.id) || evalDef.environment === undefined) continue;
+      if (!selectedIds.has(evalDef.id) || evalDef.environment === undefined) continue;
       if (run.resolvedSandboxes?.has(evalDef.id)) continue;
       const derived = deriveSpec(spec, evalDef.environment);
       if (derived === undefined) {
@@ -101,8 +102,9 @@ export function resolvedSandboxRecommendedConcurrency(
   const recommendations: number[] = [];
   for (const run of runs) {
     if (run.agent.kind !== "sandbox") continue;
+    const selectedIds = new Set(run.selectedEvalIds);
     for (const evalDef of evals) {
-      if (!run.evalFilter(evalDef.id)) continue;
+      if (!selectedIds.has(evalDef.id)) continue;
       recommendations.push(sandboxRecommendedConcurrency(sandboxForEval(run, evalDef, fallback)));
     }
   }