niceeval 0.5.3 → 0.6.0

package/src/runner/reporters/quiet.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, it } from "vitest";
+import { quietLine } from "./quiet.ts";
+import type { EvalResult } from "../../types.ts";
+
+function baseResult(overrides: Partial<EvalResult> = {}): EvalResult {
+  return {
+    id: "algebra/quadratic",
+    agent: "codex",
+    verdict: "passed",
+    attempt: 0,
+    durationMs: 42_000,
+    assertions: [],
+    ...overrides,
+  };
+}
+
+describe("quietLine", () => {
+  it("passed / skipped 静默(返回 undefined)", () => {
+    expect(quietLine(baseResult())).toBeUndefined();
+    expect(quietLine(baseResult({ verdict: "skipped", skipReason: "no api key" }))).toBeUndefined();
+  });
+
+  it("errored 带 eval id、[who] 与截断后的 error", () => {
+    const line = quietLine(
+      baseResult({
+        verdict: "errored",
+        model: "gpt-5",
+        error: "sandbox create failed: e2b timeout after 3.9s",
+      }),
+    );
+    expect(line).toContain("algebra/quadratic");
+    expect(line).toContain("[codex/gpt-5]");
+    expect(line).toContain("errored");
+    expect(line).toContain("e2b timeout after 3.9s");
+  });
+
+  it("[who] 与进度行同源:有 experimentId 时用其 basename", () => {
+    const line = quietLine(
+      baseResult({ verdict: "errored", experimentId: "compare/xxx--agents-md", error: "boom" }),
+    );
+    expect(line).toContain("[xxx--agents-md]");
+  });
+
+  it("failed 无 error 时取首个失败断言(severity、阈值、detail)", () => {
+    const line = quietLine(
+      baseResult({
+        verdict: "failed",
+        assertions: [
+          { name: "compiles", severity: "gate", score: 1, passed: true },
+          { name: "closedQA", severity: "gate", score: 0.2, passed: false, threshold: 0.7, detail: "wrong city" },
+        ],
+      }),
+    );
+    expect(line).toContain("failed");
+    expect(line).toContain("closedQA");
+    expect(line).toContain("0.20");
+    expect(line).toContain("wrong city");
+    expect(line).not.toContain("compiles");
+  });
+
+  it("超长 error 截断到 200 字符并加省略号", () => {
+    const line = quietLine(baseResult({ verdict: "errored", error: "x".repeat(500) }))!;
+    expect(line).toContain("…");
+    expect(line.length).toBeLessThan(300);
+  });
+});

package/src/runner/reporters/quiet.ts

@@ -0,0 +1,49 @@
+// Quiet 报告器:--quiet 下的最小结果流。进度流照旧(attempt 无 onProgress 时直写 stderr),
+// 这里只补「坏结果」:verdict 为 errored / failed 的结果各写一行 stderr,passed / skipped
+// 静默 —— 保证 --quiet 下起沙箱失败这类执行错不会全程无声、只能事后读 summary.json。
+// 结果流仍走统一的 Reporter 管线,不在 attempt 里散写。
+
+import type { EvalResult, Reporter } from "../../types.ts";
+import { t } from "../../i18n/index.ts";
+import { runWho } from "../types.ts";
+import { verdictSymbol } from "./shared.ts";
+
+/** error / 断言 detail 的截断上限;比 Console 的 400 更紧,--quiet 只要能定位问题。 */
+const DETAIL_MAX = 200;
+
+/** 纯函数:一条结果 → 该写 stderr 的行;passed / skipped 返回 undefined(静默)。 */
+export function quietLine(result: EvalResult): string | undefined {
+  if (result.verdict !== "errored" && result.verdict !== "failed") return undefined;
+  // [who] 与 attempt 进度行同源(runWho),两条流才能对上同一个运行配置。
+  const who = runWho({ agentName: result.agent, model: result.model, experimentId: result.experimentId });
+  const verdict = result.verdict === "errored" ? t("report.errored") : t("report.failed");
+  const detail =
+    result.error !== undefined
+      ? `${t("report.error")}: ${truncate(result.error, DETAIL_MAX)}`
+      : firstFailedAssertion(result);
+  return `  ${verdictSymbol(result.verdict)} ${result.id} ${verdict}  [${who}]${detail ? `  ${detail}` : ""}\n`;
+}
+
+function firstFailedAssertion(result: EvalResult): string | undefined {
+  const a = result.assertions.find((x) => !x.passed);
+  if (!a) return undefined;
+  const sev = a.severity === "gate" ? t("report.gate") : t("report.soft");
+  const thr =
+    a.threshold !== undefined
+      ? t("report.assertionThreshold", { score: a.score.toFixed(2), threshold: a.threshold })
+      : "";
+  return `${sev}: ${a.name}${thr}${a.detail ? ` — ${truncate(a.detail, DETAIL_MAX)}` : ""}`;
+}
+
+function truncate(s: string, n: number): string {
+  return s.length > n ? s.slice(0, n) + "…" : s;
+}
+
+export function Quiet(): Reporter {
+  return {
+    onEvalComplete(result: EvalResult) {
+      const line = quietLine(result);
+      if (line !== undefined) process.stderr.write(line);
+    },
+  };
+}

package/src/runner/reporters/shared.ts

@@ -1,18 +1,18 @@
 // reporter 共用的展示常量。符号表只此一份 —— console/live/table 各抄一份时,
 // 改一个符号要同步三处且不报错。
 
-import type { ResultOutcome } from "../../types.ts";
+import type { Verdict } from "../../types.ts";
 
-export const OUTCOME_SYM: Record<ResultOutcome, string> = {
+export const VERDICT_SYM: Record<Verdict, string> = {
   passed: "✓",
   failed: "✗",
   errored: "!",
   skipped: "○",
 };
 
-export function outcomeSymbol(outcome: string): string {
-  return OUTCOME_SYM[outcome as ResultOutcome] ?? "?";
+export function verdictSymbol(verdict: string): string {
+  return VERDICT_SYM[verdict as Verdict] ?? "?";
 }
 
-/** live 表格里「还没抢到并发名额」的行:和转圈的 SPINNER、完成后的 OUTCOME_SYM 三态区分开。 */
+/** live 表格里「还没抢到并发名额」的行:和转圈的 SPINNER、完成后的 VERDICT_SYM 三态区分开。 */
 export const WAITING_SYM = "·";

package/src/runner/reporters/table.ts

@@ -1,11 +1,11 @@
 import type { EvalResult, RunSummary, Usage } from "../../types.ts";
 import { t } from "../../i18n/index.ts";
-import { outcomeSymbol } from "./shared.ts";
-import { foldEvalOutcome, evalLevelStats as sharedEvalLevelStats } from "../../shared/outcome.ts";
+import { verdictSymbol } from "./shared.ts";
+import { foldEvalVerdict, evalLevelStats as sharedEvalLevelStats } from "../../shared/verdict.ts";
 // 展示口径(耗时/成本/百分比)与聚合小工具(标签/求和/排序)与 view 共用一份实现;
 // console.ts 经这里 re-export formatDuration。
 import { formatCost, formatDuration, formatPercent } from "../../shared/format.ts";
-import { OUTCOME_ORDER, avg, displayExperimentName, fallbackExperimentLabel, sumMaybe, totalTokens } from "../../shared/aggregate.ts";
+import { VERDICT_ORDER, avg, displayExperimentName, fallbackExperimentLabel, sumMaybe, totalTokens } from "../../shared/aggregate.ts";
 
 export { formatCost, formatDuration } from "../../shared/format.ts";
 
@@ -29,7 +29,7 @@ interface ExperimentRow {
 
 interface EvalRow {
   id: string;
-  outcome: EvalResult["outcome"];
+  verdict: EvalResult["verdict"];
   reason: string;
   durationMs: number;
   tokens: number;
@@ -77,7 +77,7 @@ export function renderRunReport(summary: RunSummary): string {
         t("report.table.runs"),
       ],
       aggregateEvalRows(row.results).map((evalRow) => [
-        formatOutcome(evalRow.outcome),
+        formatVerdict(evalRow.verdict),
         evalRow.id,
         evalRow.reason,
         formatDuration(evalRow.durationMs),
@@ -114,14 +114,14 @@ export function formatTokens(n: number): string {
   return String(n);
 }
 
-export function formatOutcome(outcome: string): string {
-  const sym = outcomeSymbol(outcome);
-  switch (outcome) {
+export function formatVerdict(verdict: string): string {
+  const sym = verdictSymbol(verdict);
+  switch (verdict) {
     case "passed": return `${sym} ${t("report.passed")}`;
     case "failed": return `${sym} ${t("report.failed")}`;
     case "errored": return `${sym} ${t("report.errored")}`;
     case "skipped": return `${sym} ${t("report.skipped")}`;
-    default: return `${sym} ${outcome}`;
+    default: return `${sym} ${verdict}`;
   }
 }
 
@@ -162,26 +162,26 @@ function aggregateEvalRows(results: EvalResult[]): EvalRow[] {
   for (const result of results) byEval.set(result.id, [...(byEval.get(result.id) ?? []), result]);
   return [...byEval.entries()]
     .map(([id, evalResults]) => {
-      const outcome = foldEvalOutcome(evalResults);
+      const verdict = foldEvalVerdict(evalResults);
       const representative =
-        evalResults.find((r) => r.outcome === outcome) ??
-        evalResults.find((r) => r.outcome !== "passed") ??
+        evalResults.find((r) => r.verdict === verdict) ??
+        evalResults.find((r) => r.verdict !== "passed") ??
         evalResults[0]!;
       return {
         id,
-        outcome,
-        reason: outcome === "passed" ? t("report.passed") : reasonFor(representative),
+        verdict,
+        reason: verdict === "passed" ? t("report.passed") : reasonFor(representative),
         durationMs: avg(evalResults.map((r) => r.durationMs)),
         tokens: totalTokens(evalResults.map((r) => r.usage)),
         cost: sumMaybe(evalResults.map((r) => r.estimatedCostUSD)),
-        passedAttempts: evalResults.filter((r) => r.outcome === "passed").length,
+        passedAttempts: evalResults.filter((r) => r.verdict === "passed").length,
         attempts: evalResults.length,
       };
     })
-    .sort((a, b) => OUTCOME_ORDER[a.outcome] - OUTCOME_ORDER[b.outcome] || a.id.localeCompare(b.id));
+    .sort((a, b) => VERDICT_ORDER[a.verdict] - VERDICT_ORDER[b.verdict] || a.id.localeCompare(b.id));
 }
 
-// 折叠 / 计票口径与 view 同一份实现(src/shared/outcome.ts),CLI 表格和网页榜单永不打架。
+// 折叠 / 计票口径与 view 同一份实现(src/shared/verdict.ts),CLI 表格和网页榜单永不打架。
 function evalLevelStats(results: EvalResult[]) {
   return sharedEvalLevelStats(results, (r) => r.id);
 }

package/src/runner/run.ts

@@ -3,13 +3,14 @@
 // reporter 编排 / 汇总在 report.ts,Sandbox 适配器在 remote-sandbox.ts。
 
 import { readFile } from "node:fs/promises";
-import { Effect, Cause, Duration, Exit } from "effect";
+import { Effect, Cause, Exit } from "effect";
 import { probeJudge } from "../scoring/judge.ts";
 import { t } from "../i18n/index.ts";
-import { cacheKey, computeFingerprint } from "./fingerprint.ts";
+import { cacheKey, planCarry } from "./fingerprint.ts";
 import { OtelReceiverPool } from "../o11y/otlp/turn-otel.ts";
 import { runAttemptEffect } from "./attempt.ts";
 import { runReporter, emitReporterEvent, scopeReporter, summarize } from "./report.ts";
+import { writeStderrLine } from "../tty-line.ts";
 import type { Agent, EvalResult, JudgeConfig, Reporter, RunShape, RunSummary } from "../types.ts";
 import type { AgentRun, Attempt, RunOptions } from "./types.ts";
 
@@ -54,42 +55,14 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     return p;
   };
 
-  const plannedFingerprints = new Map<string, string>();
-  {
-    const jobs: Promise<void>[] = [];
-    for (const run of opts.agentRuns) {
-      for (const evalDef of opts.evals.filter((e) => run.evalFilter(e.id))) {
-        jobs.push(
-          computeFingerprint(evalDef, run, sourceCache).then((fp) => {
-            plannedFingerprints.set(cacheKey(run, evalDef.id), fp);
-          }),
-        );
-      }
-    }
-    await Promise.all(jobs);
-  }
-
-  // 跨实验结果复用:只有上次 passed 且 fingerprint 匹配的 (experimentId, evalId) 组合直接携入。
-  // 失败/错误/跳过/fingerprint 不匹配都会重跑。--force 跳过此逻辑。
-  const priorRunKeys = new Set<string>();
-  const carriedResults: EvalResult[] = [];
-  if (opts.priorResults?.length) {
-    for (const r of opts.priorResults) {
-      if (!r.experimentId) continue;
-      const key = `${r.experimentId}|${r.id}`;
-      if (r.outcome === "passed" && r.fingerprint !== undefined && r.fingerprint === plannedFingerprints.get(key)) {
-        priorRunKeys.add(key);
-      }
-    }
-    for (const r of opts.priorResults) {
-      if (!r.experimentId || !priorRunKeys.has(`${r.experimentId}|${r.id}`)) continue;
-      // artifactsDir 是相对"本次 run 自己目录"的路径,新 summary 换了目录就会失效,必须清掉。
-      // artifactBase 不一样:它已经是 loadLatestResultsPerEval(经 withViewRefs)拼好的、
-      // 相对稳定的 .niceeval 根的路径,指向旧 run 的产物目录依然可解析,原样带过来
-      // ——不然 view 就再也找不到这条携带结果的源码/转录/trace 了。
-      carriedResults.push({ ...r, artifactsDir: undefined });
-    }
-  }
+  // 跨实验结果复用:上次 passed 或 failed 且 fingerprint 匹配的 (experimentId, evalId) 组合
+  // 直接携入 —— 两者都是"跑完了、判定确定"的终态,没有理由重花一次 agent/sandbox 成本去
+  // 复现同一个已知结果。errored 是框架/环境层面的不确定失败(超时、沙箱挂了、judge 探测失败
+  // 等),判定本身不可信,必须重跑。跳过/fingerprint 不匹配同样重跑。--force 跳过此逻辑
+  // (cli.ts 在 --force 时不传 priorResults,也不算 carryPlan)。
+  // carryPlan 优先用调用方(cli.ts,为了 live 表格)已经算好的那份,不重算一遍。
+  const { plannedFingerprints, priorRunKeys, carriedResults } =
+    opts.carryPlan ?? (await planCarry(opts.evals, opts.agentRuns, opts.priorResults));
 
   // 展开 attempts
   // 外层按「round」(run index)迭代,内层按 eval 迭代:同一 key 的第 i+1 次 attempt 排在
@@ -207,17 +180,16 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   // 真的没过)才代表 agent 行为的样本,值得跑满 runs 去测通过率。earlyExit 开时两者都提前收尾。
   const erroredKeys = new Set<string>();
 
-  // budget 护栏带「在飞预扣」:实测成本只有 attempt 完成后才知道,若只检查已完成花费,
-  // maxConcurrency 个 attempt 会在任何成本回写前全部起飞,实际花费能冲到 budget 的十几倍。
-  // 口径:还没有任何【带成本】的完成样本时,同一 budgetKey 只放一个 attempt 在飞(探单次成本);
-  // 有样本后,用平均实测成本给每个在飞 attempt 预扣,预计总额到顶就等,已花到顶就停。
-  // costSamples 只数报了成本的 attempt —— 不报成本的完成(agent 无用量、模型不在价格表、
-  // 早期 errored)不能算 0 元样本,否则均值被拉成 0,护栏彻底失效。连续多次完成都拿不到
-  // 成本时,说明这个 agent 的 budget 根本不可执行:警告一次然后放行,而不是永远串行装样子。
+  // budget 护栏:只按「已完成 attempt 的实测花费」判断,不做预测性节流。之前的实现会按
+  // 「平均成本 × 在飞数」预扣,快到顶就让还没起飞的 attempt 排队等——这在探测阶段(还没有任何
+  // 成本样本时)等价于把同一 budgetKey 的并发摁到一个很小的数,且完全没有文档承诺过这个副作用
+  // (`docs-site/zh/guides/write-experiment.mdx` 对 `budget` 的描述只有一句「这一格配置的预算
+  // 上限」)。新语义:已完成 attempt 的花费加总一旦到顶,就不再放新 attempt 起飞(已经在飞的
+  // 照常跑完,不会被中途打断);到顶之前不做任何预测性限流,并发完全由 globalSem / runSem 决定。
+  // 代价是「已花 + 在飞未结算」的总花费可能短暂超出 budget——这是有意识的取舍:budget 是防止
+  // 无限烧钱的安全网,不是精确计费闸,不应该反过来限制吞吐。
   interface BudgetState {
     spent: number;
-    inflight: number;
-    costSamples: number;
     completedNoCost: number;
     unenforceableWarned: boolean;
   }
@@ -225,7 +197,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   const budgetState = (key: string): BudgetState => {
     let s = budgetStates.get(key);
     if (!s) {
-      s = { spent: 0, inflight: 0, costSamples: 0, completedNoCost: 0, unenforceableWarned: false };
+      s = { spent: 0, completedNoCost: 0, unenforceableWarned: false };
       budgetStates.set(key, s);
     }
     return s;
@@ -236,7 +208,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   // 并发 worker 交错写 → 用一个 permit=1 的信号量串起来(替代原先手搓的 reportQueue 链)。
   const reportMutex = Effect.runSync(Effect.makeSemaphore(1));
   // 沙箱启动单独限流:与 agent 并发(maxConcurrency)解耦,防高并发下 daemon/API 过载。
-  // 未显式指定时跟 maxConcurrency 走——各 backend 的推荐值已在 cli 层写进 maxConcurrency 默认值。
+  // 未显式指定时跟 maxConcurrency 走——各 provider 的推荐值已在 cli 层写进 maxConcurrency 默认值。
   const sandboxSem = Effect.runSync(Effect.makeSemaphore(opts.maxConcurrency));
 
   // 两级并发闸:全局(opts.maxConcurrency)+ 实验级(AgentRun.maxConcurrency,可选)。
@@ -286,10 +258,15 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     Effect.forEach(
       attempts,
       (a) => {
-        const body = Effect.gen(function* () {
-            // 早停:同 key 已通过,或已 errored(重跑只会重复同一个框架错误)且开了 earlyExit
-            // → 跳过未启动的 attempt。检查必须在拿到 permit 之后(fiber 是 unbounded 一次性
-            // 全建的,建时结果还没出来);跳过路径短暂占一个 permit,可忽略。
+        const budgetKey = a.run.experimentId ?? a.run.agent.name;
+
+        // preflight:「要不要开始跑」的许可判断(首过即停 + budget 上限检查),不持有
+        // globalSem——两类判断都是即时返回,不该占着全局并发槽位做无谓等待。runSem(实验自己的
+        // maxConcurrency)例外:它是实验私有资源,preflight 占着不影响别的实验,且和 mempal
+        // 那类「必须串行」的语义一致,所以仍然把 preflight 包在 runSem 里面(见下方)。
+        const preflight = Effect.gen(function* () {
+            // 首过即停:同 key 已通过,或已 errored(重跑只会重复同一个框架错误)且开了 earlyExit
+            // → 跳过未启动的 attempt。
             if (a.run.earlyExit && (passedKeys.has(a.key) || erroredKeys.has(a.key))) {
               yield* reportMutex.withPermits(1)(
                 Effect.promise(() =>
@@ -300,49 +277,32 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
                   }),
                 ),
               );
-              return;
+              return false;
             }
 
             const budget = a.run.budget;
-            const budgetKey = a.run.experimentId ?? a.run.agent.name;
             if (budget !== undefined) {
-              // 预扣循环:预计花费(已花 + 在飞×均值)到顶就等在飞的结算,已花到顶就整段停。
-              // 注意等待的 fiber 仍占一个并发位(简单可靠;混跑「带 budget + 不带 budget」
-              // 的多实验时会牺牲些吞吐 —— budget 本就是拿速度换花费上限的模式)。
-              for (;;) {
-                const s = budgetState(budgetKey);
-                if (s.spent >= budget) {
-                  if (!budgetReported.has(budgetKey)) {
-                    budgetReported.add(budgetKey);
-                    yield* reportMutex.withPermits(1)(
-                      Effect.promise(() =>
-                        emitReporterEvent(reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
-                      ),
-                    );
-                  }
-                  return;
-                }
-                if (s.costSamples === 0 && s.completedNoCost >= 3 && !s.unenforceableWarned) {
-                  // 连续几次完成都拿不到成本:budget 对这个 agent 不可执行,说清楚再放行。
-                  s.unenforceableWarned = true;
-                  process.stderr.write(t("runner.budgetUnenforceable", { budgetKey }));
-                }
-                if (s.costSamples === 0 && s.unenforceableWarned) {
-                  s.inflight += 1;
-                  break;
-                }
-                const avg = s.costSamples > 0 ? s.spent / s.costSamples : undefined;
-                const projected =
-                  avg === undefined ? (s.inflight > 0 ? Number.POSITIVE_INFINITY : 0) : s.spent + s.inflight * avg;
-                if (projected < budget) {
-                  s.inflight += 1;
-                  break;
+              // 只看已完成 attempt 的实测花费(见上方 BudgetState 注释),到顶就跳过新 attempt,
+              // 没到顶就立即放行——不等待、不做预测性节流。
+              const s = budgetState(budgetKey);
+              if (s.spent >= budget) {
+                if (!budgetReported.has(budgetKey)) {
+                  budgetReported.add(budgetKey);
+                  yield* reportMutex.withPermits(1)(
+                    Effect.promise(() =>
+                      emitReporterEvent(reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
+                    ),
+                  );
                 }
-                yield* Effect.sleep(Duration.millis(200));
+                return false;
               }
             }
+            return true;
+          });
 
-            // 合并全局信号与本 eval 的早停信号:任一 abort → 本 attempt 的信号 abort。
+        // body:preflight 放行之后才跑,只有这一段真正占用全局并发槽位(globalSem)。
+        const body = Effect.gen(function* () {
+            // 合并全局信号与本 eval 的首过即停信号:任一 abort → 本 attempt 的信号 abort。
             const evalAc = evalAbortControllers.get(a.key);
             const attemptSignal =
               evalAc && opts.signal
@@ -361,34 +321,29 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
                 }),
               ),
             );
-            const result = yield* runAttemptEffect(a, opts, sandboxSem, attemptSignal).pipe(
-              Effect.ensuring(
-                Effect.sync(() => {
-                  if (a.run.budget !== undefined) {
-                    const s = budgetState(budgetKey);
-                    s.inflight = Math.max(0, s.inflight - 1);
-                  }
-                }),
-              ),
-            );
+            const result = yield* runAttemptEffect(a, opts, sandboxSem, attemptSignal);
             if (a.run.budget !== undefined) {
               const s = budgetState(budgetKey);
               if (result.estimatedCostUSD !== undefined) {
                 s.spent += result.estimatedCostUSD;
-                s.costSamples += 1;
               } else {
                 s.completedNoCost += 1;
+                if (s.spent === 0 && s.completedNoCost >= 3 && !s.unenforceableWarned) {
+                  // 连续几次完成都拿不到成本:budget 对这个 agent 不可执行,说清楚一次。
+                  s.unenforceableWarned = true;
+                  writeStderrLine(t("runner.budgetUnenforceable", { budgetKey }));
+                }
               }
             }
 
-            if (result.outcome === "passed") {
+            if (result.verdict === "passed") {
               passedKeys.add(a.key);
               evalAc?.abort(); // 让同 key 并发 attempt 尽早退出
             } else if (a.run.earlyExit && (passedKeys.has(a.key) || erroredKeys.has(a.key))) {
               // 并发情况:同 key 另一个 attempt 已通过/已 errored 后本 attempt 才完成
               // (被 abort 后产出 errored),不计入结果。
               return;
-            } else if (result.outcome === "errored") {
+            } else if (result.verdict === "errored") {
               erroredKeys.add(a.key);
               evalAc?.abort(); // 框架层面的错误会确定性重复,让同 key 剩余 attempt 尽早退出
             }
@@ -413,7 +368,11 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
               Effect.promise(() => emitReporterEvent(reporters, { type: "eval:complete", result })),
             );
           });
-        const gated = globalSem.withPermits(1)(body);
+        const gated = Effect.gen(function* () {
+          const proceed = yield* preflight;
+          if (!proceed) return;
+          yield* globalSem.withPermits(1)(body);
+        });
         const runSem = runSems.get(a.run);
         return runSem ? runSem.withPermits(1)(gated) : gated;
       },