niceeval 0.10.3-canary.6 → 0.11.0

package/src/report/components/metric-views/compute.ts

@@ -421,7 +421,7 @@ export async function metricLineData(input: ReportInput, options: MetricLineOpti
       );
     }
     const series = options.series ? seriesKey(options.series, item) : undefined;
-    const bucketKey = `${series ?? ""}${x === null ? "null" : String(x)}`;
+    const bucketKey = `${series ?? ""}\u0000${x === null ? "null" : String(x)}`;
     const bucket = buckets.get(bucketKey);
     if (bucket) bucket.items.push(item);
     else buckets.set(bucketKey, { series, x, items: [item] });

package/src/report/model/types.ts

@@ -766,9 +766,9 @@ export interface AttemptDiagnosticsData {
  *
  * - `turns`/`toolCalls`:events 派生(与 `o11y.json` 行为摘要同源),只在有非空 events 时出现,
  *   哪怕派生值恰好是 0(有 events 但零轮/零工具调用是观测到的事实,不是缺失)。
- * - `usage`:落盘 `Usage` 原样,只在 `result.usage` 存在时出现。
- * - `uncachedInputTokens`:`usage.inputTokens − usage.cacheReadTokens`,只在两个输入都存在时派生;
- *   缺任一个不回退猜 0,整字段省略(text 面回退显示原始 `inputTokens`)。
+ * - `usage`:落盘 `Usage` 原样,只在 `result.usage` 存在时出现。桶恒互斥,`inputTokens` 本身
+ *   就是未缓存输入(契约见 docs/feature/results/architecture.md#usage);"uncached in" 标注只在
+ *   `cacheReadTokens` 在场时由 face 层给出,不派生第二个字段。
  * - `estimatedCostUSD`:能算出成本(`usage.costUSD` 或 `result.estimatedCostUSD`)时才出现。
  *
  * `turns`/`toolCalls`/`usage` 三者全部缺失时(没有 events 也没有落盘 usage)整个 data 为
@@ -783,7 +783,6 @@ export interface UsageTableData {
   turns?: number;
   toolCalls?: number;
   usage?: Usage;
-  uncachedInputTokens?: number;
   estimatedCostUSD?: number;
 }
 

package/src/results/format.ts

@@ -13,10 +13,17 @@ export const RESULT_FILE = "result.json";
 /** 快照元数据文件名。 */
 export const SNAPSHOT_FILE = "snapshot.json";
 
+/** eval 目录(相对快照根):evalId 里的 / 保留作目录层级,其余危险字符替换。 */
+export function evalDirOf(evalId: string): string {
+  return evalId.replace(/[^\w./@-]/g, "_");
+}
+
+/** attempt 目录名前缀:`a<attempt>`,同一 eval 目录下按序号并列。 */
+export const ATTEMPT_DIR_PREFIX = "a";
+
 /** attempt 目录(相对快照根):`<evalId>/a<attempt>`;evalId 里的 / 保留作目录层级,其余危险字符替换。 */
 export function attemptDirOf(r: Pick<EvalResult, "id" | "attempt">): string {
-  const id = r.id.replace(/[^\w./@-]/g, "_");
-  return `${id}/a${r.attempt}`;
+  return `${evalDirOf(r.id)}/${ATTEMPT_DIR_PREFIX}${r.attempt}`;
 }
 
 /** 实验目录名:experimentId 里的 / 与其它非 [\w.@-] 字符替换成 _。 */

package/src/results/index.ts

@@ -9,6 +9,8 @@
 
 export {
   openResults,
+  loadLatestResultsForCase,
+  withArtifactBase,
   experimentOfSnapshot,
   resolveLocator,
   LocatorNotFoundError,

package/src/results/open.ts

@@ -12,8 +12,16 @@ import type { EvalResult } from "../types.ts";
 import type { FailedCommandEvidence, O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
 import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
 import { deriveDiffData } from "../scoring/diff.ts";
-import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, classifySnapshot } from "./format.ts";
-import { isNewerSnapshot, selectCurrentResults, selectLatest } from "./select.ts";
+import {
+  ATTEMPT_DIR_PREFIX,
+  RESULT_FILE,
+  SNAPSHOT_FILE,
+  artifactFileOf,
+  classifySnapshot,
+  evalDirOf,
+  experimentDirOf,
+} from "./format.ts";
+import { isNewerSnapshot, isNewerSnapshotPlacement, selectCurrentResults, selectLatest } from "./select.ts";
 import {
   encodeAttemptLocator,
   resolveAttemptLocator,
@@ -267,9 +275,9 @@ async function handleMetaFile(path: string, state: ScanState): Promise<void> {
 
 // ───────────────────────── 快照读取 ─────────────────────────
 
-/** 快照目录:递归收集全部 result.json,组装成 evals / attempts;单个 result.json 坏 JSON 不拖垮快照。 */
-async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState): Promise<Snapshot> {
-  const snapshot: Snapshot = {
+/** snapshot.json 的元数据 → 空壳 Snapshot(evals / attempts 待填);全量扫描与收窄读共用。 */
+function makeSnapshotShell(dir: string, meta: SnapshotMeta): Snapshot {
+  return {
     experimentId: meta.experimentId,
     startedAt: meta.startedAt,
     ...(meta.completedAt !== undefined ? { completedAt: meta.completedAt } : {}),
@@ -286,6 +294,124 @@ async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState
     dir,
     ...(meta.knownEvalIds?.length ? { knownEvalIds: [...meta.knownEvalIds] } : {}),
   };
+}
+
+/**
+ * 快照级字段拼合:「缺才补」,条目自带的值(携带条目的 startedAt)优先。全量扫描与收窄读
+ * 共用这一份——收窄读若自己拼一遍,携入的条目会缺 experimentId / locator 一类字段。
+ */
+function applySnapshotDefaults(record: EvalResult, meta: SnapshotMeta): void {
+  record.experimentId ??= meta.experimentId;
+  record.agent ??= meta.agent;
+  if (record.model === undefined && meta.model !== undefined) record.model = meta.model;
+  record.startedAt ??= meta.startedAt;
+  if (record.experiment === undefined && meta.experiment !== undefined) record.experiment = meta.experiment;
+  // locator 同理「缺才补」:niceeval 自己的 writer(schemaVersion 5 起)恒会写这个字段,
+  // 携带条目原样携带上一轮的值——只有真缺失(第三方 harness 没实现 locator,或手工构造的
+  // 落盘)才按当前身份兜底算一份;这份兜底不保证跨未来的 --resume 稳定,但至少确定性、
+  // 可解析,不比完全没有 locator 差。
+  record.locator ??= encodeAttemptLocator({
+    experimentId: record.experimentId,
+    snapshotStartedAt: meta.startedAt,
+    evalId: record.id,
+    attempt: record.attempt,
+  });
+}
+
+/**
+ * 携带条目 / 新算的 artifactBase(相对结果根):条目自带值优先(`--resume` 携入的条目指向
+ * 原快照),缺则按所在 attempt 的 ref 现拼。携带条目要能被 view 找回 artifact,少了这一步
+ * 报告里的携带行会静默丢 artifact——所以全量扫描(view/data.ts)与收窄读共用这一份公式。
+ */
+export function withArtifactBase(attempt: AttemptHandle): EvalResult {
+  const r = attempt.result;
+  if (r.artifactBase !== undefined) return r;
+  return { ...r, artifactBase: `${attempt.ref.snapshot}/${attempt.ref.attempt}` };
+}
+
+/**
+ * 收窄读:只取某条 `(experimentId, evalId)` 的「当前可用」attempt,不扫全根。
+ *
+ * 口径与 `loadLatestResultsPerEval` 完全一致——整批取自**含它的最新快照**,不跨快照混装;
+ * 「哪份最新」走 `isNewerSnapshotPlacement`(snapshot.json 的 `startedAt` 优先、同刻按目录名),
+ * 目录名倒序只作为候选遍历顺序,不作判据。**不检查快照有没有 `completedAt`**:被中断或强杀的
+ * 未收尾快照里已落盘的终态照常可读(docs/runner.md「缓存:指纹去重」的「携带来源不要求快照收尾」)。
+ *
+ * 代价与该实验的历史快照数近似无关:没跑过这条 eval 的快照只付一次 readdir,不读任何文件;
+ * 只有命中的候选才读 `snapshot.json`。派发路径上的携带重查用它替代全树扫描
+ * (`openResults` 会读并 parse 全根每一个 `result.json`)。
+ */
+export async function loadLatestResultsForCase(
+  root: string,
+  experimentId: string,
+  evalId: string,
+): Promise<EvalResult[]> {
+  const experimentDir = join(resolve(root), experimentDirOf(experimentId));
+  let snapshotDirNames: string[];
+  try {
+    snapshotDirNames = (await readdir(experimentDir, { withFileTypes: true }))
+      .filter((e) => e.isDirectory())
+      .map((e) => e.name);
+  } catch {
+    return []; // 这个实验还没跑过:不是错误
+  }
+  // 目录名带时间戳前缀,倒序 = 最可能命中的先试;权威判定仍在下面的 isNewerSnapshotPlacement。
+  snapshotDirNames.sort((a, b) => b.localeCompare(a));
+  const evalSegment = evalDirOf(evalId);
+
+  let best: { dir: string; meta: SnapshotMeta; attemptDirNames: string[] } | undefined;
+  for (const name of snapshotDirNames) {
+    const snapshotDir = join(experimentDir, name);
+    let attemptDirNames: string[];
+    try {
+      attemptDirNames = (await readdir(join(snapshotDir, evalSegment), { withFileTypes: true }))
+        .filter((e) => e.isDirectory() && ATTEMPT_DIR_RE.test(e.name))
+        .map((e) => e.name);
+    } catch {
+      continue; // 这份快照没跑过这条 eval:只付一次 readdir
+    }
+    if (attemptDirNames.length === 0) continue;
+    let meta: SnapshotMeta;
+    try {
+      const classified = classifySnapshot(JSON.parse(await readFile(join(snapshotDir, SNAPSHOT_FILE), "utf-8")));
+      if (classified.kind !== "ok") continue; // 坏 / 不兼容版本的快照:跳过,不拖垮这次读
+      meta = classified.meta;
+    } catch {
+      continue;
+    }
+    if (best === undefined || isNewerSnapshotPlacement({ startedAt: meta.startedAt, dir: snapshotDir }, { startedAt: best.meta.startedAt, dir: best.dir })) {
+      best = { dir: snapshotDir, meta, attemptDirNames };
+    }
+  }
+  if (best === undefined) return [];
+
+  const snapshot = makeSnapshotShell(best.dir, best.meta);
+  const out: EvalResult[] = [];
+  for (const attemptDirName of best.attemptDirNames.sort(byAttemptIndex)) {
+    const attemptDir = join(best.dir, evalSegment, attemptDirName);
+    let record: EvalResult;
+    try {
+      record = JSON.parse(await readFile(join(attemptDir, RESULT_FILE), "utf-8")) as EvalResult;
+    } catch {
+      continue; // 缺 result.json / 坏 JSON:如实跳过(与 openResults 的 skipped 同一「不拖垮整次读」精神)
+    }
+    // 目录名是被清洗过的 evalId,两条不同的 id 理论上可以洗成同一段;记录自报的 id 才是权威。
+    if (record.id !== evalId) continue;
+    applySnapshotDefaults(record, best.meta);
+    out.push(withArtifactBase(makeAttempt(snapshot, best.dir, attemptDir, record)));
+  }
+  return out;
+}
+
+const ATTEMPT_DIR_RE = new RegExp(`^${ATTEMPT_DIR_PREFIX}\\d+$`);
+
+function byAttemptIndex(a: string, b: string): number {
+  return Number(a.slice(ATTEMPT_DIR_PREFIX.length)) - Number(b.slice(ATTEMPT_DIR_PREFIX.length));
+}
+
+/** 快照目录:递归收集全部 result.json,组装成 evals / attempts;单个 result.json 坏 JSON 不拖垮快照。 */
+async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState): Promise<Snapshot> {
+  const snapshot = makeSnapshotShell(dir, meta);
 
   const resultPaths = (await findResultFiles(dir)).sort();
   const evalsById = new Map<string, Eval>();
@@ -301,22 +427,7 @@ async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState
       continue;
     }
 
-    // 快照级字段拼合:「缺才补」,条目自带的值(携带条目的 startedAt)优先。
-    record.experimentId ??= meta.experimentId;
-    record.agent ??= meta.agent;
-    if (record.model === undefined && meta.model !== undefined) record.model = meta.model;
-    record.startedAt ??= meta.startedAt;
-    if (record.experiment === undefined && meta.experiment !== undefined) record.experiment = meta.experiment;
-    // locator 同理「缺才补」:niceeval 自己的 writer(schemaVersion 5 起)恒会写这个字段,
-    // 携带条目原样携带上一轮的值——只有真缺失(第三方 harness 没实现 locator,或手工构造的
-    // 落盘)才按当前身份兜底算一份;这份兜底不保证跨未来的 --resume 稳定,但至少确定性、
-    // 可解析,不比完全没有 locator 差。
-    record.locator ??= encodeAttemptLocator({
-      experimentId: record.experimentId,
-      snapshotStartedAt: meta.startedAt,
-      evalId: record.id,
-      attempt: record.attempt,
-    });
+    applySnapshotDefaults(record, meta);
 
     const attempt = makeAttempt(snapshot, dir, attemptDir, record);
     let ev = evalsById.get(record.id);

package/src/results/select.ts

@@ -385,6 +385,18 @@ export function dedupeAttempts(attempts: AttemptHandle[]): { attempts: AttemptHa
 
 /** 快照新旧比较:startedAt 优先,同刻按快照目录名(时间戳 + 随机后缀,字典序即时序)。 */
 export function isNewerSnapshot(a: Snapshot, b: Snapshot): boolean {
+  return isNewerSnapshotPlacement(a, b);
+}
+
+/**
+ * isNewerSnapshot 的原始口径,供还没组装成 Snapshot 的读取面共用同一判定(收窄读
+ * `loadLatestResultsForCase` 只拿到目录名 + snapshot.json 的 startedAt)。同一份磁盘在两条
+ * 读取面上必须给出同一个「哪份最新」的答案,否则携带与报告会分叉。
+ */
+export function isNewerSnapshotPlacement(
+  a: Pick<Snapshot, "startedAt" | "dir">,
+  b: Pick<Snapshot, "startedAt" | "dir">,
+): boolean {
   const byStart = a.startedAt.localeCompare(b.startedAt);
   if (byStart !== 0) return byStart > 0;
   return a.dir.localeCompare(b.dir) > 0;

package/src/runner/attempt.test.ts

@@ -20,6 +20,7 @@
 import { describe, expect, it, vi } from "vitest";
 import { Effect } from "effect";
 import { runAttemptEffect } from "./attempt.ts";
+import { activateFeedbackSink, type DiagnosticInput, type FeedbackSink } from "./feedback/sink.ts";
 import { defineSandboxAgent, defineSandbox } from "../define.ts";
 import { writeAgentSetupManifest, AGENT_SETUP_MANIFEST_PATH } from "../agents/manifest.ts";
 import { equals } from "../expect/index.ts";
@@ -921,3 +922,118 @@ describe("runAttemptEffect · 失败命令证据包装(公开 runCommand/runShel
     expect(result.commands).toBeUndefined();
   });
 });
+
+// cases: docs/engineering/testing/unit/experiments-runner.md「attempt 级诊断的对外词法与阶段标注」
+// bug: memory/diagnostic-key-doubles-as-json-warning-code.md
+describe("runAttemptEffect · attempt 级诊断进反馈流的 code 与 phase", () => {
+  /** 捕获 reportDiagnostic 的入参:其余 sink 方法一律 no-op,这里只关心诊断这一条通路。 */
+  function captureDiagnostics(): { seen: DiagnosticInput[]; deactivate: () => void } {
+    const seen: DiagnosticInput[] = [];
+    const noop = () => {};
+    const sink: FeedbackSink = {
+      activity: noop,
+      diagnostic: (input) => {
+        seen.push(input);
+      },
+      interrupted: noop,
+      reporterError: noop,
+      failure: noop,
+      budgetExhausted: noop,
+      kept: noop,
+      experimentHook: noop,
+      precheck: noop,
+      lockWait: noop,
+      experimentProgress: noop,
+      lifecycle: noop,
+    };
+    return { seen, deactivate: activateFeedbackSink(sink) };
+  }
+
+  it("作者不传 dedupeKey:折叠 key 编进 attempt 身份,code 仍是作者给的干净字面量,phase 是报上来那一刻的阶段", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-diagnostic-code",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const { seen, deactivate } = captureDiagnostics();
+    try {
+      await runOnce(agent, new FakeSandbox(), {
+        experimentId: "compare/codex",
+        evalDefOverrides: {
+          setup: async (_sandbox, ctx) => {
+            ctx.diagnostic({
+              code: "memory-warmup-degraded",
+              level: "warning",
+              message: "Memory warmup failed; continuing with a cold index",
+            });
+          },
+        },
+      });
+    } finally {
+      deactivate();
+    }
+
+    expect(seen).toHaveLength(1);
+    const [forwarded] = seen;
+    expect(forwarded!.code).toBe("memory-warmup-degraded");
+    expect(forwarded!.key).toBe("memory-warmup-degraded:compare/codex|fake/eval|0");
+    expect(forwarded!.data?.phase).toBe("eval.setup");
+    expect(forwarded!.identity).toEqual({ experimentId: "compare/codex", evalId: "fake/eval", attempt: 0 });
+  });
+
+  it("作者传了 dedupeKey:折叠按作者的 key,code 与 phase 不受影响", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-diagnostic-dedupe-key",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const { seen, deactivate } = captureDiagnostics();
+    try {
+      await runOnce(agent, new FakeSandbox(), {
+        experimentId: "compare/codex",
+        evalDefOverrides: {
+          test: async (t) => {
+            t.diagnostic({
+              code: "index-rebuilt",
+              level: "warning",
+              message: "rebuilt the index",
+              dedupeKey: "index-rebuilt:compare/codex",
+            });
+          },
+        },
+      });
+    } finally {
+      deactivate();
+    }
+
+    expect(seen).toHaveLength(1);
+    expect(seen[0]!.key).toBe("index-rebuilt:compare/codex");
+    expect(seen[0]!.code).toBe("index-rebuilt");
+    expect(seen[0]!.data?.phase).toBe("eval.run");
+  });
+
+  it("作者 data 里冒充的 phase 被运行器的实际阶段压过,其余 data 字段原样保留", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-diagnostic-forged-phase",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const { seen, deactivate } = captureDiagnostics();
+    try {
+      await runOnce(agent, new FakeSandbox(), {
+        evalDefOverrides: {
+          setup: async (_sandbox, ctx) => {
+            ctx.diagnostic({
+              code: "warmup-degraded",
+              level: "warning",
+              message: "cold index",
+              data: { phase: "scoring.evaluate", indexAgeDays: 12 },
+            });
+          },
+        },
+      });
+    } finally {
+      deactivate();
+    }
+
+    expect(seen).toHaveLength(1);
+    expect(seen[0]!.data).toEqual({ phase: "eval.setup", indexAgeDays: 12 });
+  });
+});

package/src/runner/attempt.ts

@@ -30,6 +30,11 @@ import { createChangeLedger, type ChangeLedger } from "./ledger.ts";
 import { deriveDiffData, emptyDiffData } from "../scoring/diff.ts";
 import { createRemoteSandbox, withEvalLocalPaths } from "./remote-sandbox.ts";
 import { recordFact, type FactValue } from "../shared/facts.ts";
+import {
+  attemptFailureInfo,
+  resolveAttemptFailureClass,
+  type FailureClass,
+} from "../shared/failure-class.ts";
 import type { CapturedEvalSource } from "./eval-source.ts";
 import type {
   AgentContext,
@@ -66,6 +71,18 @@ import type {
   RunOptions,
 } from "./types.ts";
 
+/**
+ * 一次终局失败的空间轴回执:止损闸的消费点(见 docs/feature/error-classification/architecture.md
+ * 「止损执行体」)。`class` 是分类链决议出的 `FailureClass`,`text` 与报错文案同源(作者的修复
+ * 提示),`phase` 是失败所在的生命周期阶段。回执是旁路:不改 verdict、不改 `AttemptError` 形状,
+ * 也不走错误通道传播(attempt fiber 的 `E` 仍恒为 `never`)。
+ */
+export interface AttemptFailureDeclaration {
+  readonly class: FailureClass;
+  readonly phase: LifecyclePhase;
+  readonly text: string;
+}
+
 export interface RunAttemptEffectOptions {
   /** 父级调度器的中断信号；测试直调时省略。 */
   parentSignal?: AbortSignal;
@@ -79,13 +96,19 @@ export interface RunAttemptEffectOptions {
    * 省略时(如测试直调)退避不释放槽位。
    */
   concurrencySlot?: ConcurrencySlot;
+  /**
+   * 终局失败的空间轴回执(每条终局失败各回调一次,含 per-attempt teardown 的失败——teardown
+   * 声明照常落闸、不改 verdict,见 architecture.md「生命周期边界」)。省略(如测试直调)时
+   * 分类照常决议、只是无人消费。回调必须不抛错:它跑在 attempt 的失败路径上,不得掩盖原始失败。
+   */
+  onFailureClass?: (declaration: AttemptFailureDeclaration) => void;
 }
 
 export function runAttemptEffect(
   a: Attempt,
   opts: RunOptions,
   sandboxSem: Effect.Semaphore,
-  { parentSignal, onPhase, concurrencySlot }: RunAttemptEffectOptions = {},
+  { parentSignal, onPhase, concurrencySlot, onFailureClass }: RunAttemptEffectOptions = {},
 ): Effect.Effect<EvalResult> {
   const config = opts.config;
   const { evalDef, run, attempt } = a;
@@ -155,6 +178,21 @@ export function runAttemptEffect(
     onPhase?.(phase);
     reportAttemptLifecycle({ type: "attempt:phase", at: Date.now(), identity, phase });
   };
+  /**
+   * 终局失败的分类与回执 —— **必须在把错误折成纯数据 `AttemptError` 之前调**:折完之后原错误
+   * 对象(连同它携带的 `_tag`/`class` 与 cause 链)就不再是任何人的引用,空间轴声明再也读不出来。
+   * 走生命周期链的三道决议(抛出点声明 → 实验分类器 → 缺省不可重试,见
+   * src/shared/failure-class.ts 的 `resolveAttemptFailureClass`);turn 失败的时间轴已由 send 重试
+   * 执行体在 context 层消费完,浮出到这里的一定是终局失败,这里只读空间轴。
+   * 缺省档(`scope` 省略或 `"attempt"`)不回执:那是「死因只属于本次执行」,没有闸可落。
+   */
+  const declareFailure = (phase: LifecyclePhase, e: unknown): void => {
+    if (!onFailureClass) return;
+    const info = attemptFailureInfo(phase, e);
+    const cls = resolveAttemptFailureClass(info, run.classifyFailure);
+    if (cls.scope !== "eval" && cls.scope !== "experiment") return;
+    onFailureClass({ class: cls, phase, text: info.text });
+  };
   // 本 attempt 累计的运行事实(与 verdict/diagnostics 独立):sandbox hook / eval.setup·teardown /
   // agent setup·send·teardown 经 ctx.fact() 上报的都落这里(同一 attempt 内后写覆盖先写),
   // 收尾时并入结果的 facts 字段(见 finally 末尾,与 diagnostics 同一种「累加器 + finally 并入」模式)。
@@ -188,12 +226,21 @@ export function runAttemptEffect(
     if (input.dedupeKey !== undefined) dedupeIndex.set(input.dedupeKey, record);
     diagnostics.push(record);
     // 同时进运行级永久事件流(human 撤下 dashboard 后追加、agent/ci 各追加一条,去重按 key)。
+    // `key` 只管折叠到多细(作者没给 dedupeKey 时折到「这一条 attempt 的这种诊断」,身份因此
+    // 编进去);对外稳定词法始终单独给 `code`,不让消费方从 key 反推(见 sink.ts 的
+    // DiagnosticInput.code、docs/feature/experiments/cli.md 的 WarningEvent)。
     reportDiagnostic({
       key: input.dedupeKey ?? `${input.code}:${encodeAttemptKey(identity)}`,
+      code: input.code,
       severity: input.level,
       message: input.message,
       identity,
-      data: input.data,
+      // phase 由运行器给,且**压过**作者 `data` 里的同名字段:`WarningEvent.phase` 是
+      // `LifecyclePhase` 闭集,取值只能由「这条诊断报上来时运行器正处在哪一步」决定。
+      // 作者的 `data` 是开放词表,让它盖住这个字段等于允许从 eval / adapter 代码里冒充一个
+      // 别的(甚至不在闭集里的)阶段,消费方按 phase 分支就此失效——与 ScopedFeedback
+      // 「两个方法都不接受 phase」是同一条纪律(见 src/shared/types.ts 的接口注释)。
+      data: { ...input.data, phase },
     });
   };
   // 作用域反馈:progress 走 attempt:progress(短命状态,归因由 runner 的当前阶段决定),
@@ -355,7 +402,12 @@ export function runAttemptEffect(
           const proto = run.agent.tracing?.protocol;
           log(t("runner.otlpReceiver", { endpoint, proto: proto ? ` (${proto})` : "" }));
         } else {
-          // 远程沙箱(e2b / vercel):在沙箱内起 collector,agent 往 localhost 端口发
+          // 远程沙箱(e2b / vercel):在沙箱内起 collector,agent 往 localhost 端口发。
+          // 这一步是「创建本次 tracing 出口」(telemetry.configure 的定义,见
+          // docs/feature/experiments/cli.md 的阶段表),而且是唯一一条要往沙箱里传脚本、
+          // 起进程、等端口的 tracing 出口——秒级耗时与失败都归它自己,不能顺着上一个还开着的
+          // 阶段记成 sandbox.create(那样 collector 起不来会伪装成「起沙箱失败」)。
+          enterPhase("telemetry.configure");
           receiver = yield* createInSandboxTraceReceiver(sandbox);
           const endpoint = receiver.endpoint("");
           const env = run.agent.tracing?.env?.(endpoint);
@@ -429,6 +481,7 @@ export function runAttemptEffect(
           facts,
           commands,
           concurrencySlot,
+          declareFailure,
           registerEvidence: (getEvents, getUsage) => {
             liveEvents = getEvents;
             liveUsage = getUsage;
@@ -543,11 +596,18 @@ export function runAttemptEffect(
     Effect.catchAllCause((cause) =>
       Cause.isInterrupted(cause)
         ? Effect.failCause(cause)
-        : Effect.succeed({
-            ...base,
-            durationMs: Date.now() - t0,
-            error: errorFromThrown(Cause.squash(cause), sendActive ? "agent.run" : lastPhase),
-            ...(commands.length > 0 ? { commands: [...commands] } : {}),
+        : Effect.suspend(() => {
+            // 资源获取 / Scope 层的意外(起沙箱失败、provisioning 的确定性配置死因)同样是终局
+            // 失败:先读空间轴回执,再折成纯数据 AttemptError(顺序不可换,见 declareFailure)。
+            const raw = Cause.squash(cause);
+            const phase = (sendActive ? "agent.run" : lastPhase) ?? "eval.run";
+            declareFailure(phase, raw);
+            return Effect.succeed({
+              ...base,
+              durationMs: Date.now() - t0,
+              error: errorFromThrown(raw, sendActive ? "agent.run" : lastPhase),
+              ...(commands.length > 0 ? { commands: [...commands] } : {}),
+            });
           }),
     ),
     // 结果封口在 Scope release 完成之后:sandbox.stop 已由 finalizer 写进 recorder,
@@ -624,6 +684,9 @@ interface AttemptResources {
   commands: FailedCommandEvidence[];
   /** turn 级重试退避期间释放/收回的全局并发槽位;透传给 createEvalContext。 */
   concurrencySlot?: ConcurrencySlot;
+  /** 终局失败的空间轴回执(runAttemptEffect 持有的同一个闭包):body 的失败路径与 finally 里的
+   *  per-attempt teardown 失败都经它上报,止损闸据此落闸(见 runAttemptEffect 的 declareFailure)。 */
+  declareFailure: (phase: LifecyclePhase, e: unknown) => void;
   /** SessionManager 一建好就登记事件/用量的读取句柄回外层(超时证据保全用,见
    *  runAttemptEffect 顶部 liveEvents/liveUsage 的注释与 docs/runner.md「超时:双层保护」)。 */
   registerEvidence: (getEvents: () => readonly StreamEvent[], getUsage: () => Usage) => void;
@@ -660,6 +723,7 @@ async function runAttemptBody(
     facts,
     commands,
     concurrencySlot,
+    declareFailure,
     registerEvidence,
     registerLedger,
   } = res;
@@ -826,6 +890,10 @@ async function runAttemptBody(
       feedback,
       fact,
       concurrencySlot,
+      // 实验分类器:turn 链上排在 adapter 的 classifyTurnError 之前(见
+      // docs/feature/error-classification/architecture.md「分类链」)。与本文件 declareFailure
+      // 走的生命周期链是同一个函数,两条链的决议序各自单源在 turn-errors.ts / failure-class.ts。
+      experimentClassifier: run.classifyFailure,
       // 题型:计分制下句柄上的 .gate() 是前置(就地求值 + 挂了中止 test()),见 collector。
       scoring: evalDef.scoring ?? "pass",
       // 前置断言就地求值要看当前已提交窗口的 agent diff(非沙箱型没有分类账,省略)。
@@ -876,10 +944,15 @@ async function runAttemptBody(
         // TurnFailed 是 eval 驱动 agent 时的可读失败(message 首行是一层摘要,后续行是
         // diagnose 的 output tail 等下钻证据,单行面各自取首行);稳定 code
         // `turn-failed`,不带控制流 stack(那指向 control-flow.ts,对定位无益)。
+        // 分类先读:turn 链决议出的 FailureClass 由 expectOk() 铸造 TurnFailed 时挂在错误上,
+        // 下一行折成纯数据后就读不到了(见 declareFailure)。
+        declareFailure(getPhase() ?? "eval.run", e);
         error = { code: "turn-failed", message: e.message, phase: getPhase() ?? "eval.run" };
       } else {
         // eval 脚本(比如引用了已改名/删掉的 API)抛出的 TypeError:message 是一层原因,完整 stack
         // 单独进 `error.stack`,niceeval show 展开时才看得到 eval 文件的 file:line。
+        // 作者从 test(t) 体内抛的 ExperimentFatalError / EvalFatalError 也走这条分支。
+        declareFailure(getPhase() ?? "eval.run", e);
         error = errorFromThrown(e, getPhase());
       }
     }
@@ -1043,6 +1116,9 @@ async function runAttemptBody(
     return value;
   } catch (e) {
     recorder.failCurrent();
+    // sandbox 钩子 / agent.setup / eval.setup / 评分链路抛出的终局失败:同样先读空间轴回执,
+    // 再折成纯数据 AttemptError(顺序不可换,见 declareFailure)。
+    declareFailure(getPhase() ?? "eval.run", e);
     const value: EvalResult = {
       ...base,
       durationMs: Date.now() - t0,
@@ -1069,6 +1145,9 @@ async function runAttemptBody(
             await withCleanupTimeout(() => evalDef.teardown!(withEvalLocalPaths(sandbox, evalDef.baseDir), hookCtx));
           } catch (e) {
             // 收尾失败只是 diagnostic,不改判定 —— 挂到 attempt.diagnostics(见 finally 末尾并入)。
+            // 但空间轴声明照常落闸:知识就是知识,兄弟 attempt 还在派发中(architecture.md
+            // 「生命周期边界」)。落闸不改 verdict,两件事互不影响。
+            declareFailure("eval.teardown", e);
             diagnostics.push(teardownDiagnostic("eval.teardown", e));
             throw e; // 让 measureClosing 把这段标 failed
           }
@@ -1082,6 +1161,7 @@ async function runAttemptBody(
           try {
             await withCleanupTimeout(() => run.agent.teardown!(sandbox, attemptCtx));
           } catch (e) {
+            declareFailure("agent.teardown", e);
             diagnostics.push(teardownDiagnostic("agent.teardown", e));
             throw e;
           }
@@ -1101,6 +1181,7 @@ async function runAttemptBody(
             try {
               await withCleanupTimeout(() => sandboxTeardownHooks[i](sandbox, hookCtx));
             } catch (e) {
+              declareFailure("sandbox.teardown", e);
               diagnostics.push(teardownDiagnostic("sandbox.teardown", e));
             }
           }

package/src/runner/discover.ts

@@ -12,6 +12,24 @@ import type { DiscoveredEval, DiscoveredExperiment, EvalDef, ExperimentDef } fro
 
 const SKIP_DIRS = new Set(["node_modules", ".git", ".niceeval", "dist", ".next"]);
 
+/**
+ * 发现阶段的动态 import 会执行被加载文件的**顶层代码**(配置文件里现拉 registry、读 .env、
+ * 连服务都很常见)。裸抛出去的话用户只看到一个不知从何而来的 `TypeError: fetch failed`——
+ * 发现要遍历整棵 `evals/` / `experiments/` 树,一个文件炸了并不会告诉你是哪一个。
+ * 这里把文件路径钉进 message,原错误挂 `cause`(`formatThrown` 会展开成 `caused by:` 链)。
+ */
+async function importDiscovered<T>(file: string, root: string, kind: "eval" | "experiment"): Promise<T> {
+  try {
+    return (await import(pathToFileURL(file).href)) as T;
+  } catch (e) {
+    throw new Error(
+      `Failed to load ${kind} file ${relative(root, file)}: its top-level code threw while being imported. ` +
+        `Fix the error below, or move the work into the ${kind} body so it only runs when this ${kind} is selected.`,
+      { cause: e },
+    );
+  }
+}
+
 async function walkFiles(dir: string, match: (name: string) => boolean): Promise<string[]> {
   const out: string[] = [];
   async function walk(current: string): Promise<void> {
@@ -40,9 +58,9 @@ export async function discoverEvals(root: string): Promise<DiscoveredEval[]> {
   const files = (await walkFiles(dir, (n) => n.endsWith(".eval.ts") || n.endsWith(".eval.tsx"))).sort();
   const out: DiscoveredEval[] = [];
   for (const file of files) {
-    const mod = (await import(pathToFileURL(file).href)) as {
+    const mod = await importDiscovered<{
       default?: EvalDef | EvalDef[] | Record<string, EvalDef>;
-    };
+    }>(file, root, "eval");
     const def = mod.default;
     if (!def) continue;
     const baseId = relative(dir, file).replace(/\.eval\.tsx?$/, "").split(sep).join("/");
@@ -107,7 +125,7 @@ export async function discoverExperiments(root: string): Promise<DiscoveredExper
   const files = (await walkFiles(dir, (n) => n.endsWith(".ts") && !n.endsWith(".d.ts"))).sort();
   const out: DiscoveredExperiment[] = [];
   for (const file of files) {
-    const mod = (await import(pathToFileURL(file).href)) as { default?: ExperimentDef };
+    const mod = await importDiscovered<{ default?: ExperimentDef }>(file, root, "experiment");
     const def = mod.default;
     if (!def || !def.agent) continue;
     const id = relative(dir, file)