niceeval 0.10.3-canary.7 → 0.11.0

package/src/o11y/parsers/bub.ts

@@ -93,12 +93,15 @@ export function parseBubTranscript(raw: string | undefined): ParsedTranscript {
   // 从 event/run 的 data.usage(或 legacy 顶层 usage)累加用量。
   const addUsage = (usage: unknown): void => {
     if (!usage || typeof usage !== "object") return;
-    const input = num(usage, "input_tokens", "prompt_tokens", "inputTokens");
+    const rawInput = num(usage, "input_tokens", "prompt_tokens", "inputTokens");
     const output = num(usage, "output_tokens", "completion_tokens", "outputTokens");
     let cache = num(usage, "cached_input_tokens", "cache_read_input_tokens", "cacheReadTokens");
     if (cache === 0) cache = num(get(usage, "prompt_tokens_details"), "cached_tokens");
     const cost = num(usage, "cost");
-    if (input === 0 && output === 0 && cache === 0 && cost === 0) return;
+    if (rawInput === 0 && output === 0 && cache === 0 && cost === 0) return;
+    // Chat Completions 口径的 prompt_tokens 含缓存命中,cached_tokens 是其子集;
+    // 落互斥桶前扣掉(docs/feature/adapters/sdk/bub/cost.md)
+    const input = Math.max(0, rawInput - cache);
     inputTokens += input;
     outputTokens += output;
     cacheReadTokens += cache;

package/src/o11y/parsers/codex.test.ts

@@ -0,0 +1,19 @@
+// cases: docs/engineering/testing/unit/results.md
+// 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:codex transcript 的 cached_input_tokens
+// 是 input_tokens 子集,聚合前逐请求扣减。
+// bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
+
+import { describe, expect, it } from "vitest";
+
+import { parseCodexTranscript } from "./codex.ts";
+
+describe("parseCodexTranscript usage 归一(OpenAI 口径)", () => {
+  it("cached_input_tokens 从 input_tokens 里扣出,cache 单独成桶", () => {
+    const raw = [
+      JSON.stringify({ type: "turn.completed", usage: { input_tokens: 1000, cached_input_tokens: 900, output_tokens: 50 } }),
+      JSON.stringify({ type: "turn.completed", usage: { input_tokens: 2000, cached_input_tokens: 1700, output_tokens: 30 } }),
+    ].join("\n");
+    const parsed = parseCodexTranscript(raw);
+    expect(parsed.usage).toMatchObject({ inputTokens: 400, cacheReadTokens: 2600, outputTokens: 80, requests: 2 });
+  });
+});

package/src/o11y/parsers/codex.ts

@@ -80,14 +80,17 @@ function readUsage(u: unknown): { input: number; output: number; cacheRead: numb
     }
     return 0;
   };
-  const input = num("input_tokens", "prompt_tokens", "inputTokens", "promptTokens");
-  const output = num("output_tokens", "completion_tokens", "outputTokens", "completionTokens");
+  const rawInput = num("input_tokens", "prompt_tokens", "inputTokens", "promptTokens");
   const cacheRead = num(
     "cached_input_tokens",
     "cache_read_input_tokens",
     "cache_read_tokens",
     "cacheReadTokens",
   );
+  // codex-rs TokenUsage 的 cached_input_tokens 是 input_tokens 的子集,落互斥桶前扣掉
+  // (docs/feature/adapters/sdk/codex-cli/cost.md)
+  const input = Math.max(0, rawInput - cacheRead);
+  const output = num("output_tokens", "completion_tokens", "outputTokens", "completionTokens");
   // codex-rs TokenUsage 结构体与 Responses API usage.output_tokens_details 都拆出这项:
   // 推理模型(o-series/gpt-5 系列)的思考 token,已计入 output 但值得单列展示。
   const reasoning = num("reasoning_output_tokens", "reasoning_tokens", "reasoningOutputTokens", "reasoningTokens");

package/src/o11y/types.ts

@@ -6,18 +6,21 @@ import type { JsonValue, SourceLoc } from "../shared/types.ts";
 /**
  * 一次运行的 token 用量(沙箱型从 transcript/OTel span 的 `gen_ai.usage.*` 属性抠,remote 由
  * send 的 `Turn.usage` 直接返回)。每个字段只在协议真实提供该值时存在——原始协议没有 usage 时
- * 省略,不编造数值;不存在「默认 0」或「默认 1」的字段(docs/feature/results/architecture.md#usage)。
+ * 省略,不编造数值;不存在「默认 0」或「默认 1」的字段。三个输入侧桶恒互斥:相加才是送进模型的
+ * 完整上下文量;把协议原生口径归一到互斥是 adapter 的落值义务——OpenAI 系报「含缓存的输入总量 +
+ * 缓存命中子集」,落 inputTokens 前先扣掉子集(docs/feature/results/architecture.md#usage,
+ * 各协议明细见 docs/feature/adapters/sdk/<name>/cost.md)。
  */
 export interface Usage {
-  /** 计费口径的输入 token 总量(协议报什么记什么,含 cache read 时如实包含,不换算)。 */
+  /** 未命中缓存、按全价计费的输入 token;与两个 cache 桶互斥。 */
   inputTokens?: number;
   /** 输出(completion)token 数。 */
   outputTokens?: number;
-  /** 从缓存命中的输入部分;与 inputTokens 同一计量口径(省略表示该 agent 不上报此项)。 */
+  /** 从提示缓存命中的输入 token;独立计价桶,不包含在 inputTokens 里(省略表示该 agent 不上报此项)。 */
   cacheReadTokens?: number;
-  /** 写入 prompt 缓存创建的 token 数(省略表示该 agent 不上报此项)。 */
+  /** 写入提示缓存的输入 token;独立计价桶,不包含在 inputTokens 里(省略表示该 agent 不上报此项)。 */
   cacheCreationTokens?: number;
-  /** 推理(thinking)token 数,只在协议真实提供时存在(省略表示该 agent 不上报此项)。 */
+  /** 推理(thinking)token 数,outputTokens 的已含明细,单列展示用;只在协议真实提供时存在。 */
   reasoningTokens?: number;
   /** 真实发生的模型请求数。协议不提供请求计数就省略,绝不写 1 凑数。 */
   requests?: number;

package/src/report/components/attempt-detail/UsageTable.tsx

@@ -12,12 +12,10 @@ export function UsageTable({ data, className }: { data: UsageTableData | null; c
   const rows: [string, string][] = [];
   if (data.turns !== undefined) rows.push(["turns", String(data.turns)]);
   if (data.toolCalls !== undefined) rows.push(["tool calls", String(data.toolCalls)]);
-  // 未缓存输入是消费端派生量,只在两个输入都在场时显示;缺任一个回退显示原始
-  // inputTokens(不猜 0),两种来源用不同行标签区分。
-  if (data.uncachedInputTokens !== undefined) {
-    rows.push(["uncached in", data.uncachedInputTokens.toLocaleString()]);
-  } else if (usage?.inputTokens !== undefined) {
-    rows.push(["in", usage.inputTokens.toLocaleString()]);
+  // 桶恒互斥,inputTokens 就是未缓存输入;"uncached in" 标签只在 cache 拆分在场时用,
+  // cache 桶缺席的数字不贴标注。
+  if (usage?.inputTokens !== undefined) {
+    rows.push([usage.cacheReadTokens !== undefined ? "uncached in" : "in", usage.inputTokens.toLocaleString()]);
   }
   if (usage?.cacheReadTokens !== undefined) rows.push(["cache read", usage.cacheReadTokens.toLocaleString()]);
   if (usage?.outputTokens !== undefined) rows.push(["out", usage.outputTokens.toLocaleString()]);

package/src/report/components/attempt-detail/attempt-components.test.tsx

@@ -422,15 +422,12 @@ describe("Attempt 详情组件族:非空/空证据矩阵", () => {
     expect("turns" in data).toBe(true);
   });
 
-  it("UsageTable:uncachedInputTokens 只在 inputTokens 与 cacheReadTokens 都存在时派生,缺任一不猜 0", () => {
+  it("UsageTable:桶恒互斥,inputTokens 落盘原样透传,不派生第二个输入字段", () => {
+    // inputTokens 本身就是未缓存输入(docs/feature/results/architecture.md#usage),
+    // "uncached in" 只是 face 层在 cacheReadTokens 在场时给的标注,data 不多一个字段。
     const both = usageTableData(evidenceOf({ result: resultOf({ usage: { inputTokens: 100, outputTokens: 1, cacheReadTokens: 40 } }) }))!;
-    expect(both.uncachedInputTokens).toBe(60);
-
-    // 只缺 cacheReadTokens(该 agent 不上报缓存命中):不派生,整字段省略(不回退猜 0)——
-    // text 面此时回退显示原始 inputTokens,见 faces.ts::usageTableText。
-    const noCacheRead = usageTableData(evidenceOf({ result: resultOf({ usage: { inputTokens: 100, outputTokens: 1 } }) }))!;
-    expect(noCacheRead.uncachedInputTokens).toBeUndefined();
-    expect("uncachedInputTokens" in noCacheRead).toBe(false);
+    expect(both.usage?.inputTokens).toBe(100);
+    expect("uncachedInputTokens" in both).toBe(false);
   });
 
   it("UsageTable:requests 缺失时不出现在 usage 对象里(落盘原样透传,不由 usageTableData 凑值)", () => {

package/src/report/components/attempt-detail/compute.ts

@@ -486,8 +486,8 @@ export function attemptDiagnosticsData(evidence: AttemptEvidence): AttemptDiagno
  * identity 字段(locator/experimentId/evalId/attempt/verdict)恒有;turns/toolCalls 是 events
  * 派生(与 o11y.json 行为摘要同源,buildO11ySummary 与 o11y.json 落盘走同一份纯函数),没有
  * events 就整对省略——不因为其中一个恰好是 0 就当作"缺失"处理,0 是观测到的事实。
- * uncachedInputTokens 只在 inputTokens 与 cacheReadTokens 都存在时派生,缺任一个不猜 0
- * (text 面回退显示原始 inputTokens)。turns/toolCalls/usage 三者全部缺失时返回 null——
+ * token 桶恒互斥,inputTokens 本身就是未缓存输入,不派生第二个字段。
+ * turns/toolCalls/usage 三者全部缺失时返回 null——
  * 没有任何用量事实可摆,与其余叶子同一条"没有 usage 时零输出"规则。
  */
 export function usageTableData(evidence: AttemptEvidence): UsageTableData | null {
@@ -498,10 +498,6 @@ export function usageTableData(evidence: AttemptEvidence): UsageTableData | null
   const usage = result.usage;
   if (turns === undefined && toolCalls === undefined && usage === undefined) return null;
 
-  const uncachedInputTokens =
-    usage && typeof usage.inputTokens === "number" && typeof usage.cacheReadTokens === "number"
-      ? usage.inputTokens - usage.cacheReadTokens
-      : undefined;
   const estimatedCostUSD = attemptCostUSD(result);
 
   return {
@@ -513,7 +509,6 @@ export function usageTableData(evidence: AttemptEvidence): UsageTableData | null
     ...(turns !== undefined ? { turns } : {}),
     ...(toolCalls !== undefined ? { toolCalls } : {}),
     ...(usage !== undefined ? { usage } : {}),
-    ...(uncachedInputTokens !== undefined ? { uncachedInputTokens } : {}),
     ...(estimatedCostUSD !== null ? { estimatedCostUSD } : {}),
   };
 }

package/src/report/components/attempt-detail/faces.ts

@@ -314,14 +314,14 @@ export function usageTableText(data: UsageTableData | null, _ctx: TextContext):
   if (data === null) return "";
   const usage = data.usage;
 
-  // 未缓存输入是消费端派生量,只在两个输入都在场时显示;缺任一个回退显示原始 inputTokens
-  // (不猜 0),这里用 label 区分"派生值"与"原始值",不假装两者是同一件事。
+  // 桶恒互斥,inputTokens 就是未缓存输入;"uncached" 标注只在 cache 拆分真实在场时给,
+  // cache 桶缺席的数字不贴标注——不给没有拆分事实的数字暗示拆分。
   const inFragment =
-    data.uncachedInputTokens !== undefined
-      ? `${formatMetricValue(data.uncachedInputTokens)} uncached in`
-      : usage?.inputTokens !== undefined
-        ? `${formatMetricValue(usage.inputTokens)} in`
-        : undefined;
+    usage?.inputTokens !== undefined
+      ? usage.cacheReadTokens !== undefined
+        ? `${formatMetricValue(usage.inputTokens)} uncached in`
+        : `${formatMetricValue(usage.inputTokens)} in`
+      : undefined;
   const cacheFragment = usage?.cacheReadTokens !== undefined ? `${formatMetricValue(usage.cacheReadTokens)} cache read` : undefined;
   const outFragment = usage?.outputTokens !== undefined ? `${formatMetricValue(usage.outputTokens)} out` : undefined;
   const inCache = [inFragment, cacheFragment].filter((s): s is string => s !== undefined).join(" + ");

package/src/report/components/attempt-detail/index.tsx

@@ -594,9 +594,6 @@ export function validateUsageData(data: unknown): string | null {
     const usageProb = usageProblem(data.usage, "usage");
     if (usageProb !== null) return usageProb;
   }
-  if (data.uncachedInputTokens !== undefined && typeof data.uncachedInputTokens !== "number") {
-    return '"uncachedInputTokens" must be a number';
-  }
   if (data.estimatedCostUSD !== undefined && typeof data.estimatedCostUSD !== "number") {
     return '"estimatedCostUSD" must be a number';
   }

package/src/report/components/metric-views/compute.ts

@@ -421,7 +421,7 @@ export async function metricLineData(input: ReportInput, options: MetricLineOpti
       );
     }
     const series = options.series ? seriesKey(options.series, item) : undefined;
-    const bucketKey = `${series ?? ""}${x === null ? "null" : String(x)}`;
+    const bucketKey = `${series ?? ""}\u0000${x === null ? "null" : String(x)}`;
     const bucket = buckets.get(bucketKey);
     if (bucket) bucket.items.push(item);
     else buckets.set(bucketKey, { series, x, items: [item] });

package/src/report/model/types.ts

@@ -766,9 +766,9 @@ export interface AttemptDiagnosticsData {
  *
  * - `turns`/`toolCalls`:events 派生(与 `o11y.json` 行为摘要同源),只在有非空 events 时出现,
  *   哪怕派生值恰好是 0(有 events 但零轮/零工具调用是观测到的事实,不是缺失)。
- * - `usage`:落盘 `Usage` 原样,只在 `result.usage` 存在时出现。
- * - `uncachedInputTokens`:`usage.inputTokens − usage.cacheReadTokens`,只在两个输入都存在时派生;
- *   缺任一个不回退猜 0,整字段省略(text 面回退显示原始 `inputTokens`)。
+ * - `usage`:落盘 `Usage` 原样,只在 `result.usage` 存在时出现。桶恒互斥,`inputTokens` 本身
+ *   就是未缓存输入(契约见 docs/feature/results/architecture.md#usage);"uncached in" 标注只在
+ *   `cacheReadTokens` 在场时由 face 层给出,不派生第二个字段。
  * - `estimatedCostUSD`:能算出成本(`usage.costUSD` 或 `result.estimatedCostUSD`)时才出现。
  *
  * `turns`/`toolCalls`/`usage` 三者全部缺失时(没有 events 也没有落盘 usage)整个 data 为
@@ -783,7 +783,6 @@ export interface UsageTableData {
   turns?: number;
   toolCalls?: number;
   usage?: Usage;
-  uncachedInputTokens?: number;
   estimatedCostUSD?: number;
 }
 

package/src/results/format.ts

@@ -13,10 +13,17 @@ export const RESULT_FILE = "result.json";
 /** 快照元数据文件名。 */
 export const SNAPSHOT_FILE = "snapshot.json";
 
+/** eval 目录(相对快照根):evalId 里的 / 保留作目录层级,其余危险字符替换。 */
+export function evalDirOf(evalId: string): string {
+  return evalId.replace(/[^\w./@-]/g, "_");
+}
+
+/** attempt 目录名前缀:`a<attempt>`,同一 eval 目录下按序号并列。 */
+export const ATTEMPT_DIR_PREFIX = "a";
+
 /** attempt 目录(相对快照根):`<evalId>/a<attempt>`;evalId 里的 / 保留作目录层级,其余危险字符替换。 */
 export function attemptDirOf(r: Pick<EvalResult, "id" | "attempt">): string {
-  const id = r.id.replace(/[^\w./@-]/g, "_");
-  return `${id}/a${r.attempt}`;
+  return `${evalDirOf(r.id)}/${ATTEMPT_DIR_PREFIX}${r.attempt}`;
 }
 
 /** 实验目录名:experimentId 里的 / 与其它非 [\w.@-] 字符替换成 _。 */

package/src/results/index.ts

@@ -9,6 +9,8 @@
 
 export {
   openResults,
+  loadLatestResultsForCase,
+  withArtifactBase,
   experimentOfSnapshot,
   resolveLocator,
   LocatorNotFoundError,

package/src/results/open.ts

@@ -12,8 +12,16 @@ import type { EvalResult } from "../types.ts";
 import type { FailedCommandEvidence, O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
 import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
 import { deriveDiffData } from "../scoring/diff.ts";
-import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, classifySnapshot } from "./format.ts";
-import { isNewerSnapshot, selectCurrentResults, selectLatest } from "./select.ts";
+import {
+  ATTEMPT_DIR_PREFIX,
+  RESULT_FILE,
+  SNAPSHOT_FILE,
+  artifactFileOf,
+  classifySnapshot,
+  evalDirOf,
+  experimentDirOf,
+} from "./format.ts";
+import { isNewerSnapshot, isNewerSnapshotPlacement, selectCurrentResults, selectLatest } from "./select.ts";
 import {
   encodeAttemptLocator,
   resolveAttemptLocator,
@@ -267,9 +275,9 @@ async function handleMetaFile(path: string, state: ScanState): Promise<void> {
 
 // ───────────────────────── 快照读取 ─────────────────────────
 
-/** 快照目录:递归收集全部 result.json,组装成 evals / attempts;单个 result.json 坏 JSON 不拖垮快照。 */
-async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState): Promise<Snapshot> {
-  const snapshot: Snapshot = {
+/** snapshot.json 的元数据 → 空壳 Snapshot(evals / attempts 待填);全量扫描与收窄读共用。 */
+function makeSnapshotShell(dir: string, meta: SnapshotMeta): Snapshot {
+  return {
     experimentId: meta.experimentId,
     startedAt: meta.startedAt,
     ...(meta.completedAt !== undefined ? { completedAt: meta.completedAt } : {}),
@@ -286,6 +294,124 @@ async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState
     dir,
     ...(meta.knownEvalIds?.length ? { knownEvalIds: [...meta.knownEvalIds] } : {}),
   };
+}
+
+/**
+ * 快照级字段拼合:「缺才补」,条目自带的值(携带条目的 startedAt)优先。全量扫描与收窄读
+ * 共用这一份——收窄读若自己拼一遍,携入的条目会缺 experimentId / locator 一类字段。
+ */
+function applySnapshotDefaults(record: EvalResult, meta: SnapshotMeta): void {
+  record.experimentId ??= meta.experimentId;
+  record.agent ??= meta.agent;
+  if (record.model === undefined && meta.model !== undefined) record.model = meta.model;
+  record.startedAt ??= meta.startedAt;
+  if (record.experiment === undefined && meta.experiment !== undefined) record.experiment = meta.experiment;
+  // locator 同理「缺才补」:niceeval 自己的 writer(schemaVersion 5 起)恒会写这个字段,
+  // 携带条目原样携带上一轮的值——只有真缺失(第三方 harness 没实现 locator,或手工构造的
+  // 落盘)才按当前身份兜底算一份;这份兜底不保证跨未来的 --resume 稳定,但至少确定性、
+  // 可解析,不比完全没有 locator 差。
+  record.locator ??= encodeAttemptLocator({
+    experimentId: record.experimentId,
+    snapshotStartedAt: meta.startedAt,
+    evalId: record.id,
+    attempt: record.attempt,
+  });
+}
+
+/**
+ * 携带条目 / 新算的 artifactBase(相对结果根):条目自带值优先(`--resume` 携入的条目指向
+ * 原快照),缺则按所在 attempt 的 ref 现拼。携带条目要能被 view 找回 artifact,少了这一步
+ * 报告里的携带行会静默丢 artifact——所以全量扫描(view/data.ts)与收窄读共用这一份公式。
+ */
+export function withArtifactBase(attempt: AttemptHandle): EvalResult {
+  const r = attempt.result;
+  if (r.artifactBase !== undefined) return r;
+  return { ...r, artifactBase: `${attempt.ref.snapshot}/${attempt.ref.attempt}` };
+}
+
+/**
+ * 收窄读:只取某条 `(experimentId, evalId)` 的「当前可用」attempt,不扫全根。
+ *
+ * 口径与 `loadLatestResultsPerEval` 完全一致——整批取自**含它的最新快照**,不跨快照混装;
+ * 「哪份最新」走 `isNewerSnapshotPlacement`(snapshot.json 的 `startedAt` 优先、同刻按目录名),
+ * 目录名倒序只作为候选遍历顺序,不作判据。**不检查快照有没有 `completedAt`**:被中断或强杀的
+ * 未收尾快照里已落盘的终态照常可读(docs/runner.md「缓存:指纹去重」的「携带来源不要求快照收尾」)。
+ *
+ * 代价与该实验的历史快照数近似无关:没跑过这条 eval 的快照只付一次 readdir,不读任何文件;
+ * 只有命中的候选才读 `snapshot.json`。派发路径上的携带重查用它替代全树扫描
+ * (`openResults` 会读并 parse 全根每一个 `result.json`)。
+ */
+export async function loadLatestResultsForCase(
+  root: string,
+  experimentId: string,
+  evalId: string,
+): Promise<EvalResult[]> {
+  const experimentDir = join(resolve(root), experimentDirOf(experimentId));
+  let snapshotDirNames: string[];
+  try {
+    snapshotDirNames = (await readdir(experimentDir, { withFileTypes: true }))
+      .filter((e) => e.isDirectory())
+      .map((e) => e.name);
+  } catch {
+    return []; // 这个实验还没跑过:不是错误
+  }
+  // 目录名带时间戳前缀,倒序 = 最可能命中的先试;权威判定仍在下面的 isNewerSnapshotPlacement。
+  snapshotDirNames.sort((a, b) => b.localeCompare(a));
+  const evalSegment = evalDirOf(evalId);
+
+  let best: { dir: string; meta: SnapshotMeta; attemptDirNames: string[] } | undefined;
+  for (const name of snapshotDirNames) {
+    const snapshotDir = join(experimentDir, name);
+    let attemptDirNames: string[];
+    try {
+      attemptDirNames = (await readdir(join(snapshotDir, evalSegment), { withFileTypes: true }))
+        .filter((e) => e.isDirectory() && ATTEMPT_DIR_RE.test(e.name))
+        .map((e) => e.name);
+    } catch {
+      continue; // 这份快照没跑过这条 eval:只付一次 readdir
+    }
+    if (attemptDirNames.length === 0) continue;
+    let meta: SnapshotMeta;
+    try {
+      const classified = classifySnapshot(JSON.parse(await readFile(join(snapshotDir, SNAPSHOT_FILE), "utf-8")));
+      if (classified.kind !== "ok") continue; // 坏 / 不兼容版本的快照:跳过,不拖垮这次读
+      meta = classified.meta;
+    } catch {
+      continue;
+    }
+    if (best === undefined || isNewerSnapshotPlacement({ startedAt: meta.startedAt, dir: snapshotDir }, { startedAt: best.meta.startedAt, dir: best.dir })) {
+      best = { dir: snapshotDir, meta, attemptDirNames };
+    }
+  }
+  if (best === undefined) return [];
+
+  const snapshot = makeSnapshotShell(best.dir, best.meta);
+  const out: EvalResult[] = [];
+  for (const attemptDirName of best.attemptDirNames.sort(byAttemptIndex)) {
+    const attemptDir = join(best.dir, evalSegment, attemptDirName);
+    let record: EvalResult;
+    try {
+      record = JSON.parse(await readFile(join(attemptDir, RESULT_FILE), "utf-8")) as EvalResult;
+    } catch {
+      continue; // 缺 result.json / 坏 JSON:如实跳过(与 openResults 的 skipped 同一「不拖垮整次读」精神)
+    }
+    // 目录名是被清洗过的 evalId,两条不同的 id 理论上可以洗成同一段;记录自报的 id 才是权威。
+    if (record.id !== evalId) continue;
+    applySnapshotDefaults(record, best.meta);
+    out.push(withArtifactBase(makeAttempt(snapshot, best.dir, attemptDir, record)));
+  }
+  return out;
+}
+
+const ATTEMPT_DIR_RE = new RegExp(`^${ATTEMPT_DIR_PREFIX}\\d+$`);
+
+function byAttemptIndex(a: string, b: string): number {
+  return Number(a.slice(ATTEMPT_DIR_PREFIX.length)) - Number(b.slice(ATTEMPT_DIR_PREFIX.length));
+}
+
+/** 快照目录:递归收集全部 result.json,组装成 evals / attempts;单个 result.json 坏 JSON 不拖垮快照。 */
+async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState): Promise<Snapshot> {
+  const snapshot = makeSnapshotShell(dir, meta);
 
   const resultPaths = (await findResultFiles(dir)).sort();
   const evalsById = new Map<string, Eval>();
@@ -301,22 +427,7 @@ async function readSnapshotDir(dir: string, meta: SnapshotMeta, state: ScanState
       continue;
     }
 
-    // 快照级字段拼合:「缺才补」,条目自带的值(携带条目的 startedAt)优先。
-    record.experimentId ??= meta.experimentId;
-    record.agent ??= meta.agent;
-    if (record.model === undefined && meta.model !== undefined) record.model = meta.model;
-    record.startedAt ??= meta.startedAt;
-    if (record.experiment === undefined && meta.experiment !== undefined) record.experiment = meta.experiment;
-    // locator 同理「缺才补」:niceeval 自己的 writer(schemaVersion 5 起)恒会写这个字段,
-    // 携带条目原样携带上一轮的值——只有真缺失(第三方 harness 没实现 locator,或手工构造的
-    // 落盘)才按当前身份兜底算一份;这份兜底不保证跨未来的 --resume 稳定,但至少确定性、
-    // 可解析,不比完全没有 locator 差。
-    record.locator ??= encodeAttemptLocator({
-      experimentId: record.experimentId,
-      snapshotStartedAt: meta.startedAt,
-      evalId: record.id,
-      attempt: record.attempt,
-    });
+    applySnapshotDefaults(record, meta);
 
     const attempt = makeAttempt(snapshot, dir, attemptDir, record);
     let ev = evalsById.get(record.id);

package/src/results/select.ts

@@ -385,6 +385,18 @@ export function dedupeAttempts(attempts: AttemptHandle[]): { attempts: AttemptHa
 
 /** 快照新旧比较:startedAt 优先,同刻按快照目录名(时间戳 + 随机后缀,字典序即时序)。 */
 export function isNewerSnapshot(a: Snapshot, b: Snapshot): boolean {
+  return isNewerSnapshotPlacement(a, b);
+}
+
+/**
+ * isNewerSnapshot 的原始口径,供还没组装成 Snapshot 的读取面共用同一判定(收窄读
+ * `loadLatestResultsForCase` 只拿到目录名 + snapshot.json 的 startedAt)。同一份磁盘在两条
+ * 读取面上必须给出同一个「哪份最新」的答案,否则携带与报告会分叉。
+ */
+export function isNewerSnapshotPlacement(
+  a: Pick<Snapshot, "startedAt" | "dir">,
+  b: Pick<Snapshot, "startedAt" | "dir">,
+): boolean {
   const byStart = a.startedAt.localeCompare(b.startedAt);
   if (byStart !== 0) return byStart > 0;
   return a.dir.localeCompare(b.dir) > 0;

package/src/runner/attempt.test.ts

@@ -20,6 +20,7 @@
 import { describe, expect, it, vi } from "vitest";
 import { Effect } from "effect";
 import { runAttemptEffect } from "./attempt.ts";
+import { activateFeedbackSink, type DiagnosticInput, type FeedbackSink } from "./feedback/sink.ts";
 import { defineSandboxAgent, defineSandbox } from "../define.ts";
 import { writeAgentSetupManifest, AGENT_SETUP_MANIFEST_PATH } from "../agents/manifest.ts";
 import { equals } from "../expect/index.ts";
@@ -921,3 +922,118 @@ describe("runAttemptEffect · 失败命令证据包装(公开 runCommand/runShel
     expect(result.commands).toBeUndefined();
   });
 });
+
+// cases: docs/engineering/testing/unit/experiments-runner.md「attempt 级诊断的对外词法与阶段标注」
+// bug: memory/diagnostic-key-doubles-as-json-warning-code.md
+describe("runAttemptEffect · attempt 级诊断进反馈流的 code 与 phase", () => {
+  /** 捕获 reportDiagnostic 的入参:其余 sink 方法一律 no-op,这里只关心诊断这一条通路。 */
+  function captureDiagnostics(): { seen: DiagnosticInput[]; deactivate: () => void } {
+    const seen: DiagnosticInput[] = [];
+    const noop = () => {};
+    const sink: FeedbackSink = {
+      activity: noop,
+      diagnostic: (input) => {
+        seen.push(input);
+      },
+      interrupted: noop,
+      reporterError: noop,
+      failure: noop,
+      budgetExhausted: noop,
+      kept: noop,
+      experimentHook: noop,
+      precheck: noop,
+      lockWait: noop,
+      experimentProgress: noop,
+      lifecycle: noop,
+    };
+    return { seen, deactivate: activateFeedbackSink(sink) };
+  }
+
+  it("作者不传 dedupeKey:折叠 key 编进 attempt 身份,code 仍是作者给的干净字面量,phase 是报上来那一刻的阶段", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-diagnostic-code",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const { seen, deactivate } = captureDiagnostics();
+    try {
+      await runOnce(agent, new FakeSandbox(), {
+        experimentId: "compare/codex",
+        evalDefOverrides: {
+          setup: async (_sandbox, ctx) => {
+            ctx.diagnostic({
+              code: "memory-warmup-degraded",
+              level: "warning",
+              message: "Memory warmup failed; continuing with a cold index",
+            });
+          },
+        },
+      });
+    } finally {
+      deactivate();
+    }
+
+    expect(seen).toHaveLength(1);
+    const [forwarded] = seen;
+    expect(forwarded!.code).toBe("memory-warmup-degraded");
+    expect(forwarded!.key).toBe("memory-warmup-degraded:compare/codex|fake/eval|0");
+    expect(forwarded!.data?.phase).toBe("eval.setup");
+    expect(forwarded!.identity).toEqual({ experimentId: "compare/codex", evalId: "fake/eval", attempt: 0 });
+  });
+
+  it("作者传了 dedupeKey:折叠按作者的 key,code 与 phase 不受影响", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-diagnostic-dedupe-key",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const { seen, deactivate } = captureDiagnostics();
+    try {
+      await runOnce(agent, new FakeSandbox(), {
+        experimentId: "compare/codex",
+        evalDefOverrides: {
+          test: async (t) => {
+            t.diagnostic({
+              code: "index-rebuilt",
+              level: "warning",
+              message: "rebuilt the index",
+              dedupeKey: "index-rebuilt:compare/codex",
+            });
+          },
+        },
+      });
+    } finally {
+      deactivate();
+    }
+
+    expect(seen).toHaveLength(1);
+    expect(seen[0]!.key).toBe("index-rebuilt:compare/codex");
+    expect(seen[0]!.code).toBe("index-rebuilt");
+    expect(seen[0]!.data?.phase).toBe("eval.run");
+  });
+
+  it("作者 data 里冒充的 phase 被运行器的实际阶段压过,其余 data 字段原样保留", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-diagnostic-forged-phase",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const { seen, deactivate } = captureDiagnostics();
+    try {
+      await runOnce(agent, new FakeSandbox(), {
+        evalDefOverrides: {
+          setup: async (_sandbox, ctx) => {
+            ctx.diagnostic({
+              code: "warmup-degraded",
+              level: "warning",
+              message: "cold index",
+              data: { phase: "scoring.evaluate", indexAgeDays: 12 },
+            });
+          },
+        },
+      });
+    } finally {
+      deactivate();
+    }
+
+    expect(seen).toHaveLength(1);
+    expect(seen[0]!.data).toEqual({ phase: "eval.setup", indexAgeDays: 12 });
+  });
+});