niceeval 0.6.0 → 0.6.2

package/src/runner/report.test.ts

@@ -1,6 +1,8 @@
-import { describe, expect, it } from "vitest";
-import { filterSummary, scopeReporter } from "./report.ts";
-import type { Agent, EvalResult, Reporter, RunShape, RunSummary } from "../types.ts";
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { emitReporterEvent, filterSummary, runReporter, scopeReporter } from "./report.ts";
+import { activateFeedbackSink, activeFeedbackSinkCount } from "./feedback/sink.ts";
+import type { Agent, EvalResult, Reporter, ReporterRegistration, RunShape, RunSummary } from "../types.ts";
 
 function result(id: string, overrides: Partial<EvalResult> = {}): EvalResult {
   return {
@@ -107,3 +109,126 @@ describe("scopeReporter", () => {
     expect(scoped.onEvent).toBeUndefined();
   });
 });
+
+// runReporter()/emitReporterEvent() 是「required/best-effort」判定实际生效的地方(见
+// `ReporterRegistration` 的字段注释):它们只负责把 reg.name/reg.required 原样转发进
+// `reportReporterError()`,不做判定本身——判定(是否让 completion/CI 退出码判红)在下游
+// (coordinator → reducer → cli.ts 的 assembleRunCompletion)。这里用一个假 FeedbackSink
+// 直接断言转发的字段,不需要拉起整个 coordinator。
+describe("runReporter / emitReporterEvent · required/best-effort 原样转发,不吞错也不中断其它 reporter", () => {
+  afterEach(() => {
+    // 每个 activateFeedbackSink() 都要在测试内退出,避免遗留在 sink.ts 的活跃栈里
+    // 污染下一个测试(与 feedback/coordinator.test.ts 同一条兜底校验)。
+    expect(activeFeedbackSinkCount()).toBe(0);
+  });
+
+  function withFakeSink<T>(
+    fn: (calls: { reporter: string; required: boolean; message: string }[]) => Promise<T>,
+  ): Promise<T> {
+    const calls: { reporter: string; required: boolean; message: string }[] = [];
+    const deactivate = activateFeedbackSink({
+      activity() {},
+      diagnostic() {},
+      interrupted() {},
+      reporterError(input) {
+        calls.push(input);
+      },
+      failure() {},
+      budgetExhausted() {},
+      kept() {},
+      lifecycle() {},
+    });
+    return fn(calls).finally(deactivate);
+  }
+
+  it("required reporter 抛错:reportReporterError 收到注册时的真实 name 与 required=true,而不是 stage 名字", () =>
+    withFakeSink(async (calls) => {
+      const reg: ReporterRegistration = { reporter: {}, name: "artifacts", required: true };
+      await runReporter(reg, "onEvalComplete", () => {
+        throw new Error("disk full");
+      });
+      expect(calls).toHaveLength(1);
+      expect(calls[0]).toMatchObject({ reporter: "artifacts", required: true });
+      // stage 是失败发生阶段的次要上下文,拼进 message,不覆盖 reporter 字段本身。
+      expect(calls[0]!.message).toContain("onEvalComplete");
+      expect(calls[0]!.message).toContain("disk full");
+    }));
+
+  it("message 只含 formatThrown() 的第一行,不把完整 .stack(本地绝对路径 + 调用帧)灌进机器 envelope", () =>
+    withFakeSink(async (calls) => {
+      const reg: ReporterRegistration = { reporter: {}, name: "json", required: true };
+      await runReporter(reg, "onRunComplete", () => {
+        // 真实 Error 的 .stack 恒为多行:第一行 "Error: message",之后每行一个 "    at ..." 调用帧
+        // (含本地绝对文件路径)。reportReporterError 的 message 是喂给 agent/ci 的单行 key=value
+        // envelope 里的一个字段值,不是 EvalResult.error 那种有专门落盘位置的完整记录——必须只取
+        // 第一行,不能把调用栈和本地路径原样透传出去。
+        throw new Error("EISDIR: illegal operation on a directory, rename");
+      });
+      expect(calls).toHaveLength(1);
+      const { message } = calls[0]!;
+      expect(message).toBe("onRunComplete: Error: EISDIR: illegal operation on a directory, rename");
+      expect(message).not.toContain("\n");
+      expect(message).not.toContain("    at ");
+      expect(message).not.toContain(import.meta.url.replace("file://", "")); // 本文件自己的绝对路径不出现在调用帧里
+    }));
+
+  it("best-effort reporter(如 config.reporters)抛错同样上报,但 required=false", () =>
+    withFakeSink(async (calls) => {
+      const reg: ReporterRegistration = { reporter: {}, name: "config-reporter-0", required: false };
+      await runReporter(reg, "onRunComplete", () => {
+        throw new Error("network blip");
+      });
+      expect(calls[0]).toMatchObject({ reporter: "config-reporter-0", required: false });
+    }));
+
+  it("runReporter 永不 reject——即便 reporter 抛错,调用方(Promise.all 聚合)仍能等到它 resolve", () =>
+    withFakeSink(async () => {
+      await expect(
+        runReporter({ reporter: {}, name: "x", required: true }, "onRunStart", () => {
+          throw new Error("boom");
+        }),
+      ).resolves.toBeUndefined();
+    }));
+
+  it("emitReporterEvent 对每个注册项独立兜错:一个 reporter 抛错不阻止其它 reporter 收到同一个事件", () =>
+    withFakeSink(async (calls) => {
+      const seen: string[] = [];
+      const throwing: ReporterRegistration = {
+        reporter: {
+          onEvent: () => {
+            throw new Error("boom");
+          },
+        },
+        name: "throwing",
+        required: false,
+      };
+      const ok: ReporterRegistration = {
+        reporter: { onEvent: (e) => void seen.push(e.type) },
+        name: "ok",
+        required: false,
+      };
+      await emitReporterEvent([throwing, ok], { type: "run:earlyExit", evalId: "e/1" });
+      expect(seen).toEqual(["run:earlyExit"]); // ok reporter 仍然收到了同一个事件
+      expect(calls).toHaveLength(1);
+      expect(calls[0]).toMatchObject({ reporter: "throwing", required: false });
+    }));
+
+  it("没有活跃 coordinator 时(run 未激活 / 已 finish)退回 bootstrap stderr,不吞错、仍然 resolve", async () => {
+    expect(activeFeedbackSinkCount()).toBe(0); // 确认这条测试真的走的是「没有活跃 sink」分支
+    const writes: string[] = [];
+    const spy = vi.spyOn(process.stderr, "write").mockImplementation((chunk: unknown) => {
+      writes.push(String(chunk));
+      return true;
+    });
+    try {
+      await expect(
+        runReporter({ reporter: {}, name: "artifacts", required: true }, "onRunStart", () => {
+          throw new Error("disk full");
+        }),
+      ).resolves.toBeUndefined();
+      expect(writes.join("")).toContain("disk full");
+    } finally {
+      spy.mockRestore();
+    }
+  });
+});

package/src/runner/report.ts

@@ -1,22 +1,46 @@
 // reporter 编排与运行级汇总。reporter 是「结果消费方」:单个 reporter 抛错只记
 // diagnostic,不能让整次调度崩。
 
-import type { EvalResult, LocalizedText, Reporter, ReporterEvent, RunShape, RunSummary } from "../types.ts";
-import { t } from "../i18n/index.ts";
-import { formatThrown } from "../util.ts";
-import { writeStderrLine } from "../tty-line.ts";
+import type {
+  EvalResult,
+  LocalizedText,
+  Reporter,
+  ReporterEvent,
+  ReporterRegistration,
+  RunShape,
+  RunSummary,
+} from "../types.ts";
+import { firstLine, formatThrown } from "../util.ts";
+import { reportReporterError } from "./feedback/sink.ts";
 
-/** reporter 调用的统一兜错。返回 void,永不 reject(供 Promise.all 安全聚合)。 */
-export async function runReporter(stage: string, fn: () => unknown): Promise<void> {
+/**
+ * reporter 调用的统一兜错。返回 void,永不 reject(供 Promise.all 安全聚合,单个 reporter
+ * 抛错不终止其它 reporter 的收尾,也不中断在飞的 attempt)。`reg.name`/`reg.required` 决定
+ * `reportReporterError()` 收到的诊断身份与判定权重(见 `ReporterRegistration` 的字段注释:
+ * 同一个 reporter 反复失败按 `name` 去重折叠,`required` 决定它是否让 completion/CI 退出码
+ * 判红)。`stage` 只是这次失败发生在哪个回调阶段的次要上下文,拼进 message,不参与去重身份——
+ * 同一 reporter 在 onEvalComplete 与 onRunComplete 两个阶段各失败一次,仍然折成同一条诊断、
+ * count 累加到 2,而不是拆成两条互不相关的诊断。message 只取 `formatThrown()` 的第一行(与
+ * run.ts 的 `describeFailureReason` 同一原则)——reporter 抛出的 Error 完整 `.stack` 含本地
+ * 绝对文件路径和调用帧,不适合塞进 agent/ci 那种单行、稳定、机器消费的 envelope;这里没有
+ * `EvalResult.error` 那样的落盘字段能保留完整栈,所以「一层可行动摘要」就是这条诊断的全部内容,
+ * 不是从更详细的记录里摘出来的简化版——排查真正需要栈时,重跑该 reporter 单独复现更可靠。
+ */
+export async function runReporter(reg: ReporterRegistration, stage: string, fn: () => unknown): Promise<void> {
   try {
     await fn();
   } catch (e) {
-    writeStderrLine(t("runner.reporterDiagnostic", { stage, message: formatThrown(e) }));
+    reportReporterError({ reporter: reg.name, required: reg.required, message: `${stage}: ${firstLine(formatThrown(e))}` });
   }
 }
 
-export async function emitReporterEvent(reporters: readonly Reporter[], event: ReporterEvent): Promise<void> {
-  await Promise.all(reporters.map((r) => runReporter(`event:${event.type}`, () => r.onEvent?.(event))));
+export async function emitReporterEvent(
+  registrations: readonly ReporterRegistration[],
+  event: ReporterEvent,
+): Promise<void> {
+  await Promise.all(
+    registrations.map((reg) => runReporter(reg, `event:${event.type}`, () => reg.reporter.onEvent?.(event))),
+  );
 }
 
 /** 按 eval id 过滤 RunSummary 并重新计数 —— eval 级 reporter 只看它观测的那部分。 */

package/src/runner/reporters/artifacts.ts

@@ -7,7 +7,7 @@
 // 重数据分文件,snapshot.json 只留快照元数据,view 展开某条 trace 时再按需 fetch 它的 trace.json。
 
 import { readFile } from "node:fs/promises";
-import type { Reporter } from "../../types.ts";
+import type { Reporter, RunShape } from "../../types.ts";
 import { createResultsWriter, type ResultsWriter } from "../../results/writer.ts";
 
 /** niceeval 自身的 npm 版本,写进 producer.version;版本不匹配时读取器靠它拼 npx 提示。 */
@@ -28,11 +28,17 @@ export function Artifacts(root = ".niceeval"): ArtifactsReporter {
   return {
     outputDirs: () => writer?.snapshotDirs() ?? [],
 
-    async onRunStart() {
+    async onRunStart(_evals, _agent, shape?: RunShape) {
       // 每次 run 换一个新 writer(同一个 reporter 实例可能被复用):writer 内部按
       // experimentId 懒建各自的快照目录,这里只重置引用。
+      // snapshotStartedAt 显式接收自 runner(shape.snapshotStartedAt,见 RunShape 的注释)——
+      // 不再各自按「该 experiment 第一条落盘 result 的 attempt startedAt」猜,与 runner
+      // 在 result.locator 里编码的身份锚点是同一个值。省略只出现在没有真实 shape 的
+      // 直调场景(如测试手写 Reporter 调用),此时退回 createResultsWriter 自己的兜底
+      // (result.startedAt,见 writer.ts 的 ResultsWriterOptions.snapshotStartedAt)。
       writer = createResultsWriter(root, {
         producer: { name: "niceeval", version: await producerVersion() },
+        snapshotStartedAt: shape?.snapshotStartedAt,
       });
     },
 

package/src/runner/reporters/braintrust.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
 import { describe, expect, it } from "vitest";
 import { toBraintrustEvent } from "./braintrust.ts";
 import type { EvalResult } from "../../types.ts";
@@ -19,8 +20,8 @@ describe("toBraintrustEvent", () => {
     const event = toBraintrustEvent(
       baseResult({
         assertions: [
-          { name: "compiles", severity: "gate", score: 1, passed: true },
-          { name: "closedQA", severity: "soft", score: 0.7, passed: true },
+          { name: "compiles", severity: "gate", score: 1, outcome: "passed" as const },
+          { name: "closedQA", severity: "soft", score: 0.7, outcome: "passed" as const },
         ],
       }),
     );
@@ -31,9 +32,9 @@ describe("toBraintrustEvent", () => {
     const event = toBraintrustEvent(
       baseResult({
         assertions: [
-          { name: "check", severity: "soft", score: 0.5, passed: true },
-          { name: "check", severity: "soft", score: 2, passed: true },
-          { name: "check", severity: "soft", score: -1, passed: false },
+          { name: "check", severity: "soft", score: 0.5, outcome: "passed" as const },
+          { name: "check", severity: "soft", score: 2, outcome: "passed" as const },
+          { name: "check", severity: "soft", score: -1, outcome: "failed" as const },
         ],
       }),
     );
@@ -85,8 +86,8 @@ describe("toBraintrustEvent", () => {
         model: "gpt-5.2",
         attempt: 1,
         verdict: "failed",
-        experiment: { id: "compare/codex", flags: { tape: true } },
-        assertions: [{ name: "compiles", severity: "gate", score: 0, passed: false, detail: "tsc failed" }],
+        experiment: { flags: { tape: true }, runs: 1, earlyExit: true, selectedEvalIds: [] },
+        assertions: [{ name: "compiles", severity: "gate", score: 0, outcome: "failed" as const, detail: "tsc failed" }],
       }),
     );
     expect(event.metadata).toEqual({

package/src/runner/reporters/braintrust.ts

@@ -4,6 +4,7 @@
 // (reporter 错误按框架约定只记 diagnostic,不会让运行崩)。
 
 import type { EvalResult, Reporter } from "../../types.ts";
+import { reportActivity } from "../feedback/sink.ts";
 
 export interface BraintrustConfig {
   /** Braintrust 项目名;省略时用 "niceeval"。 */
@@ -98,7 +99,7 @@ export function Braintrust(config: BraintrustConfig = {}): Reporter {
         await sdk?.flush();
         const summary = await experiment.summarize();
         if (summary.experimentUrl) {
-          process.stderr.write(`Braintrust experiment: ${summary.experimentUrl}\n`);
+          reportActivity(`Braintrust experiment: ${summary.experimentUrl}`);
         }
       } finally {
         experiment = undefined;
@@ -118,6 +119,8 @@ export function Braintrust(config: BraintrustConfig = {}): Reporter {
 export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
   const scores: Record<string, number> = {};
   for (const a of result.assertions) {
+    // unavailable 是没有分数的独立态:不写 0 分(评不了 ≠ 0 分),缺口进 metadata。
+    if (a.outcome === "unavailable") continue;
     const base = a.severity === "gate" ? `gate:${a.name}` : a.name;
     let key = base;
     for (let n = 2; key in scores; n++) key = `${base}#${n}`;
@@ -155,9 +158,13 @@ export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
   }
   if (result.skipReason !== undefined) metadata.skipReason = result.skipReason;
   const failed = result.assertions
-    .filter((a) => !a.passed)
+    .filter((a) => a.outcome === "failed")
     .map((a) => ({ name: a.name, detail: a.detail }));
   if (failed.length > 0) metadata.failedAssertions = failed;
+  const unavailable = result.assertions
+    .filter((a) => a.outcome === "unavailable")
+    .map((a) => ({ name: a.name, reason: a.outcome === "unavailable" ? a.reason : "" }));
+  if (unavailable.length > 0) metadata.unavailableAssertions = unavailable;
 
   // 一次运行内 (experiment, eval, agent, model, attempt) 唯一;Braintrust 按 id 合并重复行。
   const id = [result.experimentId ?? "", result.id, result.agent, result.model ?? "", `a${result.attempt}`].join("|");

package/src/runner/reporters/index.ts

@@ -1,7 +1,7 @@
-// 内置报告器:Console(默认)/ Artifacts / Json / JUnit / Braintrust。
+// 内置报告器:Artifacts(默认落盘)/ Json / JUnit / Braintrust。
 // 其它第三方实验跟踪平台也走同一条 Reporter 通道。
+// 终端反馈不是 Reporter——三种 --output profile 见 src/runner/feedback/。
 
-export { Console } from "./console.ts";
 export { Artifacts } from "./artifacts.ts";
 export { Json, JUnit } from "./json.ts";
 export { Braintrust, type BraintrustConfig } from "./braintrust.ts";

package/src/runner/reporters/json.test.ts

@@ -0,0 +1,162 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+// Json/JUnit 的同目录 temp → write → rename 原子替换契约(见 docs/feature/experiments/cli.md
+// 「输出流和落盘节奏」):成功后不留 temp;写入失败保留旧目标文件、不留半成品、不留 temp。
+//
+// `node:fs/promises` 的具名导出在这个 Vitest 环境下是不可配置的 ESM 绑定(`vi.spyOn` 直接
+// 报 "Module namespace is not configurable")——只能靠 `vi.mock` 在模块解析层整体替换,
+// 其余全部转发给真实实现(`importOriginal`),只在 `failureState` 显式设置时让 rename/
+// writeFile 失败一次(用后即清,不影响后续调用或其它测试)。
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { mkdtemp, readdir, readFile, rm, writeFile as realWriteFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Json, JUnit } from "./json.ts";
+import type { RunSummary } from "../../types.ts";
+
+const failureState = vi.hoisted(() => ({
+  renameError: undefined as Error | undefined,
+  writeFileError: undefined as Error | undefined,
+}));
+
+vi.mock("node:fs/promises", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("node:fs/promises")>();
+  return {
+    ...actual,
+    rename: async (...args: Parameters<typeof actual.rename>) => {
+      if (failureState.renameError) {
+        const e = failureState.renameError;
+        failureState.renameError = undefined;
+        throw e;
+      }
+      return actual.rename(...args);
+    },
+    writeFile: async (...args: Parameters<typeof actual.writeFile>) => {
+      if (failureState.writeFileError) {
+        const e = failureState.writeFileError;
+        failureState.writeFileError = undefined;
+        throw e;
+      }
+      return actual.writeFile(...args);
+    },
+  };
+});
+
+function summary(overrides: Partial<RunSummary> = {}): RunSummary {
+  return {
+    agent: "codex",
+    startedAt: "2026-07-07T00:00:00.000Z",
+    completedAt: "2026-07-07T00:01:00.000Z",
+    passed: 1,
+    failed: 0,
+    skipped: 0,
+    errored: 0,
+    durationMs: 1000,
+    results: [],
+    ...overrides,
+  };
+}
+
+const roots: string[] = [];
+async function makeRoot(): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "niceeval-json-atomic-"));
+  roots.push(root);
+  return root;
+}
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
+  failureState.renameError = undefined;
+  failureState.writeFileError = undefined;
+});
+
+/** 目录里是否残留 `.tmp` 临时文件(atomicWriteFile 用 `.niceeval-<uuid>.tmp` 命名)。 */
+async function tmpFilesIn(dir: string): Promise<string[]> {
+  return (await readdir(dir)).filter((name) => name.endsWith(".tmp"));
+}
+
+describe("Json · 原子替换", () => {
+  it("成功写入合法 JSON,写完不留 temp 文件", async () => {
+    const root = await makeRoot();
+    const path = join(root, "out.json");
+
+    await Json(path).onRunComplete!(summary({ passed: 3 }));
+
+    const parsed = JSON.parse(await readFile(path, "utf-8"));
+    expect(parsed.passed).toBe(3);
+    expect(await tmpFilesIn(root)).toEqual([]);
+  });
+
+  it("rename 失败时保留旧目标文件原内容,不留截断 JSON,不留 temp 文件", async () => {
+    const root = await makeRoot();
+    const path = join(root, "out.json");
+    await realWriteFile(path, '{"old":"content"}', "utf-8"); // 上一轮已经成功写过一份
+
+    failureState.renameError = new Error("simulated rename failure");
+    await expect(Json(path).onRunComplete!(summary({ passed: 99 }))).rejects.toThrow("simulated rename failure");
+
+    // 旧内容原封不动——没有被替换成半成品,也没有被新内容(passed: 99)覆盖。
+    expect(await readFile(path, "utf-8")).toBe('{"old":"content"}');
+    expect(await tmpFilesIn(root)).toEqual([]);
+  });
+
+  it("写临时文件本身失败时不产生新目标文件,也不留 temp 文件", async () => {
+    const root = await makeRoot();
+    const path = join(root, "out.json"); // 目标此前不存在
+
+    failureState.writeFileError = new Error("simulated write failure");
+    await expect(Json(path).onRunComplete!(summary())).rejects.toThrow("simulated write failure");
+
+    expect(await readdir(root)).toEqual([]); // 没有凭空出现半成品目标文件
+  });
+
+  it("rename 目标是一个已存在目录时必然失败(POSIX 语义,不依赖权限/是否 root)——旧目录原样保留,不留 temp 文件", async () => {
+    const root = await makeRoot();
+    const path = join(root, "out.json");
+    const { mkdir } = await import("node:fs/promises");
+    await mkdir(path); // 占用目标路径的不是文件而是目录——rename 一个文件覆盖已存在目录恒失败
+
+    await expect(Json(path).onRunComplete!(summary())).rejects.toThrow();
+
+    const entries = await readdir(root);
+    expect(entries).toEqual(["out.json"]); // 目录还在,没有被替换、也没有多出 temp 文件残留
+  });
+});
+
+describe("JUnit · 原子替换", () => {
+  it("成功写入合法 XML,写完不留 temp 文件", async () => {
+    const root = await makeRoot();
+    const path = join(root, "out.xml");
+
+    await JUnit(path).onRunComplete!(
+      summary({
+        results: [
+          {
+            id: "a/1",
+            agent: "codex",
+            verdict: "passed",
+            attempt: 0,
+            durationMs: 10,
+            assertions: [],
+          },
+        ],
+      }),
+    );
+
+    const xml = await readFile(path, "utf-8");
+    expect(xml).toContain("<?xml");
+    expect(xml).toContain("<testsuite");
+    expect(await tmpFilesIn(root)).toEqual([]);
+  });
+
+  it("rename 失败时保留旧目标文件原内容,不留截断 XML,不留 temp 文件", async () => {
+    const root = await makeRoot();
+    const path = join(root, "out.xml");
+    await realWriteFile(path, "<old/>", "utf-8");
+
+    failureState.renameError = new Error("simulated rename failure");
+    await expect(JUnit(path).onRunComplete!(summary())).rejects.toThrow("simulated rename failure");
+
+    expect(await readFile(path, "utf-8")).toBe("<old/>");
+    expect(await tmpFilesIn(root)).toEqual([]);
+  });
+});

package/src/runner/reporters/json.ts

@@ -1,14 +1,42 @@
 // JSON / JUnit 报告器:把运行结果落成机器可读 artifact,接 CI 或下游 dashboard。
+// 两者都是「整次运行结束时写一份聚合文件」(不像 Artifacts 的逐 attempt 增量落盘),见
+// docs/feature/experiments/cli.md「输出流和落盘节奏」——一旦开始写就必须原子替换目标:
+// 半途失败(磁盘满、权限问题、进程被杀)不能把上一次成功的 JSON/JUnit 覆盖成截断文件,
+// 那会让 CI 读到一份看似存在、实际损坏的报告,比「文件不存在」更危险。
 
-import { mkdir, writeFile } from "node:fs/promises";
-import { dirname } from "node:path";
+import { mkdir, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { randomUUID } from "node:crypto";
 import type { Reporter, RunSummary } from "../../types.ts";
 
+/**
+ * 同目录 temp → write → rename 的原子替换:内容先整体写进同目录下的临时文件,写完才
+ * `rename()` 覆盖真正的目标路径——`rename()` 在同一文件系统内是单一原子操作,读者不可能
+ * 观察到「文件存在但内容只写了一半」的中间态。同目录是必须的,跨目录/跨文件系统的
+ * rename 不保证原子性(可能退化成先复制再删除)。
+ *
+ * 失败路径:临时文件写入或 rename 本身抛错时,`rename()` 还没被调用或没有成功,旧目标文件
+ * (如果之前存在)完全没被触碰;没有旧目标文件时目录里也不会凭空出现一个新的。失败后立即
+ * 清理临时文件(`rm force`,即便清理本身失败也吞掉——不能让「清理垃圾」的次要错误掩盖调用方
+ * 真正需要看到的写入失败),成功路径下 `rename()` 之后临时文件已经不在原路径,同样不留残留。
+ */
+async function atomicWriteFile(path: string, content: string): Promise<void> {
+  const dir = dirname(path);
+  await mkdir(dir, { recursive: true });
+  const tmpPath = join(dir, `.niceeval-${randomUUID()}.tmp`);
+  try {
+    await writeFile(tmpPath, content, "utf-8");
+    await rename(tmpPath, path);
+  } catch (e) {
+    await rm(tmpPath, { force: true }).catch(() => {});
+    throw e;
+  }
+}
+
 export function Json(path: string): Reporter {
   return {
     async onRunComplete(summary: RunSummary) {
-      await mkdir(dirname(path), { recursive: true });
-      await writeFile(path, JSON.stringify(summary, null, 2), "utf-8");
+      await atomicWriteFile(path, JSON.stringify(summary, null, 2));
     },
   };
 }
@@ -16,16 +44,15 @@ export function Json(path: string): Reporter {
 export function JUnit(path: string): Reporter {
   return {
     async onRunComplete(summary: RunSummary) {
-      await mkdir(dirname(path), { recursive: true });
       const cases = summary.results
         .map((r) => {
           const name = xmlAttr(`${r.id} [${r.agent}${r.model ? "/" + r.model : ""}]`);
           const time = (r.durationMs / 1000).toFixed(3);
           if (r.verdict === "errored") {
-            return `    <testcase name="${name}" time="${time}"><error message="${xmlAttr(r.error ?? "execution error")}"/></testcase>`;
+            return `    <testcase name="${name}" time="${time}"><error message="${xmlAttr(r.error?.message ?? "execution error")}"/></testcase>`;
           }
           if (r.verdict === "failed") {
-            const msg = xmlAttr(r.assertions.filter((a) => !a.passed).map((a) => a.name).join("; "));
+            const msg = xmlAttr(r.assertions.filter((a) => a.outcome === "failed").map((a) => a.name).join("; "));
             return `    <testcase name="${name}" time="${time}"><failure message="${msg}"/></testcase>`;
           }
           if (r.verdict === "skipped") {
@@ -38,7 +65,7 @@ export function JUnit(path: string): Reporter {
         `<?xml version="1.0" encoding="UTF-8"?>\n` +
         `<testsuite name="niceeval" tests="${summary.results.length}" failures="${summary.failed}" errors="${summary.errored}" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">\n` +
         `${cases}\n</testsuite>\n`;
-      await writeFile(path, xml, "utf-8");
+      await atomicWriteFile(path, xml);
     },
   };
 }

package/src/runner/reporters/shared.ts

@@ -1,5 +1,4 @@
-// reporter 共用的展示常量。符号表只此一份 —— console/live/table 各抄一份时,
-// 改一个符号要同步三处且不报错。
+// verdict 符号表,当前唯一消费方是 src/runner/feedback/human.ts 的 dashboard/完成页。
 
 import type { Verdict } from "../../types.ts";
 
@@ -13,6 +12,3 @@ export const VERDICT_SYM: Record<Verdict, string> = {
 export function verdictSymbol(verdict: string): string {
   return VERDICT_SYM[verdict as Verdict] ?? "?";
 }
-
-/** live 表格里「还没抢到并发名额」的行:和转圈的 SPINNER、完成后的 VERDICT_SYM 三态区分开。 */
-export const WAITING_SYM = "·";