niceeval 0.6.1 → 0.7.0

package/src/results/publish.ts

@@ -0,0 +1,149 @@
+// 发布消毒与发布预算(见 docs/feature/results/library.md「复制与瘦身:copySnapshots」)。
+// 结果数据分两个等级:.niceeval/ 是本地事实根(未消毒);任何要离开本机的拷贝是发布拷贝,
+// 只能经 copySnapshots 这一条管线产出,消毒在这里做且没有隐式默认(redact 必填)。
+//
+// redact 逐值消毒,范围由 schema 的自由文本标注决定:redactor 只对自由文本字段调用——
+// 格式、判定、身份、路径、哈希这类结构字段永不经过它,发布根不会因 redact 变得不可读或
+// 引用断裂。自由文本清单在下面的 FREE_TEXT 标注单点维护(与 AttemptList.redact 同一口径)。
+
+import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
+
+/** 发布前整文件预检的单文件上限(50 MiB,为 GitHub 100 MB 硬限保留余量);不是可调旋钮。 */
+export const PUBLISH_FILE_MAX_BYTES = 50 * 1024 * 1024;
+
+export type Redactor = (text: string) => string;
+
+/**
+ * 结构字段键名(取值是格式、判定、身份、路径、哈希——redactor 永不触碰):
+ * 事件的 type/callId/status/role/tool/skill/requestId/optionId、span 的 ids/kind、
+ * 断言的 name/severity/outcome、错误的 code/phase、locator/artifactBase/fingerprint、
+ * 源码路径与 sha256、provider 名、时间戳等。新增字符串字段先判断体裁再决定进不进这张表。
+ */
+const STRUCTURAL_KEYS = new Set([
+  "type",
+  "callId",
+  "status",
+  "role",
+  "tool",
+  "requestId",
+  "optionId",
+  "id",
+  "traceId",
+  "spanId",
+  "parentSpanId",
+  "kind",
+  "format",
+  "verdict",
+  "severity",
+  "outcome",
+  "artifactBase",
+  "locator",
+  "fingerprint",
+  "sha256",
+  "path",
+  "file",
+  "provider",
+  "sandboxId",
+  "code",
+  "phase",
+  "level",
+  "agent",
+  "model",
+  "experimentId",
+  "startedAt",
+  "completedAt",
+  "schemaVersion",
+  "evalFilterFingerprint",
+  "reasoningEffort",
+  "skill",
+  "window",
+  "net",
+  "loc",
+  "dedupeKey",
+]);
+
+/** 深度遍历 JSON 值:字符串按「键名是否结构字段」决定过不过 redactor;结构键下整棵子树跳过。 */
+export function redactJsonValue(value: unknown, redact: Redactor, key?: string): unknown {
+  if (typeof value === "string") {
+    if (key !== undefined && STRUCTURAL_KEYS.has(key)) return value;
+    return redact(value);
+  }
+  if (Array.isArray(value)) return value.map((v) => redactJsonValue(v, redact, key));
+  if (value !== null && typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (STRUCTURAL_KEYS.has(k) && typeof v === "string") {
+        out[k] = v;
+        continue;
+      }
+      out[k] = redactJsonValue(v, redact, k);
+    }
+    return out;
+  }
+  return value;
+}
+
+/** events.json 的发布消毒:消息与工具入出参是自由文本,type/callId/status 等结构字段不动。 */
+export function redactEvents(events: StreamEvent[], redact: Redactor): StreamEvent[] {
+  return redactJsonValue(events, redact) as StreamEvent[];
+}
+
+/** trace.json 的发布消毒:属性值与可携带动态内容的 span name 过 redactor,ids/kind 不动。 */
+export function redactSpans(spans: TraceSpan[], redact: Redactor): TraceSpan[] {
+  return spans.map((span) => ({
+    ...span,
+    name: redact(span.name),
+    ...(span.attributes !== undefined
+      ? { attributes: redactJsonValue(span.attributes, redact) as TraceSpan["attributes"] }
+      : {}),
+  }));
+}
+
+/** result.json 的发布消毒:断言 detail/evidence/expected/received、error/diagnostic 的
+ *  message/cause/stack、skipReason、description 是自由文本;判定与身份字段不动。 */
+export function redactResultRecord(record: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
+  const r = record as Partial<EvalResult> & Record<string, unknown>;
+  const out: Record<string, unknown> = { ...record };
+  if (typeof r.description === "string") out.description = redact(r.description);
+  if (typeof r.skipReason === "string") out.skipReason = redact(r.skipReason);
+  if (Array.isArray(r.assertions)) {
+    out.assertions = r.assertions.map((a) => ({
+      ...a,
+      ...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
+      ...(a.outcome !== "unavailable" && a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
+      ...(a.outcome !== "unavailable" && a.expected !== undefined ? { expected: redact(a.expected) } : {}),
+      ...(a.outcome !== "unavailable" && a.received !== undefined ? { received: redact(a.received) } : {}),
+    }));
+  }
+  if (r.error !== undefined) {
+    out.error = {
+      ...r.error,
+      message: redact(r.error.message),
+      ...(r.error.stack !== undefined ? { stack: redact(r.error.stack) } : {}),
+      ...(r.error.cause !== undefined ? { cause: { ...r.error.cause, message: redact(r.error.cause.message) } } : {}),
+    };
+  }
+  if (Array.isArray(r.diagnostics)) {
+    out.diagnostics = r.diagnostics.map((d) => ({
+      ...d,
+      message: redact(d.message),
+      ...(d.data !== undefined ? { data: redactJsonValue(d.data, redact) as typeof d.data } : {}),
+    }));
+  }
+  if (r.experiment !== undefined) {
+    out.experiment = redactExperimentInfo(r.experiment as unknown as Record<string, unknown>, redact);
+  }
+  return out;
+}
+
+/** ExperimentRunInfo 的发布消毒:description、flags 与 sandbox params 的字符串值。 */
+export function redactExperimentInfo(info: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
+  const out: Record<string, unknown> = { ...info };
+  if (typeof info.description === "string") out.description = redact(info.description);
+  if (info.flags !== undefined) out.flags = redactJsonValue(info.flags, redact);
+  const sandbox = info.sandbox as { provider: string; params?: Record<string, unknown> } | undefined;
+  if (sandbox?.params !== undefined) {
+    out.sandbox = { ...sandbox, params: redactJsonValue(sandbox.params, redact) };
+  }
+  return out;
+}

package/src/results/results.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/results/cases.md
 // niceeval/results 的单测:临时目录里构造最小 snapshot.json / result.json / artifact fixture,
 // 覆盖定稿契约(docs/feature/results/library.md、docs/feature/results/architecture.md):分层读取(含快照级字段注入)、
 // 懒加载与 artifactBase 回退、skipped 三种原因、unfinished-snapshot、latest() 三种警告、
@@ -342,7 +343,7 @@ describe("results.latest() · Selection", () => {
     const stale = latest.warnings.filter((w) => w.kind === "stale-snapshot");
     expect(stale).toHaveLength(1);
     expect(stale[0]).toMatchObject({ experimentId: "mid/a", startedAt: "2026-07-01T08:00:00.000Z", latestStartedAt: "2026-07-05T08:00:00.000Z" });
-    expect(stale[0].message).toContain("predates the latest run in this selection by 4 days");
+    expect(stale[0].message).toContain("predates the latest run in this scope by 4 days");
   });
 
   it("unfinished-snapshot:选中快照缺 completedAt", async () => {
@@ -464,7 +465,7 @@ describe("createResultsWriter", () => {
     await snapA.writeAttempt({ id: "q2", verdict: "failed", attempt: 1, durationMs: 50, assertions: [] });
 
     const snapB = await writer.snapshot({ experimentId: "compare/b", agent: "codex", startedAt: "2026-07-02T09:00:00.000Z" });
-    await snapB.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 80, assertions: [] }, { diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] } });
+    await snapB.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 80, assertions: [] }, { diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }] });
 
     await writer.finish();
     expect(writer.snapshotDirs().map((s) => s.experimentId).sort()).toEqual(["compare/a", "compare/b"]);
@@ -506,7 +507,9 @@ describe("createResultsWriter", () => {
 
     const b = results.experiments[1].latest;
     expect(b.model).toBeUndefined();
-    expect(await b.attempts[0].diff()).toEqual({ generatedFiles: { "a.txt": "1" }, deletedFiles: [] });
+    const bDiff = await b.attempts[0].diff();
+    expect(bDiff?.windows).toEqual([{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }]);
+    expect(bDiff?.get("a.txt")).toBe("1");
 
     const partial = results.latest().warnings.find((w) => w.kind === "partial-coverage")!;
     expect(partial).toMatchObject({ experimentId: "compare/a", covered: 2, total: 3 });
@@ -549,7 +552,7 @@ describe("createResultsWriter", () => {
     expect(await q2.agentSetup()).toBeNull();
 
     const dest = join(await makeRoot(), "published");
-    await copySnapshots(results.latest(), dest, { artifacts: ["agentSetup"] });
+    await copySnapshots(results.latest(), dest, { ...{ artifacts: ["agentSetup"] }, redact: false });
     const copied = join(dest, "skill-ab_claude-effect", basename(snap.dir), "q1", "a1", "agent-setup.json");
     expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
   });
@@ -590,27 +593,32 @@ describe("createResultsWriter", () => {
 
   it("快照目录独占创建:撞名换随机后缀重试直到成功(EEXIST 不会覆盖已有目录)", async () => {
     const root = await makeRoot();
-    vi.useFakeTimers();
-    vi.setSystemTime(new Date("2026-07-01T08:00:00.000Z"));
+    // 受控随机序列:writer B 面对同一身份时重置序列,首次尝试重放 writer A 消耗过的
+    // 随机值,必然撞上 A 的目录;重试消耗序列后续值,得到不同后缀。测试不假设一个
+    // 后缀消耗几次 random,也不假设数值到后缀字符的映射。
     let call = 0;
     const randomSpy = vi.spyOn(Math, "random").mockImplementation(() => {
       call += 1;
-      return call <= 4 ? 0 : 0.03; // 第一次 randomSuffix() 产出 "aaaa",第二次产出 "bbbb"
+      return ((call * 37) % 977) / 977;
     });
     try {
-      const collidingDir = join(root, "e", "2026-07-01T08-00-00-000Z-aaaa");
-      await mkdir(collidingDir, { recursive: true });
-
-      const writer = createResultsWriter(root, { producer: { name: "x" } });
-      const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
-      expect(snap.dir).not.toBe(collidingDir);
-      expect(snap.dir.endsWith("-bbbb")).toBe(true);
-      expect(await exists(join(snap.dir, "snapshot.json"))).toBe(true);
-      // 被占用的目录没有被写入 snapshot.json —— 独占创建不会覆盖已有内容。
-      expect(await exists(join(collidingDir, "snapshot.json"))).toBe(false);
+      const identity = { experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" } as const;
+      const writerA = createResultsWriter(root, { producer: { name: "x" } });
+      const snapA = await writerA.snapshot(identity);
+      const canary = join(snapA.dir, "canary.txt");
+      await writeFile(canary, "occupied", "utf8");
+
+      call = 0; // 重放同一序列,让 writer B 的首次尝试与 snapA 同名
+      const writerB = createResultsWriter(root, { producer: { name: "x" } });
+      const snapB = await writerB.snapshot(identity);
+
+      expect(snapB.dir).not.toBe(snapA.dir);
+      expect(await exists(join(snapB.dir, "snapshot.json"))).toBe(true);
+      // 被占用的目录内容原样保留 —— 独占创建不会覆盖已有内容。
+      expect(await exists(canary)).toBe(true);
+      expect(await exists(join(snapB.dir, "canary.txt"))).toBe(false);
     } finally {
       randomSpy.mockRestore();
-      vi.useRealTimers();
     }
   });
 
@@ -621,7 +629,7 @@ describe("createResultsWriter", () => {
     const fresh: EvalResult = {
       id: "algebra/q1",
       experimentId: "compare/bub",
-      experiment: { id: "compare/bub", flags: { style: "concise" } },
+      experiment: { flags: { style: "concise" }, runs: 1, earlyExit: true, selectedEvalIds: ["algebra/q1"] },
       agent: "bub",
       model: "gpt-5.4",
       verdict: "passed",
@@ -636,7 +644,7 @@ describe("createResultsWriter", () => {
       sources: [{ path: "evals/a.ts", content: "x" }],
       trace: [{ name: "turn", kind: "turn" } as never],
       o11y: { toolCalls: 2 } as never,
-      diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] },
+      diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }],
       rawTranscript: "raw",
     };
     const carried: EvalResult = {
@@ -714,7 +722,7 @@ describe("copySnapshots", () => {
     await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true, hasTrace: true }));
     await writeArtifactFile(monday, "q1/a1", "events.json", [{ n: 1 }]);
     await writeArtifactFile(monday, "q1/a1", "trace.json", [{ name: "turn" }]);
-    await writeArtifactFile(monday, "q1/a1", "diff.json", { generatedFiles: {}, deletedFiles: [] });
+    await writeArtifactFile(monday, "q1/a1", "diff.json", [{ window: "s1/t1", changes: {} }]);
     await writeResultFile(monday, "q2/a1", record({ id: "q2", attempt: 1 }));
 
     // 周五只重跑了 q1:最新快照残缺。
@@ -729,7 +737,7 @@ describe("copySnapshots", () => {
 
     const results = await openResults(root);
     const dest = join(await makeRoot(), "site/data/run");
-    const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events"] });
+    const copied = await copySnapshots(results.latest(), dest, { ...{ artifacts: ["events"] }, redact: false });
 
     expect(copied.warnings).toHaveLength(0);
     expect(copied.dir).toBe(dest);
@@ -767,15 +775,15 @@ describe("copySnapshots", () => {
 
     const occupied = await makeRoot();
     await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
-    await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
+    await expect(copySnapshots(results.latest(), occupied, { redact: false })).rejects.toThrow(/not empty/);
 
-    await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] })).rejects.toThrow(/Unknown artifact kind/);
+    await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { ...{ artifacts: ["evnets" as never] }, redact: false })).rejects.toThrow(/Unknown artifact kind/);
 
-    await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
+    await expect(copySnapshots([], join(await makeRoot(), "out"), { redact: false })).rejects.toThrow(/no snapshots/);
 
     // 手工传入同一 experiment 的两个快照(未走 latest 去重):只带最新,记 warning。
     const dest2 = join(await makeRoot(), "run2");
-    const collided = await copySnapshots(results.experiments[0].snapshots, dest2);
+    const collided = await copySnapshots(results.experiments[0].snapshots, dest2, { redact: false });
     expect(collided.warnings).toHaveLength(1);
     expect(collided.warnings[0]).toMatch(/multiple snapshots selected/);
     const destDirs = await readdir(join(dest2, "e"));
@@ -968,8 +976,9 @@ describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
     const handMadeResults: Results = {
       experiments: [{ id: "e", snapshots: [snapshot], latest: snapshot, evalIds: ["q1"] }],
       skipped: [],
-      // filter() 本测试不调用,用不到,给个占位实现即可满足 Selection 接口。
-      latest: () => ({ snapshots: [snapshot], warnings: [], filter: () => { throw new Error("not implemented"); } }),
+      // filter() 本测试不调用,用不到,给个占位实现即可满足 Scope 接口。
+      latest: () => ({ mode: "latest-snapshots" as const, snapshots: [snapshot], attempts: snapshot.attempts, warnings: [], filter: () => { throw new Error("not implemented"); } }),
+      current: () => { throw new Error("not implemented"); },
     };
     // 这份 locator 语法合法、甚至真的对应 handMadeResults 里那个 attempt 的身份,
     // 但 handMadeResults 没经过 openResults(),locatorIndexByResults 里查不到它 —— 空索引,not-found。
@@ -996,48 +1005,74 @@ describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
     const locator1 = a1!.locator!;
 
     const dest = join(await makeRoot(), "published");
-    await copySnapshots(results.latest(), dest, { artifacts: [] });
+    await copySnapshots(results.latest(), dest, { ...{ artifacts: [] }, redact: false });
 
     const destResults = await openResults(dest);
     expect(resolveLocator(destResults, locator0).result.attempt).toBe(0);
     expect(resolveLocator(destResults, locator1).result.attempt).toBe(1);
   });
-});
-
-// ───────────────────────── sources 去重仓库 ─────────────────────────
 
-describe("sources · 快照级去重仓库", () => {
-  it("两个 attempt 共享字节相同的 eval 源码:去重仓库只落一份 blob(文件数 = 1)", async () => {
+  it("createResultsWriter({ snapshotStartedAt }):writeAttemptFor() 的隐式声明统一用这个锚点,不再按各 result 自己的 attempt startedAt 分别猜", async () => {
     const root = await makeRoot();
-    const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
-    const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
-    const content = "export default { test() {} };\n";
-    await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: [{ path: "evals/shared.eval.ts", content }] });
-    await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: [{ path: "evals/shared.eval.ts", content }] });
+    const snapshotStartedAt = "2026-07-10T00:00:00.000Z";
+    const writer = createResultsWriter(root, {
+      producer: { name: "niceeval", version: "1.0.0" },
+      snapshotStartedAt,
+    });
+
+    // 两个不同 experiment,各自的 result.startedAt 与 writer 级锚点都不同 —— 如果还在按
+    // 「该 experiment 首条落盘结果的 attempt 时刻」猜,两份 snapshot.json 的 startedAt 会
+    // 分别变成各自 result 的 attempt 时刻,而不是这里统一传入的 snapshotStartedAt。
+    await writer.writeAttemptFor({
+      id: "q1", experimentId: "e1", agent: "bub", verdict: "passed", attempt: 0,
+      startedAt: "2020-01-01T00:00:00.000Z", durationMs: 1, assertions: [],
+    });
+    await writer.writeAttemptFor({
+      id: "q1", experimentId: "e2", agent: "bub", verdict: "passed", attempt: 0,
+      startedAt: "2021-06-15T00:00:00.000Z", durationMs: 1, assertions: [],
+    });
     await writer.finish();
 
-    const storeFiles = await readdir(join(snap.dir, "sources"));
-    expect(storeFiles).toHaveLength(1);
+    const results = await openResults(root);
+    const expE1 = results.experiments.find((e) => e.id === "e1")!;
+    const expE2 = results.experiments.find((e) => e.id === "e2")!;
+    expect(expE1.latest.startedAt).toBe(snapshotStartedAt);
+    expect(expE2.latest.startedAt).toBe(snapshotStartedAt); // 两个 experiment 共用同一个锚点,不碰撞
+
+    // attempt 级 startedAt(墙钟事实)依然各自独立保留,没有被锚点覆盖 ——
+    // 快照身份锚点与 attempt 墙钟事实是两回事,继续分别保存。
+    const attempt1 = expE1.latest.evals[0]!.attempts[0]!;
+    const attempt2 = expE2.latest.evals[0]!.attempts[0]!;
+    expect(attempt1.result.startedAt).toBe("2020-01-01T00:00:00.000Z");
+    expect(attempt2.result.startedAt).toBe("2021-06-15T00:00:00.000Z");
+
+    // locator 由统一锚点 + experimentId 派生,两边不因为共享锚点而碰撞。
+    expect(attempt1.locator).not.toBe(attempt2.locator);
+    expect(attempt1.locator).toBe(
+      encodeAttemptLocator({ experimentId: "e1", snapshotStartedAt, evalId: "q1", attempt: 0 }),
+    );
+    expect(attempt2.locator).toBe(
+      encodeAttemptLocator({ experimentId: "e2", snapshotStartedAt, evalId: "q1", attempt: 0 }),
+    );
   });
 
-  it("两个 attempt 的 eval 源码内容不同:各自落一份 blob(文件数 = 2)", async () => {
+  it("createResultsWriter() 省略 snapshotStartedAt 时保留旧的按首条 result.startedAt 猜测行为(第三方直调兼容)", async () => {
     const root = await makeRoot();
     const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
-    const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
-    await snap.writeAttempt(
-      { id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
-      { sources: [{ path: "evals/a.eval.ts", content: "export default { test() {} };\n" }] },
-    );
-    await snap.writeAttempt(
-      { id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
-      { sources: [{ path: "evals/b.eval.ts", content: "export default { test() { /* different */ } };\n" }] },
-    );
+    await writer.writeAttemptFor({
+      id: "q1", experimentId: "e", agent: "bub", verdict: "passed", attempt: 0,
+      startedAt: "2020-03-03T00:00:00.000Z", durationMs: 1, assertions: [],
+    });
     await writer.finish();
 
-    const storeFiles = await readdir(join(snap.dir, "sources"));
-    expect(storeFiles).toHaveLength(2);
+    const results = await openResults(root);
+    expect(results.experiments[0]!.latest.startedAt).toBe("2020-03-03T00:00:00.000Z");
   });
+});
 
+// ───────────────────────── sources 去重仓库 ─────────────────────────
+
+describe("sources · 快照级去重仓库", () => {
   it("经真实 --resume carry 流程(writeAttemptFor 的 artifactBase 分支)携带的 attempt,其 sources() 引用在新快照里依然能解到原快照内容", async () => {
     const root = await makeRoot();
     const content = "export default { test() {} };\n";
@@ -1157,7 +1192,7 @@ describe("sources · 快照级去重仓库", () => {
 
     const results = await openResults(root);
     const dest = join(await makeRoot(), "published");
-    await copySnapshots(results.latest(), dest, { artifacts: ["sources"] });
+    await copySnapshots(results.latest(), dest, { ...{ artifacts: ["sources"] }, redact: false });
 
     const destSnapDir = join(dest, "e", basename(snap.dir));
     const destStoreFiles = await readdir(join(destSnapDir, "sources"));