niceeval 0.10.3-canary.7 → 0.11.0

package/src/runner/feedback/json.test.ts

@@ -8,8 +8,10 @@
 // docs/engineering/testing/e2e/cli.md「反馈输出格式」在真实进程输出上验收。
 
 import { describe, expect, it } from "vitest";
-import { computeExitCode, renderJsonPlanDocument } from "./json.ts";
-import type { InvocationCompletion, InvocationSummary } from "../types.ts";
+import { computeExitCode, createJsonRenderer, renderJsonPlanDocument } from "./json.ts";
+import { createInitialRunFeedbackState, reduceRunFeedback } from "./reducer.ts";
+import { createFakeFeedbackIO } from "./testing.ts";
+import { HALT_DIAGNOSTIC_CODE, type DurableFeedbackEvent, type InvocationCompletion, type InvocationSummary } from "../types.ts";
 
 function summary(overrides: Partial<InvocationSummary> = {}): InvocationSummary {
   return {
@@ -77,6 +79,113 @@ describe("computeExitCode:CompletionStatus 驱动退出码,不只看 failed/erro
   });
 });
 
+/** 依次喂进 reducer 再交给 json renderer(与生产的 coordinator 同序:先 reduce 后 render),
+ *  返回逐行解析出的事件对象。 */
+function emitDurable(events: readonly DurableFeedbackEvent[]): Record<string, unknown>[] {
+  const { io, stdout } = createFakeFeedbackIO();
+  const renderer = createJsonRenderer({ io });
+  let state = createInitialRunFeedbackState();
+  for (const event of events) {
+    state = reduceRunFeedback(state, event);
+    renderer.appendDurable(event, state);
+  }
+  return stdout.writes
+    .join("")
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as Record<string, unknown>);
+}
+
+describe("warning 事件:code 是稳定词法,折叠身份走具名字段", () => {
+  it("code 用 DiagnosticInput.code 的干净字面量,不透传编了身份的去重 key", () => {
+    const [warning] = emitDurable([
+      {
+        type: "diagnostic",
+        at: 1,
+        key: "lock-taken-over:compare/codex|memory/retention",
+        code: "lock-taken-over",
+        severity: "warning",
+        message: "took over a stale lock",
+        data: { experimentId: "compare/codex", evalId: "memory/retention" },
+      },
+    ]);
+    expect(warning).toMatchObject({
+      event: "warning",
+      code: "lock-taken-over",
+      level: "warning",
+      experimentId: "compare/codex",
+      evalId: "memory/retention",
+    });
+  });
+
+  it("没给 code 的诊断回落到 key(折叠身份本就不进 key 的那些天生是干净字面量)", () => {
+    const [warning] = emitDurable([
+      { type: "diagnostic", at: 1, key: "memory-warmup-degraded", severity: "warning", message: "cold index" },
+    ]);
+    expect(warning).toMatchObject({ code: "memory-warmup-degraded" });
+    expect(warning).not.toHaveProperty("experimentId");
+    expect(warning).not.toHaveProperty("evalId");
+  });
+
+  it("eval 闸的 dispatch-halted:code 干净、evalId 与 phase 都在事件流里透得出", () => {
+    const [warning] = emitDurable([
+      {
+        type: "diagnostic",
+        at: 1,
+        key: "dispatch-halted:eval:compare/codex|memory/retention",
+        code: HALT_DIAGNOSTIC_CODE,
+        severity: "error",
+        message: "eval halted: fixture db is empty; run scripts/seed.ts",
+        data: {
+          experimentId: "compare/codex",
+          scope: "eval",
+          evalId: "memory/retention",
+          phase: "eval.run",
+          unstarted: 0,
+        },
+      },
+    ]);
+    expect(warning).toMatchObject({
+      event: "warning",
+      code: "dispatch-halted",
+      level: "error",
+      phase: "eval.run",
+      experimentId: "compare/codex",
+      evalId: "memory/retention",
+    });
+  });
+
+  it("身份从 data 取(闸不是 attempt 级、不伪造 identity);有 identity 时 identity 优先", () => {
+    const [fromIdentity] = emitDurable([
+      {
+        type: "diagnostic",
+        at: 1,
+        key: "fail-fast:x",
+        code: "fail-fast",
+        severity: "warning",
+        message: "deterministic failure",
+        identity: { experimentId: "compare/codex", evalId: "memory/a", attempt: 0 },
+        data: { experimentId: "other/exp", evalId: "memory/z" },
+      },
+    ]);
+    expect(fromIdentity).toMatchObject({ experimentId: "compare/codex", evalId: "memory/a" });
+  });
+
+  it("同一 dedupeKey 只追加一次:emitter 为刷新 data.unstarted 反复报同一条闸,事件流不重复", () => {
+    const halted = (at: number, unstarted: number): DurableFeedbackEvent => ({
+      type: "diagnostic",
+      at,
+      key: "dispatch-halted:experiment:compare/codex",
+      code: HALT_DIAGNOSTIC_CODE,
+      severity: "error",
+      message: "experiment halted (dispatch-halted): shared service is down",
+      data: { experimentId: "compare/codex", scope: "experiment", phase: "eval.run", unstarted },
+    });
+    const events = emitDurable([halted(1, 0), halted(2, 1), halted(3, 2)]);
+    expect(events.filter((e) => e.event === "warning")).toHaveLength(1);
+  });
+});
+
 describe("renderJsonPlanDocument:单个 ExpPlanDocument,不是事件流", () => {
   it("输出恰好一行 JSON,可解析为单个对象而不是逐行事件序列", () => {
     const text = renderJsonPlanDocument({
@@ -101,6 +210,22 @@ describe("renderJsonPlanDocument:单个 ExpPlanDocument,不是事件流", () =>
     expect(doc.matrix).toHaveLength(2);
   });
 
+  it("locked 为 true 的行原样透传;省略的行不出现 locked 字段(JSON.stringify 丢弃 undefined 属性)", () => {
+    const text = renderJsonPlanDocument({
+      total: 2,
+      evals: 2,
+      configs: 1,
+      runs: 1,
+      matrix: [
+        { experimentId: "compare/codex", evalId: "memory/a", reused: false, locked: true },
+        { experimentId: "compare/codex", evalId: "memory/b", reused: false },
+      ],
+    });
+    const doc = JSON.parse(text);
+    expect(doc.matrix[0]).toMatchObject({ evalId: "memory/a", locked: true });
+    expect(doc.matrix[1]).not.toHaveProperty("locked");
+  });
+
   it("reused 是 matrix 逐行 reused 之和(命中数量,不是 attempt 数)", () => {
     const text = renderJsonPlanDocument({
       total: 3,

package/src/runner/feedback/json.ts

@@ -104,13 +104,24 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
           noteCheckpoint(event.at);
           if (!isFirstOccurrence(state, event.key)) return; // 去重后只追加一次(cli.md)
           const phase = typeof event.data?.phase === "string" ? (event.data.phase as LifecyclePhase) : undefined;
+          // `code` 是 cli.md `WarningEvent` 里那个稳定词法(`lock-taken-over` / `dispatch-halted`),
+          // **不是**去重 key:去重 key 常把折叠身份编进去(`lock-taken-over:<exp>|<eval>`),原样
+          // 透出会让消费方拿到一个每次运行都不同的 code、没法按值分支。折叠到哪一条实验/用例
+          // 由下面的 experimentId/evalId 两个具名字段回答。
+          const code = event.code ?? event.key;
+          // 身份两字段:attempt 级诊断从 identity 取;运行级诊断(实验闸 / eval 闸这类不属于
+          // 任何单条 attempt 的事实)不许伪造 identity,从 `data` 的同名字段取(见 ../types.ts
+          // "diagnostic" 变体的 identity 注释)。
+          const experimentId = event.identity?.experimentId ?? stringField(event.data?.experimentId);
+          const evalId = event.identity?.evalId ?? stringField(event.data?.evalId);
           writeEvent(io, {
             event: "warning",
-            code: event.key,
+            code,
             level: event.severity,
             message: event.message,
             ...(phase !== undefined ? { phase } : {}),
-            ...(event.identity?.experimentId !== undefined ? { experimentId: event.identity.experimentId } : {}),
+            ...(experimentId !== undefined ? { experimentId } : {}),
+            ...(evalId !== undefined ? { evalId } : {}),
           });
           return;
         }
@@ -148,6 +159,26 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
           return;
         }
 
+        case "lock-wait": {
+          noteCheckpoint(event.at);
+          writeEvent(io, {
+            event: "lock_wait",
+            experimentId: event.experimentId,
+            evalId: event.evalId,
+            status: event.status,
+            ...(event.holderPid !== undefined ? { holderPid: event.holderPid } : {}),
+            ...(event.holderHost !== undefined ? { holderHost: event.holderHost } : {}),
+            // 折叠 carried/dispatched 两个内部计数为一个公开 resolution:仍有 attempt 需要真实
+            // 派发(dispatched > 0)记 "dispatched",全部由携带满足才记 "carried"
+            // (见 ../types.ts 里 DurableFeedbackEvent "lock-wait" 变体的字段注释)。
+            ...(event.status === "resolved"
+              ? { resolution: (event.dispatched ?? 0) > 0 ? "dispatched" : "carried" }
+              : {}),
+            ...(event.waitedMs !== undefined ? { waitedMs: event.waitedMs } : {}),
+          });
+          return;
+        }
+
         case "interrupted": {
           noteCheckpoint(event.at);
           if (!isFirstOccurrence(state, "interrupted")) return;
@@ -208,8 +239,14 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
         total: state.total,
         reused: state.reused,
         running: state.running,
+        elsewhere: state.elsewhere,
         queued: state.queued,
-        completed: state.completed,
+        // 与 human 首行同一份状态、同一套划分:已了结的 attempt 按 verdict 分项,消费方不必
+        // 自己从合计数里猜成败(契约见 docs/feature/experiments/cli.md 的 ProgressEvent)。
+        passed: state.passed,
+        failed: state.failed,
+        errored: state.errored,
+        skipped: state.skipped,
       });
     },
 
@@ -225,6 +262,12 @@ function isFirstOccurrence(state: RunFeedbackState, key: string): boolean {
   return (state.diagnostics.find((d) => d.key === key)?.count ?? 0) <= 1;
 }
 
+/** `data` 里的一个字符串字段;非字符串(或缺失)一律当没有,不把数字/对象硬塞进只接受
+ *  字符串的事件字段里。 */
+function stringField(value: JsonValue | undefined): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
 // ───────────────────────── failure / error 事件(不设上限) ─────────────────────────
 
 function writeFailureOrError(io: FeedbackIO, event: DurableFeedbackEvent & { type: "failure" }): void {
@@ -346,6 +389,11 @@ export interface JsonPlanRow {
   evalId: string;
   /** 命中缓存指纹,本次不会派发新 attempt。 */
   reused: boolean;
+  /** 该用例正被另一条并行 Invocation 持锁运行,真实运行时将等待后携带或补跑(见
+   *  docs/feature/experiments/architecture.md「并发 Invocation:用例锁」)。`--dry` 只读锁
+   *  目录,不取锁、不等待;省略等于 `false`(JSON.stringify 丢弃 `undefined` 属性,
+   *  天然满足这条省略语义,不需要显式写 `locked: false`)。 */
+  locked?: boolean;
 }
 
 export interface JsonPlanInput {