niceeval 0.10.3-canary.6 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/bin/niceeval.js +9 -4
  2. package/dist/agents/types.d.ts +5 -4
  3. package/dist/context/turn-errors.d.ts +27 -23
  4. package/dist/i18n/en.d.ts +15 -0
  5. package/dist/i18n/en.js +16 -1
  6. package/dist/i18n/zh-CN.d.ts +16 -1
  7. package/dist/i18n/zh-CN.js +16 -1
  8. package/dist/o11y/derive.js +28 -24
  9. package/dist/o11y/types.d.ts +8 -5
  10. package/dist/report/components/attempt-detail/UsageTable.js +4 -7
  11. package/dist/report/components/attempt-detail/compute.d.ts +2 -2
  12. package/dist/report/components/attempt-detail/compute.js +2 -6
  13. package/dist/report/components/attempt-detail/faces.js +7 -7
  14. package/dist/report/components/attempt-detail/index.js +0 -3
  15. package/dist/report/components/entity-lists/EvalList.js +0 -0
  16. package/dist/report/components/metric-views/compute.js +1 -1
  17. package/dist/report/model/types.d.ts +3 -4
  18. package/dist/results/locator.js +0 -0
  19. package/dist/results/select.d.ts +6 -0
  20. package/dist/results/select.js +8 -0
  21. package/dist/runner/feedback/sink.d.ts +26 -1
  22. package/dist/runner/fingerprint.d.ts +23 -0
  23. package/dist/runner/types.d.ts +105 -7
  24. package/dist/sandbox/errors.d.ts +29 -0
  25. package/dist/sandbox/resolve.d.ts +9 -0
  26. package/dist/shared/failure-class.d.ts +91 -0
  27. package/dist/types.d.ts +1 -0
  28. package/dist/util.d.ts +3 -2
  29. package/dist/util.js +31 -5
  30. package/docs-site/zh/explanation/runner.mdx +35 -0
  31. package/docs-site/zh/reference/cli.mdx +10 -1
  32. package/docs-site/zh/reference/events.mdx +4 -4
  33. package/docs-site/zh/troubleshooting/debugging.mdx +11 -0
  34. package/docs-site/zh/tutorials/viewing-results.mdx +35 -3
  35. package/package.json +28 -24
  36. package/src/agents/ai-sdk.test.ts +26 -0
  37. package/src/agents/ai-sdk.ts +7 -4
  38. package/src/agents/index.ts +5 -3
  39. package/src/agents/langgraph.test.ts +30 -0
  40. package/src/agents/langgraph.ts +5 -2
  41. package/src/agents/openai-compat.test.ts +35 -0
  42. package/src/agents/openai-compat.ts +16 -4
  43. package/src/agents/sdk-streams.test.ts +66 -0
  44. package/src/agents/sdk-streams.ts +3 -1
  45. package/src/agents/types.ts +5 -4
  46. package/src/cli.ts +81 -16
  47. package/src/context/context.test.ts +34 -0
  48. package/src/context/context.ts +14 -5
  49. package/src/context/send-retry.test.ts +86 -0
  50. package/src/context/send-retry.ts +37 -12
  51. package/src/context/session.ts +24 -0
  52. package/src/context/turn-errors.test.ts +124 -17
  53. package/src/context/turn-errors.ts +60 -50
  54. package/src/define.ts +5 -0
  55. package/src/i18n/en.ts +22 -1
  56. package/src/i18n/zh-CN.ts +22 -1
  57. package/src/index.ts +8 -0
  58. package/src/o11y/cost.ts +4 -2
  59. package/src/o11y/derive.test.ts +40 -0
  60. package/src/o11y/derive.ts +28 -22
  61. package/src/o11y/otlp/sandbox-receiver.test.ts +201 -0
  62. package/src/o11y/otlp/sandbox-receiver.ts +73 -27
  63. package/src/o11y/parsers/bub.test.ts +30 -0
  64. package/src/o11y/parsers/bub.ts +5 -2
  65. package/src/o11y/parsers/codex.test.ts +19 -0
  66. package/src/o11y/parsers/codex.ts +5 -2
  67. package/src/o11y/types.ts +8 -5
  68. package/src/report/components/attempt-detail/UsageTable.tsx +4 -6
  69. package/src/report/components/attempt-detail/attempt-components.test.tsx +5 -8
  70. package/src/report/components/attempt-detail/compute.ts +2 -7
  71. package/src/report/components/attempt-detail/faces.ts +7 -7
  72. package/src/report/components/attempt-detail/index.tsx +0 -3
  73. package/src/report/components/entity-lists/EvalList.tsx +0 -0
  74. package/src/report/components/metric-views/compute.ts +1 -1
  75. package/src/report/model/types.ts +3 -4
  76. package/src/results/format.ts +9 -2
  77. package/src/results/index.ts +2 -0
  78. package/src/results/locator.ts +0 -0
  79. package/src/results/open.ts +132 -21
  80. package/src/results/select.ts +12 -0
  81. package/src/results/skipped-notice.ts +0 -0
  82. package/src/runner/attempt.test.ts +116 -0
  83. package/src/runner/attempt.ts +89 -8
  84. package/src/runner/discover.ts +21 -3
  85. package/src/runner/feedback/coordinator.ts +24 -2
  86. package/src/runner/feedback/eval-conclusions.ts +6 -3
  87. package/src/runner/feedback/human.test.ts +300 -6
  88. package/src/runner/feedback/human.ts +132 -30
  89. package/src/runner/feedback/json.test.ts +127 -2
  90. package/src/runner/feedback/json.ts +51 -3
  91. package/src/runner/feedback/reducer.test.ts +328 -29
  92. package/src/runner/feedback/reducer.ts +84 -9
  93. package/src/runner/feedback/sink.ts +39 -1
  94. package/src/runner/fingerprint.ts +49 -19
  95. package/src/runner/gate-lease.test.ts +510 -0
  96. package/src/runner/gate-lease.ts +350 -0
  97. package/src/runner/lock.test.ts +454 -0
  98. package/src/runner/lock.ts +288 -0
  99. package/src/runner/report.test.ts +1 -0
  100. package/src/runner/run.test.ts +2044 -9
  101. package/src/runner/run.ts +825 -61
  102. package/src/runner/teardown-registry.ts +20 -78
  103. package/src/runner/types.ts +103 -7
  104. package/src/sandbox/errors.test.ts +72 -0
  105. package/src/sandbox/errors.ts +83 -0
  106. package/src/sandbox/keep-registry.ts +22 -46
  107. package/src/sandbox/resolve.test.ts +100 -0
  108. package/src/sandbox/resolve.ts +84 -27
  109. package/src/shared/entry-file-store.test.ts +149 -0
  110. package/src/shared/entry-file-store.ts +117 -0
  111. package/src/shared/failure-class.test.ts +137 -0
  112. package/src/shared/failure-class.ts +175 -0
  113. package/src/show/index.ts +5 -6
  114. package/src/show/render.test.ts +116 -15
  115. package/src/show/render.ts +124 -35
  116. package/src/types.ts +9 -0
  117. package/src/util.ts +31 -4
  118. package/src/view/app/App.tsx +5 -1
  119. package/src/view/client-dist/app.js +1 -1
  120. package/src/view/data.ts +5 -9
  121. package/src/view/shared/types.ts +7 -1
  122. package/src/view/view-report.test.ts +54 -0
@@ -20,7 +20,7 @@ import { t } from "../../i18n/index.ts";
20
20
  import { verdictSymbol } from "../reporters/shared.ts";
21
21
  import { formatCost } from "../../shared/format.ts";
22
22
  import { assertionSummaryLines } from "../../scoring/display.ts";
23
- import { encodeAttemptKey } from "../types.ts";
23
+ import { encodeAttemptKey, HALT_DIAGNOSTIC_CODE } from "../types.ts";
24
24
  import {
25
25
  panelCapabilityOf as panelCapability,
26
26
  panelContentWidth,
@@ -32,6 +32,7 @@ import { stringWidth } from "../../report/model/text-layout.ts";
32
32
  import type {
33
33
  ActiveAttempt,
34
34
  ActiveExperimentHook,
35
+ ActiveLockWait,
35
36
  ActivePrecheck,
36
37
  AttemptKey,
37
38
  ExperimentHookName,
@@ -140,6 +141,40 @@ export function renderDurableLines(
140
141
  const duration = event.durationMs !== undefined ? ` (${formatElapsed(event.durationMs)})` : "";
141
142
  return [`${t("feedback.human.precheckJudgeDone")}${duration}`];
142
143
  }
144
+ case "lock-wait": {
145
+ // 只服务非 TTY 退化流(TTY dashboard 的 appendDurable 对这个事件直接返回,运行级行由
146
+ // state.lockWaits 驱动,不进 scrollback,见 cli.md「等待并发 run 的显示」)。按实验聚合
147
+ // ——同一实验可能有多个用例先后撞锁,只在这个「有等待用例」窗口第一次打开(这是当前
148
+ // 唯一一条等待中的用例)与最后一次关闭(等待全部解决)各打印一行,中途加入/解决的用例
149
+ // 不逐条刷屏,与诊断按 dedupeKey 折叠同一种克制(state 已经是这条事件 reduce 之后的
150
+ // 快照,size 天然反映"这条事件之后"的计数)。
151
+ const agg = state.lockWaits.get(event.experimentId);
152
+ if (event.status === "started") {
153
+ if (!agg || agg.waiting.size !== 1) return []; // 不是这个窗口的第一条,静默
154
+ const holder = agg.waiting.get(event.evalId);
155
+ return [
156
+ t("feedback.human.lockWaitStarted", {
157
+ experimentId: event.experimentId,
158
+ count: agg.waiting.size,
159
+ pid: holder?.holderPid ?? "?",
160
+ }),
161
+ ];
162
+ }
163
+ if (!agg || agg.waiting.size !== 0) return []; // 窗口还没关闭,静默
164
+ const parts: string[] = [];
165
+ if (agg.resolvedCarried > 0) parts.push(t("feedback.human.lockWaitCarried", { count: agg.resolvedCarried }));
166
+ if (agg.resolvedDispatched > 0) {
167
+ parts.push(t("feedback.human.lockWaitDispatched", { count: agg.resolvedDispatched }));
168
+ }
169
+ const summary = parts.length > 0 ? parts.join(" · ") : t("feedback.human.lockWaitCarried", { count: 0 });
170
+ return [
171
+ t("feedback.human.lockWaitResolved", {
172
+ experimentId: event.experimentId,
173
+ summary,
174
+ elapsed: formatElapsed(event.waitedMs ?? 0),
175
+ }),
176
+ ];
177
+ }
143
178
  case "summary":
144
179
  return buildSummaryLines(event, state, panel);
145
180
  case "saved":
@@ -208,8 +243,27 @@ function buildDiagnosticLines(event: DurableFeedbackEvent & { type: "diagnostic"
208
243
  // count 从 state.diagnostics 读(reducer 已经按 key 去重累加),不在这里自己维护第二份计数。
209
244
  const count = state.diagnostics.find((d) => d.key === event.key)?.count ?? 1;
210
245
  const sym = event.severity === "error" ? "✗" : "!";
246
+ if (event.code === HALT_DIAGNOSTIC_CODE) {
247
+ // 止损闸落闸:一行 error 级通知,文案已经是完整的一句话(`experiment halted
248
+ // (dispatch-halted): <message>` / `eval halted: <message>`,见 docs/feature/
249
+ // error-classification/architecture.md「观察面」),不再加标题行、也不加 ×N 后缀——
250
+ // emitter 对每条未派发 attempt 都刷一次这条诊断以更新 data.unstarted,逐次打印就是
251
+ // 同一页文档明令禁止的「被中止的等待集 attempt 逐条刷屏」;未派发的数量由完成状态的
252
+ // `unstarted` 回答,不在这行重复。因此只在第一次出现时落一行(与 json profile 的
253
+ // isFirstOccurrence 同一条去重纪律)。
254
+ if (count > 1) return [];
255
+ return [`${sym} ${event.message}`];
256
+ }
257
+ // 标题用稳定词法(`code`),不是把折叠身份一起编进去的去重 key —— 人读的一行要能一眼认出
258
+ // 「这是哪一类诊断」,`compare/codex|memory/x` 那串身份属于 message 与机器面的具名字段。
211
259
  const suffix = count > 1 ? ` (${count} attempts)` : "";
212
- return [`${sym} ${event.key}${suffix}`, ` ${event.message}`];
260
+ // 阶段标签走与失败行(`buildFailureLine`)同一个 `phaseLabel()` 投影:「在哪一步降级的」是
261
+ // 读者的第一个问题,message 里未必答得上。attempt 级诊断的 phase 由运行器写进 `data`
262
+ // (见 attempt.ts 的 recordDiagnostic);运行级诊断(止损闸、锁接管、budget)不属于任何
263
+ // 单条 attempt,天然没有 phase,标题退化成只有 code 一段。
264
+ const phase = typeof event.data?.phase === "string" ? (event.data.phase as LifecyclePhase) : undefined;
265
+ const heading = phase !== undefined ? `${phaseLabel(phase)} · ${event.code ?? event.key}` : (event.code ?? event.key);
266
+ return [`${sym} ${heading}${suffix}`, ` ${event.message}`];
213
267
  }
214
268
 
215
269
  /** 结束结论(`FAILED`/`PASSED`/…)+ `FAILURES`(有失败才出现)+ `KEPT SANDBOXES`(有留存才
@@ -418,14 +472,37 @@ function experimentHookLabel(hook: ExperimentHookName): string {
418
472
  return hook === "setup" ? t("feedback.phase.experimentSetup") : t("feedback.phase.experimentTeardown");
419
473
  }
420
474
 
475
+ /** 首行守恒计数的文案(见 cli.md「运行中的 live 面板」)。四项结局恒显示、零值不省略——
476
+ * 「0 errored」是一句有价值的肯定;`elsewhere` 只在非零时出现,没有并发 run 的场景少一项。
477
+ * 行长不是压缩它的理由:首行跟随终端全宽,九项写满仍是一行。两个调用点(live dashboard
478
+ * 首行、非 TTY heartbeat)共用这一份,不各自维护一份键选择逻辑。 */
479
+ function countsText(state: RunFeedbackState): string {
480
+ const outcomes = {
481
+ passed: state.passed,
482
+ failed: state.failed,
483
+ errored: state.errored,
484
+ skipped: state.skipped,
485
+ };
486
+ return state.elsewhere > 0
487
+ ? t("feedback.human.countsWithElsewhere", {
488
+ total: state.total,
489
+ reused: state.reused,
490
+ running: state.running,
491
+ elsewhere: state.elsewhere,
492
+ queued: state.queued,
493
+ ...outcomes,
494
+ })
495
+ : t("feedback.human.counts", {
496
+ total: state.total,
497
+ reused: state.reused,
498
+ running: state.running,
499
+ queued: state.queued,
500
+ ...outcomes,
501
+ });
502
+ }
503
+
421
504
  function formatCounts(state: RunFeedbackState): string {
422
- const counts = t("feedback.human.counts", {
423
- total: state.total,
424
- reused: state.reused,
425
- running: state.running,
426
- queued: state.queued,
427
- completed: state.completed,
428
- });
505
+ const counts = countsText(state);
429
506
  if (state.estimatedCostUSD === undefined || state.estimatedCostUSD <= 0) return counts;
430
507
  return `${counts} ${formatCost(state.estimatedCostUSD)}`;
431
508
  }
@@ -479,30 +556,23 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
479
556
  // renderPanel 内部按另一个宽度钳制,行尾会被框吃掉——这正是 memory/
480
557
  // live-dashboard-active-row-width-clamp-mismatch.md 的根因类别。
481
558
  const contentWidth = panelContentWidth(capability.width, capability.mode, false);
482
- const rows: PanelRow[] = [
483
- {
484
- kind: "line",
485
- text: t("feedback.human.counts", {
486
- total: state.total,
487
- reused: state.reused,
488
- running: state.running,
489
- queued: state.queued,
490
- completed: state.completed,
491
- }),
492
- },
493
- ];
494
- // 运行级行(judge 预检 + 实验钩子)排在 attempt 行前面(见 cli.md「judge 预检的显示」/
495
- // 「实验级钩子的显示」):它们解释了为什么后面的 attempt 还停在 queued。预检又排在实验
496
- // 钩子前面——它发生在最前(任何 attempt 派发之前)。Map 按插入序迭代,天然满足稳定 slot。
559
+ const rows: PanelRow[] = [{ kind: "line", text: countsText(state) }];
560
+ // 运行级行(judge 预检 + 实验钩子 + 用例锁等待)排在 attempt 行前面(见 cli.md「judge 预检
561
+ // 的显示」/「实验级钩子的显示」/「等待并发 run 的显示」):它们解释了为什么后面的 attempt
562
+ // 还停在 queued。预检排最前(发生在任何 attempt 派发之前),其次实验钩子,再是锁等待
563
+ // (排在实验钩子行之后、attempt 行之前)。Map 按插入序迭代,天然满足稳定 slot。
497
564
  const precheck = state.activePrecheck;
498
565
  const hookRows = [...state.experimentHooks.values()];
499
- if (activeOrder.length > 0 || hookRows.length > 0 || precheck) {
566
+ // 只有仍在等待(waiting 非空)的实验才占运行级行;窗口已关闭(全部 resolved)的条目只是
567
+ // 给非 TTY 聚合收尾行留的历史计数,TTY 不展示。
568
+ const lockWaitRows = [...state.lockWaits.values()].filter((w) => w.waiting.size > 0);
569
+ if (activeOrder.length > 0 || hookRows.length > 0 || lockWaitRows.length > 0 || precheck) {
500
570
  rows.push({ kind: "divider", title: t("feedback.human.active") });
501
571
  // 固定开销:上边框 + counts 行 + ACTIVE 横隔 + 下边框(boxed);plain 时同样按 4 行估算,
502
572
  // 差一两行不影响「窄/矮终端先减 active slots」这条大方向。
503
573
  const rowBudget = Math.max(0, io.stderr.rows - 4 - DASHBOARD_ROW_RESERVE);
504
574
  const precheckCount = precheck ? 1 : 0;
505
- const total = precheckCount + hookRows.length + activeOrder.length;
575
+ const total = precheckCount + hookRows.length + lockWaitRows.length + activeOrder.length;
506
576
  // 窄/矮终端先减 active slots(减少行数),而不是先压缩单行内容 ——
507
577
  // 单行内容的截断在 formatActiveRow 里按 contentWidth 单独处理。
508
578
  const showCount = total <= rowBudget ? total : Math.max(0, rowBudget - 1);
@@ -512,7 +582,11 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
512
582
  // 的行又观测到更长的值再推宽,导致同一帧内本该对齐的列错位。
513
583
  const shownPrecheck = precheck && showCount > 0 ? precheck : undefined;
514
584
  const shownHooks = hookRows.slice(0, Math.max(0, showCount - (shownPrecheck ? 1 : 0)));
515
- const shownRunLevel = (shownPrecheck ? 1 : 0) + shownHooks.length;
585
+ const shownLockWaits = lockWaitRows.slice(
586
+ 0,
587
+ Math.max(0, showCount - (shownPrecheck ? 1 : 0) - shownHooks.length),
588
+ );
589
+ const shownRunLevel = (shownPrecheck ? 1 : 0) + shownHooks.length + shownLockWaits.length;
516
590
  const shownActive: ActiveAttempt[] = [];
517
591
  for (const key of activeOrder) {
518
592
  if (shownRunLevel + shownActive.length >= showCount) break;
@@ -533,6 +607,9 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
533
607
  for (const hookRow of shownHooks) {
534
608
  activeLines.push(formatExperimentHookRow(hookRow, io, contentWidth, evalWidth, whoWidth));
535
609
  }
610
+ for (const lockWaitRow of shownLockWaits) {
611
+ activeLines.push(formatLockWaitRow(lockWaitRow, io, contentWidth));
612
+ }
536
613
  for (const active of shownActive) {
537
614
  activeLines.push(formatActiveRow(active, io, contentWidth, evalWidth, whoWidth));
538
615
  }
@@ -584,8 +661,9 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
584
661
  // 更新,coordinator 紧接着的 redrawDynamic 会画出来);成功钩子不写 scrollback 永久行
585
662
  // (见 cli.md「实验级钩子的显示」)。非 TTY 退化流才逐行追加(见 renderDurableLines)。
586
663
  // judge 预检同理:TTY 下只驱动 state.activePrecheck 的运行级 active 行(coordinator 紧接着的
587
- // redrawDynamic 会画出来),不写 scrollback 永久行(见 cli.md「judge 预检的显示」)。
588
- if (event.type === "experiment-hook" || event.type === "precheck") return;
664
+ // redrawDynamic 会画出来),不写 scrollback 永久行(见 cli.md「judge 预检的显示」)。用例锁
665
+ // 等待同理:TTY 下由 state.lockWaits 驱动运行级 active 行(见 cli.md「等待并发 run 的显示」)。
666
+ if (event.type === "experiment-hook" || event.type === "precheck" || event.type === "lock-wait") return;
589
667
  writeDurable(io, event, state, false);
590
668
  },
591
669
  activity(text) {
@@ -682,6 +760,26 @@ function formatExperimentHookRow(
682
760
  return prefix + (hook.detail ?? "").slice(0, budget);
683
761
  }
684
762
 
763
+ /** 用例锁等待的运行级行:`● waiting on another run · <exp> <elapsed> <n> evals · pid <pid>`
764
+ * (cli.md「等待并发 run 的显示」)。elapsed 从最早一条等待的 startedAt 算(存活性证明,
765
+ * 与其它运行级行同一约定);pid 取最早一条等待对应的持有方——一个实验可能同时撞上多把不同
766
+ * 持有方的锁,这里选一个稳定的代表值展示,不逐条列出(与 earlyExit 代表 attempt 的选法同一
767
+ * 种「挑一个确定性代表」思路)。不吃 evalWidth/whoWidth 身份列宽约束——与 `formatPrecheckRow`
768
+ * 同理:选中用例全在等锁时,本实验没有派发中的 attempt,那两个宽度还压在初始值 0,会把
769
+ * label 截成 "w…"(与 memory/live-dashboard-active-row-width-clamp-mismatch.md 同一根因类别,
770
+ * 只是发生在锁等待场景);label 直接用整行宽度。 */
771
+ function formatLockWaitRow(wait: ActiveLockWait, io: FeedbackIO, columns: number): string {
772
+ const entries = [...wait.waiting.values()].sort((a, b) => a.startedAt - b.startedAt);
773
+ const earliest = entries[0]!;
774
+ const elapsed = formatElapsed(io.clock.now() - earliest.startedAt).padStart(6);
775
+ const sym = "● ";
776
+ const label = `${t("feedback.human.waitingOnAnotherRun")} · ${wait.experimentId}`;
777
+ const prefix = `${sym}${label} ${elapsed} `;
778
+ const budget = Math.max(0, columns - prefix.length);
779
+ const detail = t("feedback.human.lockWaitDetail", { count: entries.length, pid: earliest.holderPid ?? "?" });
780
+ return padTrunc(prefix + detail.slice(0, budget), columns);
781
+ }
782
+
685
783
  // ───────────────────────── 非 TTY:human 文案的纯追加流 ─────────────────────────
686
784
  //
687
785
  // 单一 stdout 有序流(见 memory/exp-output-two-forms-ruling.md 的补充裁决):从 start 到结束
@@ -730,6 +828,9 @@ function createPlainRenderer(io: FeedbackIO): FeedbackRenderer {
730
828
  export interface HumanDryPlanRow {
731
829
  experimentId: string;
732
830
  evalId: string;
831
+ /** 该用例正被另一条并行 Invocation 持锁运行(见 docs/feature/experiments/architecture.md
832
+ * 「并发 Invocation:用例锁」);计划行尾如实标注,`--dry` 本身不取锁、不等待。 */
833
+ locked?: boolean;
733
834
  }
734
835
 
735
836
  export interface HumanDryPlanInput {
@@ -772,7 +873,8 @@ export function renderHumanDryPlan(input: HumanDryPlanInput): string {
772
873
  }
773
874
  const idWidth = Math.max(0, ...input.rows.map((row) => stringWidth(row.experimentId)));
774
875
  for (const row of input.rows) {
775
- lines.push(`${row.experimentId}${" ".repeat(idWidth - stringWidth(row.experimentId) + 2)}${row.evalId}`);
876
+ const base = `${row.experimentId}${" ".repeat(idWidth - stringWidth(row.experimentId) + 2)}${row.evalId}`;
877
+ lines.push(row.locked ? `${base} ${t("feedback.human.lockedRowSuffix")}` : base);
776
878
  }
777
879
  return `${lines.join("\n")}\n`;
778
880
  }
@@ -8,8 +8,10 @@
8
8
  // docs/engineering/testing/e2e/cli.md「反馈输出格式」在真实进程输出上验收。
9
9
 
10
10
  import { describe, expect, it } from "vitest";
11
- import { computeExitCode, renderJsonPlanDocument } from "./json.ts";
12
- import type { InvocationCompletion, InvocationSummary } from "../types.ts";
11
+ import { computeExitCode, createJsonRenderer, renderJsonPlanDocument } from "./json.ts";
12
+ import { createInitialRunFeedbackState, reduceRunFeedback } from "./reducer.ts";
13
+ import { createFakeFeedbackIO } from "./testing.ts";
14
+ import { HALT_DIAGNOSTIC_CODE, type DurableFeedbackEvent, type InvocationCompletion, type InvocationSummary } from "../types.ts";
13
15
 
14
16
  function summary(overrides: Partial<InvocationSummary> = {}): InvocationSummary {
15
17
  return {
@@ -77,6 +79,113 @@ describe("computeExitCode:CompletionStatus 驱动退出码,不只看 failed/erro
77
79
  });
78
80
  });
79
81
 
82
+ /** 依次喂进 reducer 再交给 json renderer(与生产的 coordinator 同序:先 reduce 后 render),
83
+ * 返回逐行解析出的事件对象。 */
84
+ function emitDurable(events: readonly DurableFeedbackEvent[]): Record<string, unknown>[] {
85
+ const { io, stdout } = createFakeFeedbackIO();
86
+ const renderer = createJsonRenderer({ io });
87
+ let state = createInitialRunFeedbackState();
88
+ for (const event of events) {
89
+ state = reduceRunFeedback(state, event);
90
+ renderer.appendDurable(event, state);
91
+ }
92
+ return stdout.writes
93
+ .join("")
94
+ .split("\n")
95
+ .filter(Boolean)
96
+ .map((line) => JSON.parse(line) as Record<string, unknown>);
97
+ }
98
+
99
+ describe("warning 事件:code 是稳定词法,折叠身份走具名字段", () => {
100
+ it("code 用 DiagnosticInput.code 的干净字面量,不透传编了身份的去重 key", () => {
101
+ const [warning] = emitDurable([
102
+ {
103
+ type: "diagnostic",
104
+ at: 1,
105
+ key: "lock-taken-over:compare/codex|memory/retention",
106
+ code: "lock-taken-over",
107
+ severity: "warning",
108
+ message: "took over a stale lock",
109
+ data: { experimentId: "compare/codex", evalId: "memory/retention" },
110
+ },
111
+ ]);
112
+ expect(warning).toMatchObject({
113
+ event: "warning",
114
+ code: "lock-taken-over",
115
+ level: "warning",
116
+ experimentId: "compare/codex",
117
+ evalId: "memory/retention",
118
+ });
119
+ });
120
+
121
+ it("没给 code 的诊断回落到 key(折叠身份本就不进 key 的那些天生是干净字面量)", () => {
122
+ const [warning] = emitDurable([
123
+ { type: "diagnostic", at: 1, key: "memory-warmup-degraded", severity: "warning", message: "cold index" },
124
+ ]);
125
+ expect(warning).toMatchObject({ code: "memory-warmup-degraded" });
126
+ expect(warning).not.toHaveProperty("experimentId");
127
+ expect(warning).not.toHaveProperty("evalId");
128
+ });
129
+
130
+ it("eval 闸的 dispatch-halted:code 干净、evalId 与 phase 都在事件流里透得出", () => {
131
+ const [warning] = emitDurable([
132
+ {
133
+ type: "diagnostic",
134
+ at: 1,
135
+ key: "dispatch-halted:eval:compare/codex|memory/retention",
136
+ code: HALT_DIAGNOSTIC_CODE,
137
+ severity: "error",
138
+ message: "eval halted: fixture db is empty; run scripts/seed.ts",
139
+ data: {
140
+ experimentId: "compare/codex",
141
+ scope: "eval",
142
+ evalId: "memory/retention",
143
+ phase: "eval.run",
144
+ unstarted: 0,
145
+ },
146
+ },
147
+ ]);
148
+ expect(warning).toMatchObject({
149
+ event: "warning",
150
+ code: "dispatch-halted",
151
+ level: "error",
152
+ phase: "eval.run",
153
+ experimentId: "compare/codex",
154
+ evalId: "memory/retention",
155
+ });
156
+ });
157
+
158
+ it("身份从 data 取(闸不是 attempt 级、不伪造 identity);有 identity 时 identity 优先", () => {
159
+ const [fromIdentity] = emitDurable([
160
+ {
161
+ type: "diagnostic",
162
+ at: 1,
163
+ key: "fail-fast:x",
164
+ code: "fail-fast",
165
+ severity: "warning",
166
+ message: "deterministic failure",
167
+ identity: { experimentId: "compare/codex", evalId: "memory/a", attempt: 0 },
168
+ data: { experimentId: "other/exp", evalId: "memory/z" },
169
+ },
170
+ ]);
171
+ expect(fromIdentity).toMatchObject({ experimentId: "compare/codex", evalId: "memory/a" });
172
+ });
173
+
174
+ it("同一 dedupeKey 只追加一次:emitter 为刷新 data.unstarted 反复报同一条闸,事件流不重复", () => {
175
+ const halted = (at: number, unstarted: number): DurableFeedbackEvent => ({
176
+ type: "diagnostic",
177
+ at,
178
+ key: "dispatch-halted:experiment:compare/codex",
179
+ code: HALT_DIAGNOSTIC_CODE,
180
+ severity: "error",
181
+ message: "experiment halted (dispatch-halted): shared service is down",
182
+ data: { experimentId: "compare/codex", scope: "experiment", phase: "eval.run", unstarted },
183
+ });
184
+ const events = emitDurable([halted(1, 0), halted(2, 1), halted(3, 2)]);
185
+ expect(events.filter((e) => e.event === "warning")).toHaveLength(1);
186
+ });
187
+ });
188
+
80
189
  describe("renderJsonPlanDocument:单个 ExpPlanDocument,不是事件流", () => {
81
190
  it("输出恰好一行 JSON,可解析为单个对象而不是逐行事件序列", () => {
82
191
  const text = renderJsonPlanDocument({
@@ -101,6 +210,22 @@ describe("renderJsonPlanDocument:单个 ExpPlanDocument,不是事件流", () =>
101
210
  expect(doc.matrix).toHaveLength(2);
102
211
  });
103
212
 
213
+ it("locked 为 true 的行原样透传;省略的行不出现 locked 字段(JSON.stringify 丢弃 undefined 属性)", () => {
214
+ const text = renderJsonPlanDocument({
215
+ total: 2,
216
+ evals: 2,
217
+ configs: 1,
218
+ runs: 1,
219
+ matrix: [
220
+ { experimentId: "compare/codex", evalId: "memory/a", reused: false, locked: true },
221
+ { experimentId: "compare/codex", evalId: "memory/b", reused: false },
222
+ ],
223
+ });
224
+ const doc = JSON.parse(text);
225
+ expect(doc.matrix[0]).toMatchObject({ evalId: "memory/a", locked: true });
226
+ expect(doc.matrix[1]).not.toHaveProperty("locked");
227
+ });
228
+
104
229
  it("reused 是 matrix 逐行 reused 之和(命中数量,不是 attempt 数)", () => {
105
230
  const text = renderJsonPlanDocument({
106
231
  total: 3,
@@ -104,13 +104,24 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
104
104
  noteCheckpoint(event.at);
105
105
  if (!isFirstOccurrence(state, event.key)) return; // 去重后只追加一次(cli.md)
106
106
  const phase = typeof event.data?.phase === "string" ? (event.data.phase as LifecyclePhase) : undefined;
107
+ // `code` 是 cli.md `WarningEvent` 里那个稳定词法(`lock-taken-over` / `dispatch-halted`),
108
+ // **不是**去重 key:去重 key 常把折叠身份编进去(`lock-taken-over:<exp>|<eval>`),原样
109
+ // 透出会让消费方拿到一个每次运行都不同的 code、没法按值分支。折叠到哪一条实验/用例
110
+ // 由下面的 experimentId/evalId 两个具名字段回答。
111
+ const code = event.code ?? event.key;
112
+ // 身份两字段:attempt 级诊断从 identity 取;运行级诊断(实验闸 / eval 闸这类不属于
113
+ // 任何单条 attempt 的事实)不许伪造 identity,从 `data` 的同名字段取(见 ../types.ts
114
+ // "diagnostic" 变体的 identity 注释)。
115
+ const experimentId = event.identity?.experimentId ?? stringField(event.data?.experimentId);
116
+ const evalId = event.identity?.evalId ?? stringField(event.data?.evalId);
107
117
  writeEvent(io, {
108
118
  event: "warning",
109
- code: event.key,
119
+ code,
110
120
  level: event.severity,
111
121
  message: event.message,
112
122
  ...(phase !== undefined ? { phase } : {}),
113
- ...(event.identity?.experimentId !== undefined ? { experimentId: event.identity.experimentId } : {}),
123
+ ...(experimentId !== undefined ? { experimentId } : {}),
124
+ ...(evalId !== undefined ? { evalId } : {}),
114
125
  });
115
126
  return;
116
127
  }
@@ -148,6 +159,26 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
148
159
  return;
149
160
  }
150
161
 
162
+ case "lock-wait": {
163
+ noteCheckpoint(event.at);
164
+ writeEvent(io, {
165
+ event: "lock_wait",
166
+ experimentId: event.experimentId,
167
+ evalId: event.evalId,
168
+ status: event.status,
169
+ ...(event.holderPid !== undefined ? { holderPid: event.holderPid } : {}),
170
+ ...(event.holderHost !== undefined ? { holderHost: event.holderHost } : {}),
171
+ // 折叠 carried/dispatched 两个内部计数为一个公开 resolution:仍有 attempt 需要真实
172
+ // 派发(dispatched > 0)记 "dispatched",全部由携带满足才记 "carried"
173
+ // (见 ../types.ts 里 DurableFeedbackEvent "lock-wait" 变体的字段注释)。
174
+ ...(event.status === "resolved"
175
+ ? { resolution: (event.dispatched ?? 0) > 0 ? "dispatched" : "carried" }
176
+ : {}),
177
+ ...(event.waitedMs !== undefined ? { waitedMs: event.waitedMs } : {}),
178
+ });
179
+ return;
180
+ }
181
+
151
182
  case "interrupted": {
152
183
  noteCheckpoint(event.at);
153
184
  if (!isFirstOccurrence(state, "interrupted")) return;
@@ -208,8 +239,14 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
208
239
  total: state.total,
209
240
  reused: state.reused,
210
241
  running: state.running,
242
+ elsewhere: state.elsewhere,
211
243
  queued: state.queued,
212
- completed: state.completed,
244
+ // 与 human 首行同一份状态、同一套划分:已了结的 attempt 按 verdict 分项,消费方不必
245
+ // 自己从合计数里猜成败(契约见 docs/feature/experiments/cli.md 的 ProgressEvent)。
246
+ passed: state.passed,
247
+ failed: state.failed,
248
+ errored: state.errored,
249
+ skipped: state.skipped,
213
250
  });
214
251
  },
215
252
 
@@ -225,6 +262,12 @@ function isFirstOccurrence(state: RunFeedbackState, key: string): boolean {
225
262
  return (state.diagnostics.find((d) => d.key === key)?.count ?? 0) <= 1;
226
263
  }
227
264
 
265
+ /** `data` 里的一个字符串字段;非字符串(或缺失)一律当没有,不把数字/对象硬塞进只接受
266
+ * 字符串的事件字段里。 */
267
+ function stringField(value: JsonValue | undefined): string | undefined {
268
+ return typeof value === "string" ? value : undefined;
269
+ }
270
+
228
271
  // ───────────────────────── failure / error 事件(不设上限) ─────────────────────────
229
272
 
230
273
  function writeFailureOrError(io: FeedbackIO, event: DurableFeedbackEvent & { type: "failure" }): void {
@@ -346,6 +389,11 @@ export interface JsonPlanRow {
346
389
  evalId: string;
347
390
  /** 命中缓存指纹,本次不会派发新 attempt。 */
348
391
  reused: boolean;
392
+ /** 该用例正被另一条并行 Invocation 持锁运行,真实运行时将等待后携带或补跑(见
393
+ * docs/feature/experiments/architecture.md「并发 Invocation:用例锁」)。`--dry` 只读锁
394
+ * 目录,不取锁、不等待;省略等于 `false`(JSON.stringify 丢弃 `undefined` 属性,
395
+ * 天然满足这条省略语义,不需要显式写 `locked: false`)。 */
396
+ locked?: boolean;
349
397
  }
350
398
 
351
399
  export interface JsonPlanInput {