niceeval 0.10.3-canary.6 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/niceeval.js +9 -4
- package/dist/agents/types.d.ts +5 -4
- package/dist/context/turn-errors.d.ts +27 -23
- package/dist/i18n/en.d.ts +15 -0
- package/dist/i18n/en.js +16 -1
- package/dist/i18n/zh-CN.d.ts +16 -1
- package/dist/i18n/zh-CN.js +16 -1
- package/dist/o11y/derive.js +28 -24
- package/dist/o11y/types.d.ts +8 -5
- package/dist/report/components/attempt-detail/UsageTable.js +4 -7
- package/dist/report/components/attempt-detail/compute.d.ts +2 -2
- package/dist/report/components/attempt-detail/compute.js +2 -6
- package/dist/report/components/attempt-detail/faces.js +7 -7
- package/dist/report/components/attempt-detail/index.js +0 -3
- package/dist/report/components/entity-lists/EvalList.js +0 -0
- package/dist/report/components/metric-views/compute.js +1 -1
- package/dist/report/model/types.d.ts +3 -4
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +6 -0
- package/dist/results/select.js +8 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +105 -7
- package/dist/sandbox/errors.d.ts +29 -0
- package/dist/sandbox/resolve.d.ts +9 -0
- package/dist/shared/failure-class.d.ts +91 -0
- package/dist/types.d.ts +1 -0
- package/dist/util.d.ts +3 -2
- package/dist/util.js +31 -5
- package/docs-site/zh/explanation/runner.mdx +35 -0
- package/docs-site/zh/reference/cli.mdx +10 -1
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/troubleshooting/debugging.mdx +11 -0
- package/docs-site/zh/tutorials/viewing-results.mdx +35 -3
- package/package.json +28 -24
- package/src/agents/ai-sdk.test.ts +26 -0
- package/src/agents/ai-sdk.ts +7 -4
- package/src/agents/index.ts +5 -3
- package/src/agents/langgraph.test.ts +30 -0
- package/src/agents/langgraph.ts +5 -2
- package/src/agents/openai-compat.test.ts +35 -0
- package/src/agents/openai-compat.ts +16 -4
- package/src/agents/sdk-streams.test.ts +66 -0
- package/src/agents/sdk-streams.ts +3 -1
- package/src/agents/types.ts +5 -4
- package/src/cli.ts +81 -16
- package/src/context/context.test.ts +34 -0
- package/src/context/context.ts +14 -5
- package/src/context/send-retry.test.ts +86 -0
- package/src/context/send-retry.ts +37 -12
- package/src/context/session.ts +24 -0
- package/src/context/turn-errors.test.ts +124 -17
- package/src/context/turn-errors.ts +60 -50
- package/src/define.ts +5 -0
- package/src/i18n/en.ts +22 -1
- package/src/i18n/zh-CN.ts +22 -1
- package/src/index.ts +8 -0
- package/src/o11y/cost.ts +4 -2
- package/src/o11y/derive.test.ts +40 -0
- package/src/o11y/derive.ts +28 -22
- package/src/o11y/otlp/sandbox-receiver.test.ts +201 -0
- package/src/o11y/otlp/sandbox-receiver.ts +73 -27
- package/src/o11y/parsers/bub.test.ts +30 -0
- package/src/o11y/parsers/bub.ts +5 -2
- package/src/o11y/parsers/codex.test.ts +19 -0
- package/src/o11y/parsers/codex.ts +5 -2
- package/src/o11y/types.ts +8 -5
- package/src/report/components/attempt-detail/UsageTable.tsx +4 -6
- package/src/report/components/attempt-detail/attempt-components.test.tsx +5 -8
- package/src/report/components/attempt-detail/compute.ts +2 -7
- package/src/report/components/attempt-detail/faces.ts +7 -7
- package/src/report/components/attempt-detail/index.tsx +0 -3
- package/src/report/components/entity-lists/EvalList.tsx +0 -0
- package/src/report/components/metric-views/compute.ts +1 -1
- package/src/report/model/types.ts +3 -4
- package/src/results/format.ts +9 -2
- package/src/results/index.ts +2 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +132 -21
- package/src/results/select.ts +12 -0
- package/src/results/skipped-notice.ts +0 -0
- package/src/runner/attempt.test.ts +116 -0
- package/src/runner/attempt.ts +89 -8
- package/src/runner/discover.ts +21 -3
- package/src/runner/feedback/coordinator.ts +24 -2
- package/src/runner/feedback/eval-conclusions.ts +6 -3
- package/src/runner/feedback/human.test.ts +300 -6
- package/src/runner/feedback/human.ts +132 -30
- package/src/runner/feedback/json.test.ts +127 -2
- package/src/runner/feedback/json.ts +51 -3
- package/src/runner/feedback/reducer.test.ts +328 -29
- package/src/runner/feedback/reducer.ts +84 -9
- package/src/runner/feedback/sink.ts +39 -1
- package/src/runner/fingerprint.ts +49 -19
- package/src/runner/gate-lease.test.ts +510 -0
- package/src/runner/gate-lease.ts +350 -0
- package/src/runner/lock.test.ts +454 -0
- package/src/runner/lock.ts +288 -0
- package/src/runner/report.test.ts +1 -0
- package/src/runner/run.test.ts +2044 -9
- package/src/runner/run.ts +825 -61
- package/src/runner/teardown-registry.ts +20 -78
- package/src/runner/types.ts +103 -7
- package/src/sandbox/errors.test.ts +72 -0
- package/src/sandbox/errors.ts +83 -0
- package/src/sandbox/keep-registry.ts +22 -46
- package/src/sandbox/resolve.test.ts +100 -0
- package/src/sandbox/resolve.ts +84 -27
- package/src/shared/entry-file-store.test.ts +149 -0
- package/src/shared/entry-file-store.ts +117 -0
- package/src/shared/failure-class.test.ts +137 -0
- package/src/shared/failure-class.ts +175 -0
- package/src/show/index.ts +5 -6
- package/src/show/render.test.ts +116 -15
- package/src/show/render.ts +124 -35
- package/src/types.ts +9 -0
- package/src/util.ts +31 -4
- package/src/view/app/App.tsx +5 -1
- package/src/view/client-dist/app.js +1 -1
- package/src/view/data.ts +5 -9
- package/src/view/shared/types.ts +7 -1
- package/src/view/view-report.test.ts +54 -0
|
@@ -20,7 +20,7 @@ import { t } from "../../i18n/index.ts";
|
|
|
20
20
|
import { verdictSymbol } from "../reporters/shared.ts";
|
|
21
21
|
import { formatCost } from "../../shared/format.ts";
|
|
22
22
|
import { assertionSummaryLines } from "../../scoring/display.ts";
|
|
23
|
-
import { encodeAttemptKey } from "../types.ts";
|
|
23
|
+
import { encodeAttemptKey, HALT_DIAGNOSTIC_CODE } from "../types.ts";
|
|
24
24
|
import {
|
|
25
25
|
panelCapabilityOf as panelCapability,
|
|
26
26
|
panelContentWidth,
|
|
@@ -32,6 +32,7 @@ import { stringWidth } from "../../report/model/text-layout.ts";
|
|
|
32
32
|
import type {
|
|
33
33
|
ActiveAttempt,
|
|
34
34
|
ActiveExperimentHook,
|
|
35
|
+
ActiveLockWait,
|
|
35
36
|
ActivePrecheck,
|
|
36
37
|
AttemptKey,
|
|
37
38
|
ExperimentHookName,
|
|
@@ -140,6 +141,40 @@ export function renderDurableLines(
|
|
|
140
141
|
const duration = event.durationMs !== undefined ? ` (${formatElapsed(event.durationMs)})` : "";
|
|
141
142
|
return [`${t("feedback.human.precheckJudgeDone")}${duration}`];
|
|
142
143
|
}
|
|
144
|
+
case "lock-wait": {
|
|
145
|
+
// 只服务非 TTY 退化流(TTY dashboard 的 appendDurable 对这个事件直接返回,运行级行由
|
|
146
|
+
// state.lockWaits 驱动,不进 scrollback,见 cli.md「等待并发 run 的显示」)。按实验聚合
|
|
147
|
+
// ——同一实验可能有多个用例先后撞锁,只在这个「有等待用例」窗口第一次打开(这是当前
|
|
148
|
+
// 唯一一条等待中的用例)与最后一次关闭(等待全部解决)各打印一行,中途加入/解决的用例
|
|
149
|
+
// 不逐条刷屏,与诊断按 dedupeKey 折叠同一种克制(state 已经是这条事件 reduce 之后的
|
|
150
|
+
// 快照,size 天然反映"这条事件之后"的计数)。
|
|
151
|
+
const agg = state.lockWaits.get(event.experimentId);
|
|
152
|
+
if (event.status === "started") {
|
|
153
|
+
if (!agg || agg.waiting.size !== 1) return []; // 不是这个窗口的第一条,静默
|
|
154
|
+
const holder = agg.waiting.get(event.evalId);
|
|
155
|
+
return [
|
|
156
|
+
t("feedback.human.lockWaitStarted", {
|
|
157
|
+
experimentId: event.experimentId,
|
|
158
|
+
count: agg.waiting.size,
|
|
159
|
+
pid: holder?.holderPid ?? "?",
|
|
160
|
+
}),
|
|
161
|
+
];
|
|
162
|
+
}
|
|
163
|
+
if (!agg || agg.waiting.size !== 0) return []; // 窗口还没关闭,静默
|
|
164
|
+
const parts: string[] = [];
|
|
165
|
+
if (agg.resolvedCarried > 0) parts.push(t("feedback.human.lockWaitCarried", { count: agg.resolvedCarried }));
|
|
166
|
+
if (agg.resolvedDispatched > 0) {
|
|
167
|
+
parts.push(t("feedback.human.lockWaitDispatched", { count: agg.resolvedDispatched }));
|
|
168
|
+
}
|
|
169
|
+
const summary = parts.length > 0 ? parts.join(" · ") : t("feedback.human.lockWaitCarried", { count: 0 });
|
|
170
|
+
return [
|
|
171
|
+
t("feedback.human.lockWaitResolved", {
|
|
172
|
+
experimentId: event.experimentId,
|
|
173
|
+
summary,
|
|
174
|
+
elapsed: formatElapsed(event.waitedMs ?? 0),
|
|
175
|
+
}),
|
|
176
|
+
];
|
|
177
|
+
}
|
|
143
178
|
case "summary":
|
|
144
179
|
return buildSummaryLines(event, state, panel);
|
|
145
180
|
case "saved":
|
|
@@ -208,8 +243,27 @@ function buildDiagnosticLines(event: DurableFeedbackEvent & { type: "diagnostic"
|
|
|
208
243
|
// count 从 state.diagnostics 读(reducer 已经按 key 去重累加),不在这里自己维护第二份计数。
|
|
209
244
|
const count = state.diagnostics.find((d) => d.key === event.key)?.count ?? 1;
|
|
210
245
|
const sym = event.severity === "error" ? "✗" : "!";
|
|
246
|
+
if (event.code === HALT_DIAGNOSTIC_CODE) {
|
|
247
|
+
// 止损闸落闸:一行 error 级通知,文案已经是完整的一句话(`experiment halted
|
|
248
|
+
// (dispatch-halted): <message>` / `eval halted: <message>`,见 docs/feature/
|
|
249
|
+
// error-classification/architecture.md「观察面」),不再加标题行、也不加 ×N 后缀——
|
|
250
|
+
// emitter 对每条未派发 attempt 都刷一次这条诊断以更新 data.unstarted,逐次打印就是
|
|
251
|
+
// 同一页文档明令禁止的「被中止的等待集 attempt 逐条刷屏」;未派发的数量由完成状态的
|
|
252
|
+
// `unstarted` 回答,不在这行重复。因此只在第一次出现时落一行(与 json profile 的
|
|
253
|
+
// isFirstOccurrence 同一条去重纪律)。
|
|
254
|
+
if (count > 1) return [];
|
|
255
|
+
return [`${sym} ${event.message}`];
|
|
256
|
+
}
|
|
257
|
+
// 标题用稳定词法(`code`),不是把折叠身份一起编进去的去重 key —— 人读的一行要能一眼认出
|
|
258
|
+
// 「这是哪一类诊断」,`compare/codex|memory/x` 那串身份属于 message 与机器面的具名字段。
|
|
211
259
|
const suffix = count > 1 ? ` (${count} attempts)` : "";
|
|
212
|
-
|
|
260
|
+
// 阶段标签走与失败行(`buildFailureLine`)同一个 `phaseLabel()` 投影:「在哪一步降级的」是
|
|
261
|
+
// 读者的第一个问题,message 里未必答得上。attempt 级诊断的 phase 由运行器写进 `data`
|
|
262
|
+
// (见 attempt.ts 的 recordDiagnostic);运行级诊断(止损闸、锁接管、budget)不属于任何
|
|
263
|
+
// 单条 attempt,天然没有 phase,标题退化成只有 code 一段。
|
|
264
|
+
const phase = typeof event.data?.phase === "string" ? (event.data.phase as LifecyclePhase) : undefined;
|
|
265
|
+
const heading = phase !== undefined ? `${phaseLabel(phase)} · ${event.code ?? event.key}` : (event.code ?? event.key);
|
|
266
|
+
return [`${sym} ${heading}${suffix}`, ` ${event.message}`];
|
|
213
267
|
}
|
|
214
268
|
|
|
215
269
|
/** 结束结论(`FAILED`/`PASSED`/…)+ `FAILURES`(有失败才出现)+ `KEPT SANDBOXES`(有留存才
|
|
@@ -418,14 +472,37 @@ function experimentHookLabel(hook: ExperimentHookName): string {
|
|
|
418
472
|
return hook === "setup" ? t("feedback.phase.experimentSetup") : t("feedback.phase.experimentTeardown");
|
|
419
473
|
}
|
|
420
474
|
|
|
475
|
+
/** 首行守恒计数的文案(见 cli.md「运行中的 live 面板」)。四项结局恒显示、零值不省略——
|
|
476
|
+
* 「0 errored」是一句有价值的肯定;`elsewhere` 只在非零时出现,没有并发 run 的场景少一项。
|
|
477
|
+
* 行长不是压缩它的理由:首行跟随终端全宽,九项写满仍是一行。两个调用点(live dashboard
|
|
478
|
+
* 首行、非 TTY heartbeat)共用这一份,不各自维护一份键选择逻辑。 */
|
|
479
|
+
function countsText(state: RunFeedbackState): string {
|
|
480
|
+
const outcomes = {
|
|
481
|
+
passed: state.passed,
|
|
482
|
+
failed: state.failed,
|
|
483
|
+
errored: state.errored,
|
|
484
|
+
skipped: state.skipped,
|
|
485
|
+
};
|
|
486
|
+
return state.elsewhere > 0
|
|
487
|
+
? t("feedback.human.countsWithElsewhere", {
|
|
488
|
+
total: state.total,
|
|
489
|
+
reused: state.reused,
|
|
490
|
+
running: state.running,
|
|
491
|
+
elsewhere: state.elsewhere,
|
|
492
|
+
queued: state.queued,
|
|
493
|
+
...outcomes,
|
|
494
|
+
})
|
|
495
|
+
: t("feedback.human.counts", {
|
|
496
|
+
total: state.total,
|
|
497
|
+
reused: state.reused,
|
|
498
|
+
running: state.running,
|
|
499
|
+
queued: state.queued,
|
|
500
|
+
...outcomes,
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
|
|
421
504
|
function formatCounts(state: RunFeedbackState): string {
|
|
422
|
-
const counts =
|
|
423
|
-
total: state.total,
|
|
424
|
-
reused: state.reused,
|
|
425
|
-
running: state.running,
|
|
426
|
-
queued: state.queued,
|
|
427
|
-
completed: state.completed,
|
|
428
|
-
});
|
|
505
|
+
const counts = countsText(state);
|
|
429
506
|
if (state.estimatedCostUSD === undefined || state.estimatedCostUSD <= 0) return counts;
|
|
430
507
|
return `${counts} ${formatCost(state.estimatedCostUSD)}`;
|
|
431
508
|
}
|
|
@@ -479,30 +556,23 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
|
|
|
479
556
|
// renderPanel 内部按另一个宽度钳制,行尾会被框吃掉——这正是 memory/
|
|
480
557
|
// live-dashboard-active-row-width-clamp-mismatch.md 的根因类别。
|
|
481
558
|
const contentWidth = panelContentWidth(capability.width, capability.mode, false);
|
|
482
|
-
const rows: PanelRow[] = [
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
reused: state.reused,
|
|
488
|
-
running: state.running,
|
|
489
|
-
queued: state.queued,
|
|
490
|
-
completed: state.completed,
|
|
491
|
-
}),
|
|
492
|
-
},
|
|
493
|
-
];
|
|
494
|
-
// 运行级行(judge 预检 + 实验钩子)排在 attempt 行前面(见 cli.md「judge 预检的显示」/
|
|
495
|
-
// 「实验级钩子的显示」):它们解释了为什么后面的 attempt 还停在 queued。预检又排在实验
|
|
496
|
-
// 钩子前面——它发生在最前(任何 attempt 派发之前)。Map 按插入序迭代,天然满足稳定 slot。
|
|
559
|
+
const rows: PanelRow[] = [{ kind: "line", text: countsText(state) }];
|
|
560
|
+
// 运行级行(judge 预检 + 实验钩子 + 用例锁等待)排在 attempt 行前面(见 cli.md「judge 预检
|
|
561
|
+
// 的显示」/「实验级钩子的显示」/「等待并发 run 的显示」):它们解释了为什么后面的 attempt
|
|
562
|
+
// 还停在 queued。预检排最前(发生在任何 attempt 派发之前),其次实验钩子,再是锁等待
|
|
563
|
+
// (排在实验钩子行之后、attempt 行之前)。Map 按插入序迭代,天然满足稳定 slot。
|
|
497
564
|
const precheck = state.activePrecheck;
|
|
498
565
|
const hookRows = [...state.experimentHooks.values()];
|
|
499
|
-
|
|
566
|
+
// 只有仍在等待(waiting 非空)的实验才占运行级行;窗口已关闭(全部 resolved)的条目只是
|
|
567
|
+
// 给非 TTY 聚合收尾行留的历史计数,TTY 不展示。
|
|
568
|
+
const lockWaitRows = [...state.lockWaits.values()].filter((w) => w.waiting.size > 0);
|
|
569
|
+
if (activeOrder.length > 0 || hookRows.length > 0 || lockWaitRows.length > 0 || precheck) {
|
|
500
570
|
rows.push({ kind: "divider", title: t("feedback.human.active") });
|
|
501
571
|
// 固定开销:上边框 + counts 行 + ACTIVE 横隔 + 下边框(boxed);plain 时同样按 4 行估算,
|
|
502
572
|
// 差一两行不影响「窄/矮终端先减 active slots」这条大方向。
|
|
503
573
|
const rowBudget = Math.max(0, io.stderr.rows - 4 - DASHBOARD_ROW_RESERVE);
|
|
504
574
|
const precheckCount = precheck ? 1 : 0;
|
|
505
|
-
const total = precheckCount + hookRows.length + activeOrder.length;
|
|
575
|
+
const total = precheckCount + hookRows.length + lockWaitRows.length + activeOrder.length;
|
|
506
576
|
// 窄/矮终端先减 active slots(减少行数),而不是先压缩单行内容 ——
|
|
507
577
|
// 单行内容的截断在 formatActiveRow 里按 contentWidth 单独处理。
|
|
508
578
|
const showCount = total <= rowBudget ? total : Math.max(0, rowBudget - 1);
|
|
@@ -512,7 +582,11 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
|
|
|
512
582
|
// 的行又观测到更长的值再推宽,导致同一帧内本该对齐的列错位。
|
|
513
583
|
const shownPrecheck = precheck && showCount > 0 ? precheck : undefined;
|
|
514
584
|
const shownHooks = hookRows.slice(0, Math.max(0, showCount - (shownPrecheck ? 1 : 0)));
|
|
515
|
-
const
|
|
585
|
+
const shownLockWaits = lockWaitRows.slice(
|
|
586
|
+
0,
|
|
587
|
+
Math.max(0, showCount - (shownPrecheck ? 1 : 0) - shownHooks.length),
|
|
588
|
+
);
|
|
589
|
+
const shownRunLevel = (shownPrecheck ? 1 : 0) + shownHooks.length + shownLockWaits.length;
|
|
516
590
|
const shownActive: ActiveAttempt[] = [];
|
|
517
591
|
for (const key of activeOrder) {
|
|
518
592
|
if (shownRunLevel + shownActive.length >= showCount) break;
|
|
@@ -533,6 +607,9 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
|
|
|
533
607
|
for (const hookRow of shownHooks) {
|
|
534
608
|
activeLines.push(formatExperimentHookRow(hookRow, io, contentWidth, evalWidth, whoWidth));
|
|
535
609
|
}
|
|
610
|
+
for (const lockWaitRow of shownLockWaits) {
|
|
611
|
+
activeLines.push(formatLockWaitRow(lockWaitRow, io, contentWidth));
|
|
612
|
+
}
|
|
536
613
|
for (const active of shownActive) {
|
|
537
614
|
activeLines.push(formatActiveRow(active, io, contentWidth, evalWidth, whoWidth));
|
|
538
615
|
}
|
|
@@ -584,8 +661,9 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
|
|
|
584
661
|
// 更新,coordinator 紧接着的 redrawDynamic 会画出来);成功钩子不写 scrollback 永久行
|
|
585
662
|
// (见 cli.md「实验级钩子的显示」)。非 TTY 退化流才逐行追加(见 renderDurableLines)。
|
|
586
663
|
// judge 预检同理:TTY 下只驱动 state.activePrecheck 的运行级 active 行(coordinator 紧接着的
|
|
587
|
-
// redrawDynamic 会画出来),不写 scrollback 永久行(见 cli.md「judge 预检的显示」)
|
|
588
|
-
|
|
664
|
+
// redrawDynamic 会画出来),不写 scrollback 永久行(见 cli.md「judge 预检的显示」)。用例锁
|
|
665
|
+
// 等待同理:TTY 下由 state.lockWaits 驱动运行级 active 行(见 cli.md「等待并发 run 的显示」)。
|
|
666
|
+
if (event.type === "experiment-hook" || event.type === "precheck" || event.type === "lock-wait") return;
|
|
589
667
|
writeDurable(io, event, state, false);
|
|
590
668
|
},
|
|
591
669
|
activity(text) {
|
|
@@ -682,6 +760,26 @@ function formatExperimentHookRow(
|
|
|
682
760
|
return prefix + (hook.detail ?? "").slice(0, budget);
|
|
683
761
|
}
|
|
684
762
|
|
|
763
|
+
/** 用例锁等待的运行级行:`● waiting on another run · <exp> <elapsed> <n> evals · pid <pid>`
|
|
764
|
+
* (cli.md「等待并发 run 的显示」)。elapsed 从最早一条等待的 startedAt 算(存活性证明,
|
|
765
|
+
* 与其它运行级行同一约定);pid 取最早一条等待对应的持有方——一个实验可能同时撞上多把不同
|
|
766
|
+
* 持有方的锁,这里选一个稳定的代表值展示,不逐条列出(与 earlyExit 代表 attempt 的选法同一
|
|
767
|
+
* 种「挑一个确定性代表」思路)。不吃 evalWidth/whoWidth 身份列宽约束——与 `formatPrecheckRow`
|
|
768
|
+
* 同理:选中用例全在等锁时,本实验没有派发中的 attempt,那两个宽度还压在初始值 0,会把
|
|
769
|
+
* label 截成 "w…"(与 memory/live-dashboard-active-row-width-clamp-mismatch.md 同一根因类别,
|
|
770
|
+
* 只是发生在锁等待场景);label 直接用整行宽度。 */
|
|
771
|
+
function formatLockWaitRow(wait: ActiveLockWait, io: FeedbackIO, columns: number): string {
|
|
772
|
+
const entries = [...wait.waiting.values()].sort((a, b) => a.startedAt - b.startedAt);
|
|
773
|
+
const earliest = entries[0]!;
|
|
774
|
+
const elapsed = formatElapsed(io.clock.now() - earliest.startedAt).padStart(6);
|
|
775
|
+
const sym = "● ";
|
|
776
|
+
const label = `${t("feedback.human.waitingOnAnotherRun")} · ${wait.experimentId}`;
|
|
777
|
+
const prefix = `${sym}${label} ${elapsed} `;
|
|
778
|
+
const budget = Math.max(0, columns - prefix.length);
|
|
779
|
+
const detail = t("feedback.human.lockWaitDetail", { count: entries.length, pid: earliest.holderPid ?? "?" });
|
|
780
|
+
return padTrunc(prefix + detail.slice(0, budget), columns);
|
|
781
|
+
}
|
|
782
|
+
|
|
685
783
|
// ───────────────────────── 非 TTY:human 文案的纯追加流 ─────────────────────────
|
|
686
784
|
//
|
|
687
785
|
// 单一 stdout 有序流(见 memory/exp-output-two-forms-ruling.md 的补充裁决):从 start 到结束
|
|
@@ -730,6 +828,9 @@ function createPlainRenderer(io: FeedbackIO): FeedbackRenderer {
|
|
|
730
828
|
export interface HumanDryPlanRow {
|
|
731
829
|
experimentId: string;
|
|
732
830
|
evalId: string;
|
|
831
|
+
/** 该用例正被另一条并行 Invocation 持锁运行(见 docs/feature/experiments/architecture.md
|
|
832
|
+
* 「并发 Invocation:用例锁」);计划行尾如实标注,`--dry` 本身不取锁、不等待。 */
|
|
833
|
+
locked?: boolean;
|
|
733
834
|
}
|
|
734
835
|
|
|
735
836
|
export interface HumanDryPlanInput {
|
|
@@ -772,7 +873,8 @@ export function renderHumanDryPlan(input: HumanDryPlanInput): string {
|
|
|
772
873
|
}
|
|
773
874
|
const idWidth = Math.max(0, ...input.rows.map((row) => stringWidth(row.experimentId)));
|
|
774
875
|
for (const row of input.rows) {
|
|
775
|
-
|
|
876
|
+
const base = `${row.experimentId}${" ".repeat(idWidth - stringWidth(row.experimentId) + 2)}${row.evalId}`;
|
|
877
|
+
lines.push(row.locked ? `${base} ${t("feedback.human.lockedRowSuffix")}` : base);
|
|
776
878
|
}
|
|
777
879
|
return `${lines.join("\n")}\n`;
|
|
778
880
|
}
|
|
@@ -8,8 +8,10 @@
|
|
|
8
8
|
// docs/engineering/testing/e2e/cli.md「反馈输出格式」在真实进程输出上验收。
|
|
9
9
|
|
|
10
10
|
import { describe, expect, it } from "vitest";
|
|
11
|
-
import { computeExitCode, renderJsonPlanDocument } from "./json.ts";
|
|
12
|
-
import
|
|
11
|
+
import { computeExitCode, createJsonRenderer, renderJsonPlanDocument } from "./json.ts";
|
|
12
|
+
import { createInitialRunFeedbackState, reduceRunFeedback } from "./reducer.ts";
|
|
13
|
+
import { createFakeFeedbackIO } from "./testing.ts";
|
|
14
|
+
import { HALT_DIAGNOSTIC_CODE, type DurableFeedbackEvent, type InvocationCompletion, type InvocationSummary } from "../types.ts";
|
|
13
15
|
|
|
14
16
|
function summary(overrides: Partial<InvocationSummary> = {}): InvocationSummary {
|
|
15
17
|
return {
|
|
@@ -77,6 +79,113 @@ describe("computeExitCode:CompletionStatus 驱动退出码,不只看 failed/erro
|
|
|
77
79
|
});
|
|
78
80
|
});
|
|
79
81
|
|
|
82
|
+
/** 依次喂进 reducer 再交给 json renderer(与生产的 coordinator 同序:先 reduce 后 render),
|
|
83
|
+
* 返回逐行解析出的事件对象。 */
|
|
84
|
+
function emitDurable(events: readonly DurableFeedbackEvent[]): Record<string, unknown>[] {
|
|
85
|
+
const { io, stdout } = createFakeFeedbackIO();
|
|
86
|
+
const renderer = createJsonRenderer({ io });
|
|
87
|
+
let state = createInitialRunFeedbackState();
|
|
88
|
+
for (const event of events) {
|
|
89
|
+
state = reduceRunFeedback(state, event);
|
|
90
|
+
renderer.appendDurable(event, state);
|
|
91
|
+
}
|
|
92
|
+
return stdout.writes
|
|
93
|
+
.join("")
|
|
94
|
+
.split("\n")
|
|
95
|
+
.filter(Boolean)
|
|
96
|
+
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
describe("warning 事件:code 是稳定词法,折叠身份走具名字段", () => {
|
|
100
|
+
it("code 用 DiagnosticInput.code 的干净字面量,不透传编了身份的去重 key", () => {
|
|
101
|
+
const [warning] = emitDurable([
|
|
102
|
+
{
|
|
103
|
+
type: "diagnostic",
|
|
104
|
+
at: 1,
|
|
105
|
+
key: "lock-taken-over:compare/codex|memory/retention",
|
|
106
|
+
code: "lock-taken-over",
|
|
107
|
+
severity: "warning",
|
|
108
|
+
message: "took over a stale lock",
|
|
109
|
+
data: { experimentId: "compare/codex", evalId: "memory/retention" },
|
|
110
|
+
},
|
|
111
|
+
]);
|
|
112
|
+
expect(warning).toMatchObject({
|
|
113
|
+
event: "warning",
|
|
114
|
+
code: "lock-taken-over",
|
|
115
|
+
level: "warning",
|
|
116
|
+
experimentId: "compare/codex",
|
|
117
|
+
evalId: "memory/retention",
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("没给 code 的诊断回落到 key(折叠身份本就不进 key 的那些天生是干净字面量)", () => {
|
|
122
|
+
const [warning] = emitDurable([
|
|
123
|
+
{ type: "diagnostic", at: 1, key: "memory-warmup-degraded", severity: "warning", message: "cold index" },
|
|
124
|
+
]);
|
|
125
|
+
expect(warning).toMatchObject({ code: "memory-warmup-degraded" });
|
|
126
|
+
expect(warning).not.toHaveProperty("experimentId");
|
|
127
|
+
expect(warning).not.toHaveProperty("evalId");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("eval 闸的 dispatch-halted:code 干净、evalId 与 phase 都在事件流里透得出", () => {
|
|
131
|
+
const [warning] = emitDurable([
|
|
132
|
+
{
|
|
133
|
+
type: "diagnostic",
|
|
134
|
+
at: 1,
|
|
135
|
+
key: "dispatch-halted:eval:compare/codex|memory/retention",
|
|
136
|
+
code: HALT_DIAGNOSTIC_CODE,
|
|
137
|
+
severity: "error",
|
|
138
|
+
message: "eval halted: fixture db is empty; run scripts/seed.ts",
|
|
139
|
+
data: {
|
|
140
|
+
experimentId: "compare/codex",
|
|
141
|
+
scope: "eval",
|
|
142
|
+
evalId: "memory/retention",
|
|
143
|
+
phase: "eval.run",
|
|
144
|
+
unstarted: 0,
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
]);
|
|
148
|
+
expect(warning).toMatchObject({
|
|
149
|
+
event: "warning",
|
|
150
|
+
code: "dispatch-halted",
|
|
151
|
+
level: "error",
|
|
152
|
+
phase: "eval.run",
|
|
153
|
+
experimentId: "compare/codex",
|
|
154
|
+
evalId: "memory/retention",
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it("身份从 data 取(闸不是 attempt 级、不伪造 identity);有 identity 时 identity 优先", () => {
|
|
159
|
+
const [fromIdentity] = emitDurable([
|
|
160
|
+
{
|
|
161
|
+
type: "diagnostic",
|
|
162
|
+
at: 1,
|
|
163
|
+
key: "fail-fast:x",
|
|
164
|
+
code: "fail-fast",
|
|
165
|
+
severity: "warning",
|
|
166
|
+
message: "deterministic failure",
|
|
167
|
+
identity: { experimentId: "compare/codex", evalId: "memory/a", attempt: 0 },
|
|
168
|
+
data: { experimentId: "other/exp", evalId: "memory/z" },
|
|
169
|
+
},
|
|
170
|
+
]);
|
|
171
|
+
expect(fromIdentity).toMatchObject({ experimentId: "compare/codex", evalId: "memory/a" });
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("同一 dedupeKey 只追加一次:emitter 为刷新 data.unstarted 反复报同一条闸,事件流不重复", () => {
|
|
175
|
+
const halted = (at: number, unstarted: number): DurableFeedbackEvent => ({
|
|
176
|
+
type: "diagnostic",
|
|
177
|
+
at,
|
|
178
|
+
key: "dispatch-halted:experiment:compare/codex",
|
|
179
|
+
code: HALT_DIAGNOSTIC_CODE,
|
|
180
|
+
severity: "error",
|
|
181
|
+
message: "experiment halted (dispatch-halted): shared service is down",
|
|
182
|
+
data: { experimentId: "compare/codex", scope: "experiment", phase: "eval.run", unstarted },
|
|
183
|
+
});
|
|
184
|
+
const events = emitDurable([halted(1, 0), halted(2, 1), halted(3, 2)]);
|
|
185
|
+
expect(events.filter((e) => e.event === "warning")).toHaveLength(1);
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
80
189
|
describe("renderJsonPlanDocument:单个 ExpPlanDocument,不是事件流", () => {
|
|
81
190
|
it("输出恰好一行 JSON,可解析为单个对象而不是逐行事件序列", () => {
|
|
82
191
|
const text = renderJsonPlanDocument({
|
|
@@ -101,6 +210,22 @@ describe("renderJsonPlanDocument:单个 ExpPlanDocument,不是事件流", () =>
|
|
|
101
210
|
expect(doc.matrix).toHaveLength(2);
|
|
102
211
|
});
|
|
103
212
|
|
|
213
|
+
it("locked 为 true 的行原样透传;省略的行不出现 locked 字段(JSON.stringify 丢弃 undefined 属性)", () => {
|
|
214
|
+
const text = renderJsonPlanDocument({
|
|
215
|
+
total: 2,
|
|
216
|
+
evals: 2,
|
|
217
|
+
configs: 1,
|
|
218
|
+
runs: 1,
|
|
219
|
+
matrix: [
|
|
220
|
+
{ experimentId: "compare/codex", evalId: "memory/a", reused: false, locked: true },
|
|
221
|
+
{ experimentId: "compare/codex", evalId: "memory/b", reused: false },
|
|
222
|
+
],
|
|
223
|
+
});
|
|
224
|
+
const doc = JSON.parse(text);
|
|
225
|
+
expect(doc.matrix[0]).toMatchObject({ evalId: "memory/a", locked: true });
|
|
226
|
+
expect(doc.matrix[1]).not.toHaveProperty("locked");
|
|
227
|
+
});
|
|
228
|
+
|
|
104
229
|
it("reused 是 matrix 逐行 reused 之和(命中数量,不是 attempt 数)", () => {
|
|
105
230
|
const text = renderJsonPlanDocument({
|
|
106
231
|
total: 3,
|
|
@@ -104,13 +104,24 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
|
|
|
104
104
|
noteCheckpoint(event.at);
|
|
105
105
|
if (!isFirstOccurrence(state, event.key)) return; // 去重后只追加一次(cli.md)
|
|
106
106
|
const phase = typeof event.data?.phase === "string" ? (event.data.phase as LifecyclePhase) : undefined;
|
|
107
|
+
// `code` 是 cli.md `WarningEvent` 里那个稳定词法(`lock-taken-over` / `dispatch-halted`),
|
|
108
|
+
// **不是**去重 key:去重 key 常把折叠身份编进去(`lock-taken-over:<exp>|<eval>`),原样
|
|
109
|
+
// 透出会让消费方拿到一个每次运行都不同的 code、没法按值分支。折叠到哪一条实验/用例
|
|
110
|
+
// 由下面的 experimentId/evalId 两个具名字段回答。
|
|
111
|
+
const code = event.code ?? event.key;
|
|
112
|
+
// 身份两字段:attempt 级诊断从 identity 取;运行级诊断(实验闸 / eval 闸这类不属于
|
|
113
|
+
// 任何单条 attempt 的事实)不许伪造 identity,从 `data` 的同名字段取(见 ../types.ts
|
|
114
|
+
// "diagnostic" 变体的 identity 注释)。
|
|
115
|
+
const experimentId = event.identity?.experimentId ?? stringField(event.data?.experimentId);
|
|
116
|
+
const evalId = event.identity?.evalId ?? stringField(event.data?.evalId);
|
|
107
117
|
writeEvent(io, {
|
|
108
118
|
event: "warning",
|
|
109
|
-
code
|
|
119
|
+
code,
|
|
110
120
|
level: event.severity,
|
|
111
121
|
message: event.message,
|
|
112
122
|
...(phase !== undefined ? { phase } : {}),
|
|
113
|
-
...(
|
|
123
|
+
...(experimentId !== undefined ? { experimentId } : {}),
|
|
124
|
+
...(evalId !== undefined ? { evalId } : {}),
|
|
114
125
|
});
|
|
115
126
|
return;
|
|
116
127
|
}
|
|
@@ -148,6 +159,26 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
|
|
|
148
159
|
return;
|
|
149
160
|
}
|
|
150
161
|
|
|
162
|
+
case "lock-wait": {
|
|
163
|
+
noteCheckpoint(event.at);
|
|
164
|
+
writeEvent(io, {
|
|
165
|
+
event: "lock_wait",
|
|
166
|
+
experimentId: event.experimentId,
|
|
167
|
+
evalId: event.evalId,
|
|
168
|
+
status: event.status,
|
|
169
|
+
...(event.holderPid !== undefined ? { holderPid: event.holderPid } : {}),
|
|
170
|
+
...(event.holderHost !== undefined ? { holderHost: event.holderHost } : {}),
|
|
171
|
+
// 折叠 carried/dispatched 两个内部计数为一个公开 resolution:仍有 attempt 需要真实
|
|
172
|
+
// 派发(dispatched > 0)记 "dispatched",全部由携带满足才记 "carried"
|
|
173
|
+
// (见 ../types.ts 里 DurableFeedbackEvent "lock-wait" 变体的字段注释)。
|
|
174
|
+
...(event.status === "resolved"
|
|
175
|
+
? { resolution: (event.dispatched ?? 0) > 0 ? "dispatched" : "carried" }
|
|
176
|
+
: {}),
|
|
177
|
+
...(event.waitedMs !== undefined ? { waitedMs: event.waitedMs } : {}),
|
|
178
|
+
});
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
|
|
151
182
|
case "interrupted": {
|
|
152
183
|
noteCheckpoint(event.at);
|
|
153
184
|
if (!isFirstOccurrence(state, "interrupted")) return;
|
|
@@ -208,8 +239,14 @@ export function createJsonRenderer(options: JsonRendererOptions): FeedbackRender
|
|
|
208
239
|
total: state.total,
|
|
209
240
|
reused: state.reused,
|
|
210
241
|
running: state.running,
|
|
242
|
+
elsewhere: state.elsewhere,
|
|
211
243
|
queued: state.queued,
|
|
212
|
-
|
|
244
|
+
// 与 human 首行同一份状态、同一套划分:已了结的 attempt 按 verdict 分项,消费方不必
|
|
245
|
+
// 自己从合计数里猜成败(契约见 docs/feature/experiments/cli.md 的 ProgressEvent)。
|
|
246
|
+
passed: state.passed,
|
|
247
|
+
failed: state.failed,
|
|
248
|
+
errored: state.errored,
|
|
249
|
+
skipped: state.skipped,
|
|
213
250
|
});
|
|
214
251
|
},
|
|
215
252
|
|
|
@@ -225,6 +262,12 @@ function isFirstOccurrence(state: RunFeedbackState, key: string): boolean {
|
|
|
225
262
|
return (state.diagnostics.find((d) => d.key === key)?.count ?? 0) <= 1;
|
|
226
263
|
}
|
|
227
264
|
|
|
265
|
+
/** `data` 里的一个字符串字段;非字符串(或缺失)一律当没有,不把数字/对象硬塞进只接受
|
|
266
|
+
* 字符串的事件字段里。 */
|
|
267
|
+
function stringField(value: JsonValue | undefined): string | undefined {
|
|
268
|
+
return typeof value === "string" ? value : undefined;
|
|
269
|
+
}
|
|
270
|
+
|
|
228
271
|
// ───────────────────────── failure / error 事件(不设上限) ─────────────────────────
|
|
229
272
|
|
|
230
273
|
function writeFailureOrError(io: FeedbackIO, event: DurableFeedbackEvent & { type: "failure" }): void {
|
|
@@ -346,6 +389,11 @@ export interface JsonPlanRow {
|
|
|
346
389
|
evalId: string;
|
|
347
390
|
/** 命中缓存指纹,本次不会派发新 attempt。 */
|
|
348
391
|
reused: boolean;
|
|
392
|
+
/** 该用例正被另一条并行 Invocation 持锁运行,真实运行时将等待后携带或补跑(见
|
|
393
|
+
* docs/feature/experiments/architecture.md「并发 Invocation:用例锁」)。`--dry` 只读锁
|
|
394
|
+
* 目录,不取锁、不等待;省略等于 `false`(JSON.stringify 丢弃 `undefined` 属性,
|
|
395
|
+
* 天然满足这条省略语义,不需要显式写 `locked: false`)。 */
|
|
396
|
+
locked?: boolean;
|
|
349
397
|
}
|
|
350
398
|
|
|
351
399
|
export interface JsonPlanInput {
|