niceeval 0.10.3-canary.6 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/bin/niceeval.js +9 -4
  2. package/dist/agents/types.d.ts +5 -4
  3. package/dist/context/turn-errors.d.ts +27 -23
  4. package/dist/i18n/en.d.ts +15 -0
  5. package/dist/i18n/en.js +16 -1
  6. package/dist/i18n/zh-CN.d.ts +16 -1
  7. package/dist/i18n/zh-CN.js +16 -1
  8. package/dist/o11y/derive.js +28 -24
  9. package/dist/o11y/types.d.ts +8 -5
  10. package/dist/report/components/attempt-detail/UsageTable.js +4 -7
  11. package/dist/report/components/attempt-detail/compute.d.ts +2 -2
  12. package/dist/report/components/attempt-detail/compute.js +2 -6
  13. package/dist/report/components/attempt-detail/faces.js +7 -7
  14. package/dist/report/components/attempt-detail/index.js +0 -3
  15. package/dist/report/components/entity-lists/EvalList.js +0 -0
  16. package/dist/report/components/metric-views/compute.js +1 -1
  17. package/dist/report/model/types.d.ts +3 -4
  18. package/dist/results/locator.js +0 -0
  19. package/dist/results/select.d.ts +6 -0
  20. package/dist/results/select.js +8 -0
  21. package/dist/runner/feedback/sink.d.ts +26 -1
  22. package/dist/runner/fingerprint.d.ts +23 -0
  23. package/dist/runner/types.d.ts +105 -7
  24. package/dist/sandbox/errors.d.ts +29 -0
  25. package/dist/sandbox/resolve.d.ts +9 -0
  26. package/dist/shared/failure-class.d.ts +91 -0
  27. package/dist/types.d.ts +1 -0
  28. package/dist/util.d.ts +3 -2
  29. package/dist/util.js +31 -5
  30. package/docs-site/zh/explanation/runner.mdx +35 -0
  31. package/docs-site/zh/reference/cli.mdx +10 -1
  32. package/docs-site/zh/reference/events.mdx +4 -4
  33. package/docs-site/zh/troubleshooting/debugging.mdx +11 -0
  34. package/docs-site/zh/tutorials/viewing-results.mdx +35 -3
  35. package/package.json +28 -24
  36. package/src/agents/ai-sdk.test.ts +26 -0
  37. package/src/agents/ai-sdk.ts +7 -4
  38. package/src/agents/index.ts +5 -3
  39. package/src/agents/langgraph.test.ts +30 -0
  40. package/src/agents/langgraph.ts +5 -2
  41. package/src/agents/openai-compat.test.ts +35 -0
  42. package/src/agents/openai-compat.ts +16 -4
  43. package/src/agents/sdk-streams.test.ts +66 -0
  44. package/src/agents/sdk-streams.ts +3 -1
  45. package/src/agents/types.ts +5 -4
  46. package/src/cli.ts +81 -16
  47. package/src/context/context.test.ts +34 -0
  48. package/src/context/context.ts +14 -5
  49. package/src/context/send-retry.test.ts +86 -0
  50. package/src/context/send-retry.ts +37 -12
  51. package/src/context/session.ts +24 -0
  52. package/src/context/turn-errors.test.ts +124 -17
  53. package/src/context/turn-errors.ts +60 -50
  54. package/src/define.ts +5 -0
  55. package/src/i18n/en.ts +22 -1
  56. package/src/i18n/zh-CN.ts +22 -1
  57. package/src/index.ts +8 -0
  58. package/src/o11y/cost.ts +4 -2
  59. package/src/o11y/derive.test.ts +40 -0
  60. package/src/o11y/derive.ts +28 -22
  61. package/src/o11y/otlp/sandbox-receiver.test.ts +201 -0
  62. package/src/o11y/otlp/sandbox-receiver.ts +73 -27
  63. package/src/o11y/parsers/bub.test.ts +30 -0
  64. package/src/o11y/parsers/bub.ts +5 -2
  65. package/src/o11y/parsers/codex.test.ts +19 -0
  66. package/src/o11y/parsers/codex.ts +5 -2
  67. package/src/o11y/types.ts +8 -5
  68. package/src/report/components/attempt-detail/UsageTable.tsx +4 -6
  69. package/src/report/components/attempt-detail/attempt-components.test.tsx +5 -8
  70. package/src/report/components/attempt-detail/compute.ts +2 -7
  71. package/src/report/components/attempt-detail/faces.ts +7 -7
  72. package/src/report/components/attempt-detail/index.tsx +0 -3
  73. package/src/report/components/entity-lists/EvalList.tsx +0 -0
  74. package/src/report/components/metric-views/compute.ts +1 -1
  75. package/src/report/model/types.ts +3 -4
  76. package/src/results/format.ts +9 -2
  77. package/src/results/index.ts +2 -0
  78. package/src/results/locator.ts +0 -0
  79. package/src/results/open.ts +132 -21
  80. package/src/results/select.ts +12 -0
  81. package/src/results/skipped-notice.ts +0 -0
  82. package/src/runner/attempt.test.ts +116 -0
  83. package/src/runner/attempt.ts +89 -8
  84. package/src/runner/discover.ts +21 -3
  85. package/src/runner/feedback/coordinator.ts +24 -2
  86. package/src/runner/feedback/eval-conclusions.ts +6 -3
  87. package/src/runner/feedback/human.test.ts +300 -6
  88. package/src/runner/feedback/human.ts +132 -30
  89. package/src/runner/feedback/json.test.ts +127 -2
  90. package/src/runner/feedback/json.ts +51 -3
  91. package/src/runner/feedback/reducer.test.ts +328 -29
  92. package/src/runner/feedback/reducer.ts +84 -9
  93. package/src/runner/feedback/sink.ts +39 -1
  94. package/src/runner/fingerprint.ts +49 -19
  95. package/src/runner/gate-lease.test.ts +510 -0
  96. package/src/runner/gate-lease.ts +350 -0
  97. package/src/runner/lock.test.ts +454 -0
  98. package/src/runner/lock.ts +288 -0
  99. package/src/runner/report.test.ts +1 -0
  100. package/src/runner/run.test.ts +2044 -9
  101. package/src/runner/run.ts +825 -61
  102. package/src/runner/teardown-registry.ts +20 -78
  103. package/src/runner/types.ts +103 -7
  104. package/src/sandbox/errors.test.ts +72 -0
  105. package/src/sandbox/errors.ts +83 -0
  106. package/src/sandbox/keep-registry.ts +22 -46
  107. package/src/sandbox/resolve.test.ts +100 -0
  108. package/src/sandbox/resolve.ts +84 -27
  109. package/src/shared/entry-file-store.test.ts +149 -0
  110. package/src/shared/entry-file-store.ts +117 -0
  111. package/src/shared/failure-class.test.ts +137 -0
  112. package/src/shared/failure-class.ts +175 -0
  113. package/src/show/index.ts +5 -6
  114. package/src/show/render.test.ts +116 -15
  115. package/src/show/render.ts +124 -35
  116. package/src/types.ts +9 -0
  117. package/src/util.ts +31 -4
  118. package/src/view/app/App.tsx +5 -1
  119. package/src/view/client-dist/app.js +1 -1
  120. package/src/view/data.ts +5 -9
  121. package/src/view/shared/types.ts +7 -1
  122. package/src/view/view-report.test.ts +54 -0
@@ -0,0 +1,175 @@
1
+ // 执行失败分类的词表、抛出点糖衣类与结构守卫——全仓唯一一份两轴词表,turn 失败、生命周期
2
+ // 阶段失败与 sandbox provisioning 失败说同一种语言(判据见
3
+ // docs/feature/error-classification/README.md「分类」)。
4
+ //
5
+ // 本模块刻意零依赖(只 import type):公开面活在 Promise 世界,糖衣类不继承任何 effect 类型,
6
+ // 生成的 .d.ts 零 effect 依赖(见 architecture.md「Effect 边界」)。turn 链的实现在
7
+ // src/context/turn-errors.ts,provisioning 的内部分类在 src/sandbox/,两边各自实现、只共享
8
+ // 这份词表。
9
+
10
+ import type { LifecyclePhase } from "../runner/types.ts";
11
+
12
+ /** 空间轴取值:失败死因的波及范围。 */
13
+ export type FailureScope = "attempt" | "eval" | "experiment";
14
+
15
+ /**
16
+ * 一次执行失败的分类:`retryable`(时间轴)与 `scope`(空间轴)是仅有的两条决策轴;
17
+ * `reason` 是开放词表的细分诊断,只进 activity 与诊断文案,不参与策略。内建兜底产出
18
+ * reason `"rate_limit"` / `"network"`;声明方可自造词。`scope` 缺省 `"attempt"`。
19
+ *
20
+ * `retryable: true` 时 `reason` 必填:可重试的失败一定出现在 activity 行与可能的耗尽摘要里,
21
+ * 那里需要一个给人读的词;不可重试的失败常常说不清是什么(这正是它不可重试的原因)。
22
+ */
23
+ export type FailureClass =
24
+ | { readonly retryable: true; readonly reason: string; readonly scope?: FailureScope }
25
+ | { readonly retryable: false; readonly reason?: string; readonly scope?: FailureScope };
26
+
27
+ /**
28
+ * 实验级分类器的输入:本实验任意 per-attempt 阶段的一次终局失败。
29
+ */
30
+ export interface AttemptFailureInfo {
31
+ /** 失败发生在哪个生命周期阶段;turn 失败恒为 `"agent.run"`。 */
32
+ readonly phase: LifecyclePhase;
33
+ /** 与报错文案同源的失败文本:thrown 取错误链(含 cause 链)message 串接,turn 失败取 `turnErrorText`。 */
34
+ readonly text: string;
35
+ /** 原始失败对象:thrown 形态是抛出的错误,turn 失败形态是那个失败 Turn。 */
36
+ readonly cause: unknown;
37
+ }
38
+
39
+ /**
40
+ * 实验可选分类器,挂载在 `ExperimentDef.classifyFailure`:识别自家共享基建的死因
41
+ * (对自家隧道 host 的拒连一类)。返回 `undefined` 表示「不认识,交给后续链路」。
42
+ * 分类器必须快、纯、不抛错——抛错按 `undefined` 回落并被吞掉,不掩盖原始失败。
43
+ */
44
+ export type AttemptFailureClassifier = (failure: AttemptFailureInfo) => FailureClass | undefined;
45
+
46
+ /** 携带分类的错误对象的判别字段值;识别只看这个数据字段,不看类身份。 */
47
+ const FAILURE_CLASS_TAG = "NiceevalClassifiedError";
48
+
49
+ /**
50
+ * 从任意 per-attempt 阶段抛出:全实验剩余 attempt 同因必死,停止派发。
51
+ * 携带 `{ retryable: false, scope: "experiment" }`,message 原样走完反馈流与
52
+ * `dispatch-halted` 诊断——把它写成「现象 + 下一步」的修复提示。
53
+ */
54
+ export class ExperimentFatalError extends Error {
55
+ readonly _tag = FAILURE_CLASS_TAG;
56
+ readonly class: FailureClass = { retryable: false, scope: "experiment" };
57
+
58
+ constructor(message: string, options?: { cause?: unknown }) {
59
+ super(message, options);
60
+ this.name = "ExperimentFatalError";
61
+ }
62
+ }
63
+
64
+ /**
65
+ * 从任意 per-attempt 阶段抛出:本 eval 剩余 attempt 同因必死,停止派发。
66
+ * 携带 `{ retryable: false, scope: "eval" }`;message 同样是走完全程的修复提示。
67
+ */
68
+ export class EvalFatalError extends Error {
69
+ readonly _tag = FAILURE_CLASS_TAG;
70
+ readonly class: FailureClass = { retryable: false, scope: "eval" };
71
+
72
+ constructor(message: string, options?: { cause?: unknown }) {
73
+ super(message, options);
74
+ this.name = "EvalFatalError";
75
+ }
76
+ }
77
+
78
+ /**
79
+ * 结构守卫:识别任何携带分类的错误对象(`_tag` + `class` 两个数据字段),沿 `cause` 链逐层
80
+ * 查找、取**最外层**命中——糖衣类被上层库包装再抛时声明不丢失。识别不依赖 `instanceof`:
81
+ * 依赖树里出现第二份 niceeval 实例(link、版本重复)时类身份静默失效,数据不会。
82
+ */
83
+ export function failureClassOf(error: unknown): FailureClass | undefined {
84
+ let current: unknown = error;
85
+ for (let depth = 0; depth < CAUSE_CHAIN_DEPTH && current != null; depth++) {
86
+ if (typeof current === "object") {
87
+ const candidate = current as { _tag?: unknown; class?: unknown };
88
+ if (candidate._tag === FAILURE_CLASS_TAG && isFailureClass(candidate.class)) return candidate.class;
89
+ }
90
+ current = causeOf(current);
91
+ }
92
+ return undefined;
93
+ }
94
+
95
+ /** cause 链与错误文本串接共用的深度上限:防御自引用 / 病态深链,不做环检测。 */
96
+ const CAUSE_CHAIN_DEPTH = 5;
97
+
98
+ function causeOf(value: unknown): unknown {
99
+ return typeof value === "object" && value !== null ? (value as { cause?: unknown }).cause : undefined;
100
+ }
101
+
102
+ function isFailureClass(value: unknown): value is FailureClass {
103
+ if (typeof value !== "object" || value === null) return false;
104
+ const v = value as { retryable?: unknown; reason?: unknown; scope?: unknown };
105
+ if (typeof v.retryable !== "boolean") return false;
106
+ if (v.retryable && typeof v.reason !== "string") return false;
107
+ if (v.reason !== undefined && typeof v.reason !== "string") return false;
108
+ if (v.scope !== undefined && v.scope !== "attempt" && v.scope !== "eval" && v.scope !== "experiment") return false;
109
+ return true;
110
+ }
111
+
112
+ /**
113
+ * @internal 把框架决议出的分类挂到即将浮出的失败对象上,供 attempt 封口经 `failureClassOf`
114
+ * 读取(止损闸的消费点)。两个字段都不可枚举——不进 JSON、不进 console 输出,只是路由标记;
115
+ * 已携带分类的对象不覆盖(抛出点声明优先),不可写对象静默跳过(分类是旁路,不制造新失败)。
116
+ */
117
+ export function attachFailureClass<T>(target: T, cls: FailureClass): T {
118
+ if (typeof target !== "object" || target === null) return target;
119
+ if (failureClassOf(target) !== undefined) return target;
120
+ try {
121
+ Object.defineProperty(target, "_tag", { value: FAILURE_CLASS_TAG, enumerable: false, configurable: true });
122
+ Object.defineProperty(target, "class", { value: cls, enumerable: false, configurable: true });
123
+ } catch {
124
+ // 冻结 / 密封的错误对象:标记不上就算了,分类退化成 attempt 档,不掩盖原始失败。
125
+ }
126
+ return target;
127
+ }
128
+
129
+ /**
130
+ * @internal 错误链(含 `cause` 链)的 message 串接:给人读的报错文案与给分类器看的失败文本
131
+ * 用这同一段,不出现「报错说 A、分类看 B」。
132
+ */
133
+ export function errorChainText(error: unknown): string {
134
+ const parts: string[] = [];
135
+ let current: unknown = error;
136
+ for (let depth = 0; depth < CAUSE_CHAIN_DEPTH && current != null; depth++) {
137
+ const message = current instanceof Error ? current.message : String(current);
138
+ if (message) parts.push(message);
139
+ current = current instanceof Error ? causeOf(current) : undefined;
140
+ }
141
+ return parts.join(" · ");
142
+ }
143
+
144
+ /**
145
+ * @internal 调分类器的统一纪律:抛错按 `undefined` 回落(自身错误被吞掉,不得用新错误掩盖
146
+ * 原始失败),返回值原样透出。turn 链与生命周期链共用这一处,两条链的纪律不会跑偏。
147
+ */
148
+ export function callClassifier<T>(classifier: ((failure: T) => FailureClass | undefined) | undefined, failure: T) {
149
+ if (!classifier) return undefined;
150
+ try {
151
+ return classifier(failure);
152
+ } catch {
153
+ return undefined;
154
+ }
155
+ }
156
+
157
+ /**
158
+ * @internal 生命周期阶段失败的分类链(三道):抛出点携带的分类 → 实验分类器 → 缺省
159
+ * `{ retryable: false }`。这些位置(sandbox 钩子、`EvalDef.setup`、`test(t)` 体内、
160
+ * per-attempt teardown)没有重试执行体,链上不挂产时间轴的兜底正则——时间轴即使给出也无人
161
+ * 消费(见 architecture.md「分类链」)。
162
+ */
163
+ export function resolveAttemptFailureClass(
164
+ info: AttemptFailureInfo,
165
+ experimentClassifier?: AttemptFailureClassifier,
166
+ ): FailureClass {
167
+ const declared = failureClassOf(info.cause);
168
+ if (declared) return declared;
169
+ return callClassifier(experimentClassifier, info) ?? { retryable: false };
170
+ }
171
+
172
+ /** @internal 从一个抛出的错误构造实验分类器的输入(文本与报错文案同源)。 */
173
+ export function attemptFailureInfo(phase: LifecyclePhase, error: unknown): AttemptFailureInfo {
174
+ return { phase, text: errorChainText(error), cause: error };
175
+ }
package/src/show/index.ts CHANGED
@@ -391,11 +391,10 @@ const MISSING_MARK = "—";
391
391
  const USAGE_COLUMNS = ["locator", "eval", "result", "turns", "tools", "uncached in", "cache read", "out", "requests", "cost"];
392
392
  const USAGE_ALIGN: readonly ColumnAlign[] = ["left", "left", "left", "right", "right", "right", "right", "right", "right", "right"];
393
393
 
394
- /** uncached in 列的取值:与 `usage:` 行同一条回退口径(见 docs/feature/reports/library/
395
- * attempt-detail.md#usagetable-组装口径单源)——优先派生的 uncachedInputTokens,两个输入
396
- * 缺一个就回退显示原始 inputTokens,不猜 0。 */
394
+ /** uncached in 列的取值:桶恒互斥,inputTokens 本身就是未缓存输入(见 docs/feature/reports/
395
+ * library/attempt-detail.md#usagetable-组装口径单源)。 */
397
396
  function uncachedInOf(row: UsageTableData): number | undefined {
398
- return row.uncachedInputTokens ?? row.usage?.inputTokens;
397
+ return row.usage?.inputTokens;
399
398
  }
400
399
 
401
400
  /**
@@ -554,7 +553,7 @@ async function renderUsageCompareSlice(
554
553
  const usageRows = await usageRowsOf(selection.attempts);
555
554
  const byConditionByEval = new Map<string, UsageTableData[]>();
556
555
  for (const row of usageRows) {
557
- const key = `${row.experimentId}${row.evalId}`;
556
+ const key = `${row.experimentId}\u0000${row.evalId}`;
558
557
  const list = byConditionByEval.get(key);
559
558
  if (list) list.push(row);
560
559
  else byConditionByEval.set(key, [row]);
@@ -567,7 +566,7 @@ async function renderUsageCompareSlice(
567
566
  const cellsText = conditions.flatMap((condition) => {
568
567
  const deltaCell = row.cells[condition];
569
568
  if (!deltaCell) return USAGE_MATRIX_METRIC_COLUMNS.map(() => MISSING_MARK);
570
- const usageCell = usageMatrixCellOf(byConditionByEval.get(`${condition}${row.key}`) ?? []);
569
+ const usageCell = usageMatrixCellOf(byConditionByEval.get(`${condition}\u0000${row.key}`) ?? []);
571
570
  return usageMatrixCellText(usageCell, deltaCell.historical);
572
571
  });
573
572
  return [row.key, ...cellsText];
@@ -4,11 +4,12 @@
4
4
  // 完整多行 message 归 attempt 详情块展开。
5
5
  // bug: memory/diagnose-tail-inline-defeats-one-line-elision.md
6
6
  //
7
- // 「execution 的预算、句柄与 grep」(docs/engineering/testing/unit/reports.md):卡片预览预算按
8
- // UTF-8 字节在字符边界截断;Agent 卡的 t<N>.c<M> 从事件序确定性派生,失败 Sandbox 命令卡的
9
- // cmd<N> 按 timing node 时序派生;--expand 命中还原完整落盘值、句柄超界的完整报错;--grep 的
10
- // 匹配面覆盖角色文本、工具名、input、result 与失败命令 display/stdout/stderr;命中计数与
11
- // 0 命中的明确输出。
7
+ // 「execution 的预算、句柄与 grep」(docs/engineering/testing/unit/reports.md):卡片预览预算按段
8
+ // 独立截断——每段最多显示前 3 行(骨架行不计入),每段另有 1 KiB 字节兜底(按字符边界回退);
9
+ // 卡尾聚合全卡各段被折的行数与字符数,没有整行被折时行数退化省略;Agent 卡的 t<N>.c<M> 从事件序
10
+ // 确定性派生,失败 Sandbox 命令卡的 cmd<N> 按 timing node 时序派生;--expand 命中还原完整落盘值、
11
+ // 句柄超界的完整报错;--grep 的匹配面覆盖角色文本、工具名、input、result 与失败命令
12
+ // display/stdout/stderr;命中计数与 0 命中的明确输出。
12
13
 
13
14
  import { describe, expect, it } from "vitest";
14
15
  import { executionText, verdictReasonLine } from "./render.ts";
@@ -164,9 +165,88 @@ describe("--execution:轮内卡片句柄 t<N>.c<M> 从事件序确定性派生",
164
165
  });
165
166
  });
166
167
 
167
- describe("--execution:卡片预览预算按 UTF-8 字节在字符边界截断", () => {
168
- function bigBodyEvidence(): AttemptEvidence {
169
- const bigText = "x".repeat(8190) + "🙂".repeat(3); // 8190 ascii(1B) + 3 emoji(4B) = 8202B > 8192B 预算
168
+ describe("--execution:单段卡按 3 行截断,骨架行不计入,尾巴 `(+N lines · M chars · …)`", () => {
169
+ function fiveLineEvidence(): AttemptEvidence {
170
+ // 5 行,前 3 行("alpha"/"bravo"/"charlie")在预算内,后 2 行("delta"/"echo")被整行折掉。
171
+ const events: StreamEvent[] = [
172
+ { type: "message", role: "user", text: "start" },
173
+ { type: "thinking", text: "alpha\nbravo\ncharlie\ndelta\necho" },
174
+ ];
175
+ return evidenceOf({ execution: buildExecutionTree(events, []), result: resultOf({ phases: twoTurnPhases() }) });
176
+ }
177
+
178
+ it("只显示前 3 行(保留原始换行),第 4/5 行整行折掉,尾巴报被折行数与字符数", () => {
179
+ const evidence = fiveLineEvidence();
180
+ const { text } = executionText(evidence, OPTS);
181
+ expect(text).toContain("alpha\n bravo\n charlie");
182
+ expect(text).not.toContain("delta");
183
+ expect(text).not.toContain("echo");
184
+ // "\ndelta\necho" 相对 "alpha\nbravo\ncharlie" 前缀被折掉的字符(含分隔行首的 \n)。
185
+ expect(text).toContain(`(+2 lines · 11 chars · niceeval show ${LOCATOR} --execution --expand t1.c2)`);
186
+ });
187
+
188
+ it("--expand 还原完整未截断内容,不带尾巴", () => {
189
+ const evidence = fiveLineEvidence();
190
+ const { text } = executionText(evidence, OPTS, { expand: "t1.c2" });
191
+ // --expand 走 renderExpand,不经 renderFull 的额外一层 turn 缩进,只剩 renderCardLines 自身
192
+ // 的基础缩进(2 空格),比预览路径(4 空格)少一层。
193
+ expect(text).toContain("alpha\n bravo\n charlie\n delta\n echo");
194
+ expect(text).not.toContain("niceeval show");
195
+ });
196
+ });
197
+
198
+ describe("--execution:TOOL 卡 input/result 各自独立按 3 行截断,尾巴聚合两段", () => {
199
+ function toolCardEvidence(): AttemptEvidence {
200
+ const events: StreamEvent[] = [
201
+ { type: "message", role: "user", text: "start" },
202
+ { type: "action.called", callId: "call-1", name: "shell", input: { command: "cmd1\ncmd2\ncmd3\ncmd4" } },
203
+ {
204
+ type: "action.result",
205
+ callId: "call-1",
206
+ output: { output: "out1\nout2\nout3\nout4\nout5", exit_code: 0 },
207
+ status: "completed",
208
+ },
209
+ ];
210
+ return evidenceOf({ execution: buildExecutionTree(events, []), result: resultOf({ phases: twoTurnPhases() }) });
211
+ }
212
+
213
+ it("input(4 行折 1 行)与 result(5 行折 2 行)各自独立截断,骨架行`input`/`result · <status>`原样全显", () => {
214
+ const evidence = toolCardEvidence();
215
+ const { text } = executionText(evidence, OPTS);
216
+ expect(text).toContain("input");
217
+ expect(text).toContain("result · completed · exit 0");
218
+ expect(text).toContain("cmd1");
219
+ expect(text).toContain("cmd2");
220
+ expect(text).toContain("cmd3");
221
+ expect(text).not.toContain("cmd4");
222
+ expect(text).toContain("out1");
223
+ expect(text).toContain("out2");
224
+ expect(text).toContain("out3");
225
+ expect(text).not.toContain("out4");
226
+ expect(text).not.toContain("out5");
227
+ });
228
+
229
+ it("卡尾只有一条,聚合两段各自被折的行数(1+2=3)与字符数(5+10=15)", () => {
230
+ const evidence = toolCardEvidence();
231
+ const { text } = executionText(evidence, OPTS);
232
+ expect(text).toContain(`(+3 lines · 15 chars · niceeval show ${LOCATOR} --execution --expand t1.c2)`);
233
+ // 每卡只出现一次尾巴,不是 input/result 各带一条。
234
+ expect(text.match(/niceeval show .* --expand/g)?.length).toBe(1);
235
+ });
236
+
237
+ it("--expand 还原两段完整内容,不截断", () => {
238
+ const evidence = toolCardEvidence();
239
+ const { text } = executionText(evidence, OPTS, { expand: "t1.c2" });
240
+ expect(text).toContain("cmd4");
241
+ expect(text).toContain("out5");
242
+ expect(text).not.toContain("niceeval show");
243
+ });
244
+ });
245
+
246
+ describe("--execution:1 KiB 字节兜底(按字符边界回退,不切分代理对),没有整行被折时 N 退化省略", () => {
247
+ function bigSingleLineEvidence(): AttemptEvidence {
248
+ // 单行(不触发 3 行上限)但 1022 ascii(1B) + 3 emoji(4B) = 1034B > 1024B 段预算。
249
+ const bigText = "x".repeat(1022) + "🙂".repeat(3);
170
250
  const events: StreamEvent[] = [
171
251
  { type: "message", role: "user", text: "start" },
172
252
  { type: "thinking", text: bigText },
@@ -174,28 +254,49 @@ describe("--execution:卡片预览预算按 UTF-8 字节在字符边界截断",
174
254
  return evidenceOf({ execution: buildExecutionTree(events, []), result: resultOf({ phases: twoTurnPhases() }) });
175
255
  }
176
256
 
177
- it("超预算截到恰好塞满 8192 字节的 codepoint 边界,不切分代理对/多字节字符", () => {
178
- const evidence = bigBodyEvidence();
257
+ it("超预算截到恰好塞满 1024 字节的 codepoint 边界,没有整行被折,尾巴退化为 `(+M chars · …)`(没有 `lines` 段)", () => {
258
+ const evidence = bigSingleLineEvidence();
179
259
  const { text } = executionText(evidence, OPTS);
180
- // 8190 个 ascii "x" 用尽预算内的 8190 字节,下一枚 emoji(4B)放不进剩余 2 字节,整枚被折掉。
260
+ // 1022 个 ascii "x" 用尽预算内的 1022 字节,下一枚 emoji(4B)放不进剩余 2 字节,整枚被折掉。
181
261
  // 用最长连续 x 串而不是全文 x 计数——尾巴里的 "--execution"/"--expand" 与 header 的 "exp/a"
182
262
  // 本身也含 'x' 字母,全文计数会被这些无关 'x' 污染。
183
263
  const longestXRun = Math.max(0, ...[...text.matchAll(/x+/g)].map((m) => m[0].length));
184
- expect(longestXRun).toBe(8190);
264
+ expect(longestXRun).toBe(1022);
185
265
  expect(text).not.toContain("🙂");
186
266
  expect(text).toContain(`(+3 chars · niceeval show ${LOCATOR} --execution --expand t1.c2)`);
267
+ expect(text).not.toContain("lines ·");
187
268
  });
188
269
 
189
270
  it("--expand 还原完整未截断内容(原始换行,不再截断)", () => {
190
- const evidence = bigBodyEvidence();
271
+ const evidence = bigSingleLineEvidence();
191
272
  const { text } = executionText(evidence, OPTS, { expand: "t1.c2" });
192
273
  expect(text).toContain("🙂🙂🙂");
193
274
  expect(text).not.toContain("niceeval show");
194
275
  const longestXRun = Math.max(0, ...[...text.matchAll(/x+/g)].map((m) => m[0].length));
195
- expect(longestXRun).toBe(8190);
276
+ expect(longestXRun).toBe(1022);
277
+ });
278
+
279
+ it("被字节兜底截到一半的行,如果全卡还有整行被折,计入被折行数(不退化)", () => {
280
+ // 4 行:前 2 行短,第 3 行(2000 个 'x')把候选(3 行)的字节数撑到 1024 以上触发字节兜底,
281
+ // 第 4 行("hiddenLine")完全在 3 行上限之外被整行折掉——两种折法同一张卡都发生。
282
+ const events: StreamEvent[] = [
283
+ { type: "message", role: "user", text: "start" },
284
+ { type: "thinking", text: `short1\nshort2\n${"x".repeat(2000)}\nhiddenLine` },
285
+ ];
286
+ const evidence = evidenceOf({ execution: buildExecutionTree(events, []), result: resultOf({ phases: twoTurnPhases() }) });
287
+ const { text } = executionText(evidence, OPTS);
288
+ expect(text).toContain("short1");
289
+ expect(text).toContain("short2");
290
+ expect(text).not.toContain("hiddenLine");
291
+ const longestXRun = Math.max(0, ...[...text.matchAll(/x+/g)].map((m) => m[0].length));
292
+ expect(longestXRun).toBe(1010); // 1024 字节预算 - "short1\nshort2\n"(14 字节) = 1010 个 x
293
+ // N = 1(hiddenLine 整行被折) + 1(第 3 行被字节兜底截到一半,全卡有整行被折时计入) = 2。
294
+ expect(text).toContain(`(+2 lines · 1001 chars · niceeval show ${LOCATOR} --execution --expand t1.c2)`);
196
295
  });
296
+ });
197
297
 
198
- it("落盘已截断的值(marker 是写入时刻烘进正文的字面文本)在预览与 --expand 中都原样透传", () => {
298
+ describe("--execution:落盘已截断的值原样透传", () => {
299
+ it("marker 是写入时刻烘进正文的字面文本,在预览与 --expand 中都原样透传(短文本,预算内不截断)", () => {
199
300
  const diskTruncated = "partial output before cut\n[niceeval] truncated 51467156 → 262144 bytes";
200
301
  const events: StreamEvent[] = [
201
302
  { type: "message", role: "user", text: "start" },
@@ -393,14 +393,16 @@ function relSeconds(ms: number, originMs: number): string {
393
393
  }
394
394
 
395
395
  /** 卡片正文的有界预览预算(docs/feature/reports/show/execution.md「卡片预览预算与 --expand」):
396
- * 8 KiB,按 UTF-8 字节计,超限时按字符边界(codepoint,不切分代理对)回退截断。 */
397
- const CARD_BODY_BUDGET_BYTES = 8 * 1024;
396
+ * 主尺度是行——每个内容段最多显示前 3 行(保留原始换行);每段另有 1 KiB(UTF-8 字节)兜底,
397
+ * 防单行超长的 JSON blob 击穿行预算。 */
398
+ const CARD_SEGMENT_MAX_LINES = 3;
399
+ const CARD_SEGMENT_BUDGET_BYTES = 1024;
398
400
 
399
401
  /**
400
402
  * 按 UTF-8 字节预算截断,永不切断一个 codepoint。快速路径:总字节数不超预算时不做任何逐字符
401
403
  * 扫描。慢速路径只在确实超限时才逐 codepoint 累加字节数,找到恰好塞满预算的前缀。
402
404
  */
403
- function truncateCardBody(text: string, maxBytes: number): { shown: string; foldedChars: number } {
405
+ function truncateByteBudget(text: string, maxBytes: number): { shown: string; foldedChars: number } {
404
406
  if (Buffer.byteLength(text, "utf8") <= maxBytes) return { shown: text, foldedChars: 0 };
405
407
  const chars = Array.from(text);
406
408
  let bytes = 0;
@@ -413,6 +415,55 @@ function truncateCardBody(text: string, maxBytes: number): { shown: string; fold
413
415
  return { shown: chars.slice(0, i).join(""), foldedChars: chars.length - i };
414
416
  }
415
417
 
418
+ /** 一段内容截断后的状态:`shown` 是保留的前缀,`linesFullyHidden` 是被 3 行上限整行折掉的行数
419
+ * (不含被字节兜底截到一半的那一行),`partialCut` 标记这段是否被字节兜底从中间截断过,
420
+ * `foldedChars` 是这段被折掉的字符总数(= 原文 codepoint 数 − `shown` 的 codepoint 数)。 */
421
+ interface SegmentTruncation {
422
+ shown: string;
423
+ linesFullyHidden: number;
424
+ partialCut: boolean;
425
+ foldedChars: number;
426
+ }
427
+
428
+ /**
429
+ * 单段的行 + 字节双重预算截断(docs/feature/reports/show/execution.md「卡片预览预算与
430
+ * --expand」):先按 3 行裁剪(保留原始换行),再对裁剪结果套 1 KiB 字节兜底(按字符边界回退,
431
+ * 不切分代理对),防单行超长的 JSON blob 击穿行预算。两次裁剪都只从末尾裁,`shown` 因此恒是
432
+ * 原文的一个前缀,折字符数可以直接用两侧 codepoint 数之差倒推。
433
+ */
434
+ function truncateSegment(text: string): SegmentTruncation {
435
+ const lines = text.split("\n");
436
+ const shownLineCount = Math.min(CARD_SEGMENT_MAX_LINES, lines.length);
437
+ const candidate = lines.slice(0, shownLineCount).join("\n");
438
+ const { shown, foldedChars: byteFoldedChars } = truncateByteBudget(candidate, CARD_SEGMENT_BUDGET_BYTES);
439
+ const totalChars = Array.from(text).length;
440
+ const shownChars = Array.from(shown).length;
441
+ return {
442
+ shown,
443
+ linesFullyHidden: lines.length - shownLineCount,
444
+ partialCut: byteFoldedChars > 0,
445
+ foldedChars: totalChars - shownChars,
446
+ };
447
+ }
448
+
449
+ /**
450
+ * 卡尾截断提示的聚合(docs/feature/reports/show/execution.md「卡片预览预算与 --expand」):
451
+ * N 是全卡各段被整行折掉的行数之和,再加上「同一张卡里确实有整行被折」时,每个被字节兜底截到
452
+ * 一半的段各计 1 行——这条规则只在 N 本来就 > 0 时生效;如果全卡没有任何一段整行被折、只是
453
+ * 某段字节兜底切了字符(单段本身没超 3 行,但这一两行太长),N 退化为 0,尾巴退化成
454
+ * `(+M chars · …)`,不虚报一行「被折」。返回 undefined 表示这张卡没有任何段被截断,调用方不
455
+ * 应追加尾巴。
456
+ */
457
+ function foldCardTail(bits: readonly SegmentTruncation[], locator: string, handle: string): string | undefined {
458
+ const totalFoldedChars = bits.reduce((sum, b) => sum + b.foldedChars, 0);
459
+ if (totalFoldedChars === 0) return undefined;
460
+ const totalHiddenLines = bits.reduce((sum, b) => sum + b.linesFullyHidden, 0);
461
+ const partialCutCount = bits.reduce((sum, b) => sum + (b.partialCut ? 1 : 0), 0);
462
+ const n = totalHiddenLines > 0 ? totalHiddenLines + partialCutCount : 0;
463
+ const head = n > 0 ? `+${n} line${n === 1 ? "" : "s"} · ${totalFoldedChars} chars` : `+${totalFoldedChars} chars`;
464
+ return `(${head} · niceeval show ${locator} --execution --expand ${handle})`;
465
+ }
466
+
416
467
  /** 逐行加前缀,保留原始换行(不做 wrapDisplay 折行——「保留原始换行」是这个区块 text 面的契约)。
417
468
  * 空行不补前缀,避免制造纯空白的尾随行。 */
418
469
  function indentLines(text: string, indent: string): string {
@@ -522,38 +573,49 @@ function nodeMeta(node: ExecutionNode, originMs: number): string {
522
573
  return `${relSeconds(span.startMs, originMs)} · ${formatDurationMs(span.endMs - span.startMs)}`;
523
574
  }
524
575
 
576
+ /** 卡片正文的一个内容段(docs/feature/reports/show/execution.md「卡片预览预算与 --expand」):
577
+ * 预览预算(3 行 + 1 KiB 兜底)按段独立截断,不是对整卡正文一次性截断。 */
578
+ interface CardSegment {
579
+ /** 段的骨架行(如 `input` / `result · completed`);不计入预算、不截断。undefined 表示这段
580
+ * 没有骨架行——单段卡的正文,或失败命令卡的命令行本身。 */
581
+ label?: string;
582
+ /** 这段的完整(未截断)原始内容,保留原始换行;预算截断与 --expand 的完整输出都基于它。 */
583
+ text: string;
584
+ }
585
+
525
586
  interface CardParts {
526
587
  /** 卡片标题行(不缩进,含 kind 标签与相对时间/耗时 meta)。 */
527
588
  header: string;
528
- /** 卡片正文的完整(未截断)原始内容,保留原始换行;预算截断与 --expand 的完整输出都基于它。 */
529
- body: string;
589
+ /** 卡片正文按结构划分的段落;角色文本/thinking 这类单段卡只有 1 段,TOOL 卡 input/result
590
+ * 各一段,失败命令卡命令行/stdout/stderr 各一段。 */
591
+ segments: CardSegment[];
530
592
  /** --grep 的匹配面:角色文本、工具名、input、result(docs/feature/reports/show/execution.md
531
593
  * 「范围化:跨 attempt 扫描与 --grep」);未经截断,grep 命中不受预览预算影响。 */
532
594
  matchText: string;
533
- /** 截断尾巴 `(+N chars · …)` 相对卡片正文的额外缩进——多段结构(如 TOOL 的 input/result)
534
- * 的正文本身已经带一层内嵌缩进,尾巴要落在同一层,不是贴着卡片左边。 */
595
+ /** 截断尾巴相对卡片正文的额外缩进——多段结构(如 TOOL 的 input/result)的正文本身已经带
596
+ * 一层内嵌缩进,尾巴要落在同一层,不是贴着卡片左边。 */
535
597
  tailIndent: string;
536
598
  }
537
599
 
538
- /** 一个 Agent 事件节点 → 卡片的标题/正文/匹配面。`node.kind` 已排除 telemetry(见调用方的
600
+ /** 一个 Agent 事件节点 → 卡片的标题/段落/匹配面。`node.kind` 已排除 telemetry(见调用方的
539
601
  * agentNodes 过滤),telemetry 分支仅为联合类型穷尽性存在,不会被触达。 */
540
602
  function agentCardParts(node: ExecutionNode, originMs: number): CardParts {
541
603
  const meta = nodeMeta(node, originMs);
542
604
  const withMeta = (title: string) => (meta ? `${title} ${meta}` : title);
543
605
  switch (node.kind) {
544
606
  case "message":
545
- return { header: withMeta(node.role.toUpperCase()), body: node.text, matchText: node.text, tailIndent: "" };
607
+ return { header: withMeta(node.role.toUpperCase()), segments: [{ text: node.text }], matchText: node.text, tailIndent: "" };
546
608
  case "thinking":
547
- return { header: withMeta("THINKING"), body: node.text, matchText: node.text, tailIndent: "" };
609
+ return { header: withMeta("THINKING"), segments: [{ text: node.text }], matchText: node.text, tailIndent: "" };
548
610
  case "context.injected":
549
611
  return {
550
612
  header: withMeta(node.source ? `CONTEXT INJECTED · ${node.source}` : "CONTEXT INJECTED"),
551
- body: node.text,
613
+ segments: [{ text: node.text }],
552
614
  matchText: node.text,
553
615
  tailIndent: "",
554
616
  };
555
617
  case "skill.loaded":
556
- return { header: withMeta(`SKILL · ${node.skill}`), body: "loaded", matchText: node.skill, tailIndent: "" };
618
+ return { header: withMeta(`SKILL · ${node.skill}`), segments: [{ text: "loaded" }], matchText: node.skill, tailIndent: "" };
557
619
  case "action": {
558
620
  const input = recordOf(node.input);
559
621
  const output = recordOf(node.output);
@@ -564,36 +626,43 @@ function agentCardParts(node: ExecutionNode, originMs: number): CardParts {
564
626
  ? jsonText(node.output)
565
627
  : "(no result)";
566
628
  const exit = typeof output?.exit_code === "number" ? ` · exit ${output.exit_code}` : "";
567
- const body = ["input", indentLines(inputText, " "), `result · ${node.status}${exit}`, indentLines(outputText, " ")].join("\n");
568
- return { header: withMeta(`TOOL · ${node.name}`), body, matchText: `${node.name} ${inputText} ${outputText}`, tailIndent: " " };
629
+ return {
630
+ header: withMeta(`TOOL · ${node.name}`),
631
+ segments: [
632
+ { label: "input", text: inputText },
633
+ { label: `result · ${node.status}${exit}`, text: outputText },
634
+ ],
635
+ matchText: `${node.name} ${inputText} ${outputText}`,
636
+ tailIndent: " ",
637
+ };
569
638
  }
570
639
  case "subagent": {
571
640
  const resultText = node.output === undefined ? node.status : `result · ${node.status}: ${jsonText(node.output)}`;
572
- return { header: withMeta(`SUBAGENT · ${node.name}`), body: resultText, matchText: `${node.name} ${resultText}`, tailIndent: "" };
641
+ return { header: withMeta(`SUBAGENT · ${node.name}`), segments: [{ text: resultText }], matchText: `${node.name} ${resultText}`, tailIndent: "" };
573
642
  }
574
643
  case "input.requested": {
575
644
  const text = node.request.prompt ?? node.request.display ?? "(input requested)";
576
645
  const matchText = [text, node.request.action, node.request.input !== undefined ? jsonText(node.request.input) : undefined]
577
646
  .filter((s): s is string => s !== undefined)
578
647
  .join(" ");
579
- return { header: withMeta("INPUT REQUESTED"), body: text, matchText, tailIndent: "" };
648
+ return { header: withMeta("INPUT REQUESTED"), segments: [{ text }], matchText, tailIndent: "" };
580
649
  }
581
650
  case "compaction":
582
- return { header: withMeta("COMPACTION"), body: node.reason ?? "context compacted", matchText: node.reason ?? "", tailIndent: "" };
651
+ return { header: withMeta("COMPACTION"), segments: [{ text: node.reason ?? "context compacted" }], matchText: node.reason ?? "", tailIndent: "" };
583
652
  case "error":
584
- return { header: withMeta("ERROR"), body: node.message, matchText: node.message, tailIndent: "" };
653
+ return { header: withMeta("ERROR"), segments: [{ text: node.message }], matchText: node.message, tailIndent: "" };
585
654
  case "telemetry":
586
- return { header: "TELEMETRY", body: "", matchText: "", tailIndent: "" };
655
+ return { header: "TELEMETRY", segments: [], matchText: "", tailIndent: "" };
587
656
  }
588
657
  }
589
658
 
590
659
  function commandCardParts(command: FailedCommandEvidence): CardParts {
591
- const sections = [command.display];
592
- if (command.stdout) sections.push("stdout", indentLines(command.stdout, " "));
593
- if (command.stderr) sections.push("stderr", indentLines(command.stderr, " "));
660
+ const segments: CardSegment[] = [{ text: command.display }];
661
+ if (command.stdout) segments.push({ label: "stdout", text: command.stdout });
662
+ if (command.stderr) segments.push({ label: "stderr", text: command.stderr });
594
663
  return {
595
664
  header: "",
596
- body: sections.join("\n"),
665
+ segments,
597
666
  matchText: `${command.display} ${command.stdout} ${command.stderr}`,
598
667
  tailIndent: " ",
599
668
  };
@@ -629,19 +698,37 @@ function turnUsageText(turn: TimingNode | undefined): string | undefined {
629
698
  return parts.length > 0 ? parts.join(" · ") : undefined;
630
699
  }
631
700
 
632
- /** 一张卡片的完整渲染:`full` 时输出未截断的落盘内容(--expand);否则按 8 KiB 预算截断,
633
- * 截断尾巴带被折字符数与展开句柄。两种形态都只做逐行前缀(indentLines),不做 wrapDisplay 折行
634
- * ——保留原始换行是这个区块 text 面的契约,不是遗漏。 */
701
+ /**
702
+ * 一张卡片的完整渲染:`full` 时逐段输出未截断的落盘内容(--expand);否则每段独立按 3 行 + 1 KiB
703
+ * 预算截断(docs/feature/reports/show/execution.md「卡片预览预算与 --expand」),卡尾追加一条
704
+ * 聚合尾巴。带 `label` 的段(TOOL 的 input/result、失败命令的 stdout/stderr)先输出骨架行,
705
+ * 段正文本身再多缩进一层;没有 label 的段(单段卡的正文,命令行本身)正文与骨架行同一层。
706
+ * 两种形态都只做逐行前缀(indentLines),不做 wrapDisplay 折行——保留原始换行是这个区块 text
707
+ * 面的契约,不是遗漏。
708
+ */
635
709
  function renderCardLines(parts: CardParts, handle: string, locator: string, full: boolean): string[] {
636
- if (full) {
637
- const body = indentLines(parts.body, " ");
638
- return parts.header ? [parts.header, ...body.split("\n")] : body.split("\n");
710
+ const bodyLines: string[] = [];
711
+ const truncations: SegmentTruncation[] = [];
712
+ for (const seg of parts.segments) {
713
+ let content: string;
714
+ if (full) {
715
+ content = seg.text;
716
+ } else {
717
+ const t = truncateSegment(seg.text);
718
+ truncations.push(t);
719
+ content = t.shown;
720
+ }
721
+ if (seg.label) {
722
+ bodyLines.push(seg.label, ...indentLines(content, " ").split("\n"));
723
+ } else {
724
+ bodyLines.push(...content.split("\n"));
725
+ }
726
+ }
727
+ if (!full) {
728
+ const tail = foldCardTail(truncations, locator, handle);
729
+ if (tail) bodyLines.push(`${parts.tailIndent}${tail}`);
639
730
  }
640
- const { shown, foldedChars } = truncateCardBody(parts.body, CARD_BODY_BUDGET_BYTES);
641
- const withTail = foldedChars > 0
642
- ? `${shown}\n${parts.tailIndent}(+${foldedChars} chars · niceeval show ${locator} --execution --expand ${handle})`
643
- : shown;
644
- const body = indentLines(withTail, " ");
731
+ const body = indentLines(bodyLines.join("\n"), " ");
645
732
  return parts.header ? [parts.header, ...body.split("\n")] : body.split("\n");
646
733
  }
647
734
 
@@ -814,7 +901,9 @@ export interface ExecutionRenderOptions {
814
901
  * (`cmd<N>`,来自 `commands.json`,经 `evidence.commands` 读取;没有失败命令时这一段自然
815
902
  * 零输出,见 `failedCommandsOf`)。
816
903
  *
817
- * 卡片正文是 8 KiB(UTF-8 字节,按字符边界回退)的有界预览,截断尾巴带被折字符数与展开句柄;
904
+ * 卡片正文按段(单段卡的正文、TOOL 卡的 input/result、失败命令卡的命令行/stdout/stderr)分别是
905
+ * 3 行(保留原始换行)的有界预览,单段另有 1 KiB 字节兜底(按字符边界回退);截断尾巴带被折
906
+ * 行数与字符数、以及展开句柄(全卡没有整行被折、只是字节兜底切了字符时,行数退化为省略)。
818
907
  * `options.expand` 精确定位一张卡片输出完整落盘内容;`options.grep` 只输出匹配面(角色文本/
819
908
  * 工具名/input/result,命令卡另加 display/stdout/stderr)命中的卡片,返回值的 `matches` 是本
820
909
  * attempt 内的命中卡片数——「N matches in M attempts」与 0 命中的措辞归调用方组装。
package/src/types.ts CHANGED
@@ -20,3 +20,12 @@ export * from "./agents/types.ts";
20
20
  export * from "./scoring/types.ts";
21
21
  export * from "./context/types.ts";
22
22
  export * from "./runner/types.ts";
23
+
24
+ // 失败分类词表:两轴 FailureClass / FailureScope 与实验分类器的输入输出形状。糖衣类与守卫
25
+ // 是运行时值,从包根(src/index.ts)直接导出,不进这个纯类型 facade。
26
+ export type {
27
+ FailureClass,
28
+ FailureScope,
29
+ AttemptFailureInfo,
30
+ AttemptFailureClassifier,
31
+ } from "./shared/failure-class.ts";