niceeval 0.10.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +5 -1
  2. package/dist/report/components/attempt-detail/AttemptConversation.js +42 -2
  3. package/dist/report/components/attempt-detail/AttemptSource.js +83 -13
  4. package/dist/report/components/attempt-detail/compute.js +44 -1
  5. package/dist/report/components/attempt-detail/faces.js +4 -0
  6. package/dist/report/components/attempt-detail/index.d.ts +2 -2
  7. package/dist/report/components/attempt-detail/index.js +24 -4
  8. package/dist/report/components/entity-lists/ExperimentList.d.ts +1 -2
  9. package/dist/report/components/entity-lists/ExperimentList.js +6 -5
  10. package/dist/report/components/entity-lists/faces.d.ts +1 -1
  11. package/dist/report/components/entity-lists/faces.js +10 -9
  12. package/dist/report/components/entity-lists/index.d.ts +5 -7
  13. package/dist/report/components/entity-lists/index.js +7 -3
  14. package/dist/report/components/metric-views/MetricScatter.js +2 -38
  15. package/dist/report/index.d.ts +1 -1
  16. package/dist/report/model/format.d.ts +5 -5
  17. package/dist/report/model/format.js +34 -8
  18. package/dist/report/model/types.d.ts +16 -2
  19. package/dist/report/react/index.d.ts +1 -1
  20. package/dist/runner/types.d.ts +2 -1
  21. package/dist/sandbox/e2b.d.ts +0 -1
  22. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +5 -5
  23. package/docs-site/zh/explanation/runner.mdx +6 -2
  24. package/docs-site/zh/reference/cli.mdx +2 -2
  25. package/docs-site/zh/reference/report-components.mdx +3 -1
  26. package/docs-site/zh/tutorials/experiments.mdx +1 -2
  27. package/docs-site/zh/tutorials/viewing-results.mdx +12 -9
  28. package/docs-site/zh/tutorials/write-experiment.mdx +1 -3
  29. package/package.json +3 -4
  30. package/src/cli.ts +4 -2
  31. package/src/o11y/prices.json +176 -99
  32. package/src/report/assets/styles.css +822 -0
  33. package/src/report/components/attempt-detail/AttemptConversation.tsx +55 -7
  34. package/src/report/components/attempt-detail/AttemptSource.tsx +186 -42
  35. package/src/report/components/attempt-detail/attempt-components.test.tsx +82 -4
  36. package/src/report/components/attempt-detail/compute.ts +50 -1
  37. package/src/report/components/attempt-detail/faces.ts +3 -0
  38. package/src/report/components/attempt-detail/index.tsx +33 -16
  39. package/src/report/components/attempt-detail/validate.test.ts +19 -2
  40. package/src/report/components/entity-lists/ExperimentList.tsx +13 -10
  41. package/src/report/components/entity-lists/faces.ts +10 -9
  42. package/src/report/components/entity-lists/index.tsx +7 -10
  43. package/src/report/components/metric-views/MetricScatter.tsx +2 -38
  44. package/src/report/components/render.test.tsx +19 -9
  45. package/src/report/index.ts +2 -0
  46. package/src/report/model/format.ts +33 -8
  47. package/src/report/model/types.ts +18 -2
  48. package/src/report/react/index.tsx +2 -0
  49. package/src/report/runtime/dual-render.test.tsx +43 -10
  50. package/src/runner/types.ts +2 -1
  51. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  52. package/src/sandbox/e2b.ts +38 -2
  53. package/src/view/view-report.test.ts +3 -1
@@ -3,8 +3,40 @@
3
3
 
4
4
  import type { ReactElement, ReactNode } from "react";
5
5
  import type { AttemptConversationData, AttemptConversationReply, AttemptConversationRound } from "../../model/types.ts";
6
+ import type { JsonValue, ToolName } from "../../../types.ts";
6
7
  import { cx } from "../shared.ts";
7
8
 
9
+ const TOOL_VERB: Partial<Record<ToolName, string>> = {
10
+ shell: "Bash",
11
+ file_read: "Read",
12
+ file_write: "Write",
13
+ file_edit: "Edit",
14
+ web_fetch: "Fetch",
15
+ web_search: "Search",
16
+ glob: "Glob",
17
+ grep: "Grep",
18
+ list_dir: "List",
19
+ agent_task: "Task",
20
+ };
21
+
22
+ function compact(value: JsonValue | undefined, max = 140): string {
23
+ if (value === undefined || value === null) return "";
24
+ const text = typeof value === "string" ? value : JSON.stringify(value);
25
+ const oneLine = text.replace(/\s+/g, " ").trim();
26
+ return oneLine.length > max ? `${oneLine.slice(0, max)}…` : oneLine;
27
+ }
28
+
29
+ function toolPrimaryArg(input: JsonValue): string {
30
+ if (typeof input === "string") return compact(input, 80);
31
+ if (input === null || Array.isArray(input) || typeof input !== "object") return "";
32
+ for (const key of ["command", "cmd", "path", "file", "file_path", "pattern", "query", "url", "prompt", "description"]) {
33
+ const value = input[key];
34
+ if (typeof value === "string" && value) return compact(value, 80);
35
+ if (key === "command" && Array.isArray(value)) return compact(value.filter((item) => typeof item === "string").join(" "), 80);
36
+ }
37
+ return compact(input, 80);
38
+ }
39
+
8
40
  function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
9
41
  switch (reply.kind) {
10
42
  case "assistant":
@@ -37,11 +69,20 @@ function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
37
69
  </div>
38
70
  );
39
71
  case "tool":
72
+ const verb = (reply.tool ? TOOL_VERB[reply.tool] : undefined) ?? reply.name;
73
+ const arg = toolPrimaryArg(reply.input);
74
+ const preview = compact(reply.output);
40
75
  return (
41
76
  <details className="nre-conv-tool">
42
77
  <summary>
43
- {reply.name}
44
- {reply.status ? ` · ${reply.status}` : ""}
78
+ <span className={cx("nre-conv-tool-dot", reply.status ? `nre-conv-tool-${reply.status}` : "nre-conv-tool-pending")} />
79
+ <span className="nre-conv-tool-name" title={arg ? `${verb}(${arg})` : verb}>
80
+ {arg ? `${verb}(${arg})` : verb}
81
+ </span>
82
+ <span className="nre-conv-tool-preview">
83
+ {reply.status ?? "pending"}
84
+ {preview ? ` · ${preview}` : ""}
85
+ </span>
45
86
  </summary>
46
87
  <pre className="nre-conv-tool-io">{JSON.stringify(reply.input, null, 2)}</pre>
47
88
  {reply.output !== undefined ? <pre className="nre-conv-tool-io">{JSON.stringify(reply.output, null, 2)}</pre> : null}
@@ -71,6 +112,17 @@ function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
71
112
  }
72
113
  }
73
114
 
115
+ /** AttemptSource 复用同一份回复 renderer,把一轮执行挂回对应的 send 源码行。 */
116
+ export function ConversationReplies({ replies }: { replies: AttemptConversationReply[] }): ReactElement {
117
+ return (
118
+ <div className="nre-conv-replies">
119
+ {replies.map((reply, i) => (
120
+ <ReplyRow key={i} reply={reply} />
121
+ ))}
122
+ </div>
123
+ );
124
+ }
125
+
74
126
  function RoundCard({ round, index }: { round: AttemptConversationRound; index: number }): ReactElement {
75
127
  return (
76
128
  <div className="nre-conv-round">
@@ -83,11 +135,7 @@ function RoundCard({ round, index }: { round: AttemptConversationRound; index: n
83
135
  ) : null}
84
136
  </div>
85
137
  {round.sentText ? <div className="nre-conv-sent">{round.sentText}</div> : null}
86
- <div className="nre-conv-replies">
87
- {round.replies.map((reply, i) => (
88
- <ReplyRow key={i} reply={reply} />
89
- ))}
90
- </div>
138
+ <ConversationReplies replies={round.replies} />
91
139
  </div>
92
140
  );
93
141
  }
@@ -1,12 +1,38 @@
1
- // AttemptSource:带 send / assertion 标注的 eval 源码;行内展开 assertion 细节。
2
- // 没有 source 时零输出(docs/feature/reports/library/attempt-detail.md)。send 标注只是
3
- // 指向 AttemptConversation 完整回复的指针(status/耗时),不复制轮内内容——区块按事实
4
- // 边界拆分,同一批事实不在两个组件里各展一份。
1
+ // AttemptSource:GitHub diff 式带标注 eval 源码。send / assertion 行按状态着色,点击
2
+ // 原生 details 在调用点展开完整回复与 assertion 细节。没有 source 时零输出
3
+ // (docs/feature/reports/library/attempt-detail.md)
5
4
 
6
- import type { ReactElement } from "react";
7
- import type { AttemptSourceData } from "../../model/types.ts";
5
+ import type { ReactElement, ReactNode } from "react";
6
+ import type { AttemptSourceData, AttemptSourceLineData, AttemptSourceTurn } from "../../model/types.ts";
8
7
  import type { AssertionResult } from "../../../types.ts";
9
8
  import { cx } from "../shared.ts";
9
+ import { ConversationReplies } from "./AttemptConversation.tsx";
10
+
11
+ const TS_HL_RE =
12
+ /(\/\/[^\n]*)|(\/\*[^]*?\*\/)|(`(?:\\.|[^`\\])*`|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|\b(import|from|export|default|const|let|var|async|await|function|return|if|else|for|of|in|new|class|extends|typeof|void|true|false|null|undefined)\b|\b(\d[\d_.]*)\b|([A-Za-z_$][\w$]*)(?=\s*\()/g;
13
+
14
+ /** 逐行零依赖 TS 高亮;token class 是稳定的 web 展示语义,不改源码文本。 */
15
+ function highlightTs(line: string): ReactNode[] {
16
+ const out: ReactNode[] = [];
17
+ let last = 0;
18
+ let i = 0;
19
+ let match: RegExpExecArray | null;
20
+ TS_HL_RE.lastIndex = 0;
21
+ while ((match = TS_HL_RE.exec(line))) {
22
+ if (match.index > last) out.push(line.slice(last, match.index));
23
+ const tokenClass =
24
+ match[1] || match[2] ? "tok-comment" : match[3] ? "tok-str" : match[4] ? "tok-kw" : match[5] ? "tok-num" : "tok-fn";
25
+ out.push(
26
+ <span key={i++} className={tokenClass}>
27
+ {match[0]}
28
+ </span>,
29
+ );
30
+ last = match.index + match[0].length;
31
+ if (match[0].length === 0) TS_HL_RE.lastIndex++;
32
+ }
33
+ if (last < line.length) out.push(line.slice(last));
34
+ return out;
35
+ }
10
36
 
11
37
  function assertTone(a: AssertionResult): "good" | "warn" | "bad" | "na" {
12
38
  if (a.outcome === "unavailable") return "na";
@@ -14,52 +40,162 @@ function assertTone(a: AssertionResult): "good" | "warn" | "bad" | "na" {
14
40
  return a.severity === "soft" ? "warn" : "bad";
15
41
  }
16
42
 
43
+ function lineTone(line: AttemptSourceLineData): "good" | "warn" | "bad" | "na" | undefined {
44
+ if (line.assertions.length === 0) return undefined;
45
+ if (line.assertions.some((a) => assertTone(a) === "bad")) return "bad";
46
+ if (line.assertions.some((a) => assertTone(a) === "warn")) return "warn";
47
+ if (line.assertions.some((a) => assertTone(a) === "na")) return "na";
48
+ return "good";
49
+ }
50
+
51
+ function formatDuration(ms: number): string {
52
+ return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`;
53
+ }
54
+
55
+ function thresholdScores(assertions: AssertionResult[]): string[] {
56
+ const scores: string[] = [];
57
+ for (const assertion of assertions) {
58
+ if (assertion.outcome !== "unavailable" && assertion.threshold !== undefined) {
59
+ scores.push(`${assertion.score}/${assertion.threshold}`);
60
+ }
61
+ }
62
+ return scores;
63
+ }
64
+
65
+ /** 行号位标记(与产品站示例卡同语言:图标顶替行号,不加独立状态列);内联 SVG,零图标依赖。 */
66
+ const MARK_ICONS: Record<"send" | "good" | "bad" | "warn" | "na", ReactElement> = {
67
+ send: <path d="M21 15a4 4 0 0 1-4 4H8l-5 3V7a4 4 0 0 1 4-4h10a4 4 0 0 1 4 4z" />,
68
+ good: (
69
+ <>
70
+ <circle cx="12" cy="12" r="10" />
71
+ <path d="m9 12 2 2 4-4" />
72
+ </>
73
+ ),
74
+ bad: (
75
+ <>
76
+ <circle cx="12" cy="12" r="10" />
77
+ <path d="m15 9-6 6" />
78
+ <path d="m9 9 6 6" />
79
+ </>
80
+ ),
81
+ warn: (
82
+ <>
83
+ <circle cx="12" cy="12" r="10" />
84
+ <line x1="12" x2="12" y1="8" y2="12" />
85
+ <line x1="12" x2="12.01" y1="16" y2="16" />
86
+ </>
87
+ ),
88
+ na: (
89
+ <>
90
+ <circle cx="12" cy="12" r="10" />
91
+ <path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3" />
92
+ <path d="M12 17h.01" />
93
+ </>
94
+ ),
95
+ };
96
+
97
+ function LineNo({ line, tone, send }: { line: number; tone: ReturnType<typeof lineTone>; send: boolean }): ReactElement {
98
+ const kind = tone ?? (send ? "send" : null);
99
+ if (kind === null) return <span className="nre-source-ln">{line}</span>;
100
+ const label = kind === "bad" ? "failed" : kind === "warn" ? "soft failed" : kind === "good" ? "passed" : kind === "na" ? "unavailable" : "send";
101
+ return (
102
+ <span className="nre-source-ln nre-source-ln-mark" role="img" aria-label={label} title={label}>
103
+ <svg viewBox="0 0 24 24" aria-hidden="true">{MARK_ICONS[kind]}</svg>
104
+ </span>
105
+ );
106
+ }
107
+
108
+ function LineSummary({ line, tone }: { line: AttemptSourceLineData; tone: ReturnType<typeof lineTone> }): ReactElement {
109
+ const interactive = line.assertions.length > 0 || line.sends.length > 0 || line.turns.length > 0;
110
+ const scores = thresholdScores(line.assertions);
111
+ return (
112
+ <span className="nre-source-line-summary">
113
+ <LineNo line={line.line} tone={tone} send={line.sends.length > 0 || line.turns.length > 0} />
114
+ <code className="nre-source-text">{highlightTs(line.text)}</code>
115
+ <span className="nre-source-line-meta">
116
+ {scores.length > 0 ? <span className="nre-source-score-badge">{scores.join(", ")}</span> : null}
117
+ {interactive ? <span className="nre-source-chevron">›</span> : null}
118
+ </span>
119
+ </span>
120
+ );
121
+ }
122
+
123
+ function TurnDetail({ turn, showMeta = false, showSent = false }: { turn: AttemptSourceTurn; showMeta?: boolean; showSent?: boolean }): ReactElement {
124
+ return (
125
+ <div className={cx("nre-source-turn", `nre-source-turn-${turn.status}`)}>
126
+ {showMeta ? (
127
+ <div className="nre-source-turn-head">
128
+ <span>{turn.label}</span>
129
+ <span>{turn.status}</span>
130
+ {turn.durationMs === undefined ? null : <span>{formatDuration(turn.durationMs)}</span>}
131
+ </div>
132
+ ) : null}
133
+ {showSent && turn.sentText ? <div className="nre-conv-sent">{turn.sentText}</div> : null}
134
+ <ConversationReplies replies={turn.replies} />
135
+ </div>
136
+ );
137
+ }
138
+
139
+ function AssertionDetail({ assertion }: { assertion: AssertionResult }): ReactElement {
140
+ return (
141
+ <div className={`nre-assertion-row nre-tone-${assertTone(assertion)}`}>
142
+ <div className="nre-source-assertion-head">
143
+ <span className="nre-assertion-badge">{assertion.outcome}</span>
144
+ <span className="nre-assertion-name">{assertion.name}</span>
145
+ {assertion.outcome !== "unavailable" ? <span className="nre-source-assertion-score">{assertion.score}</span> : null}
146
+ </div>
147
+ {assertion.detail ? <div className="nre-assertion-detail">{assertion.detail}</div> : null}
148
+ {assertion.outcome === "unavailable" ? (
149
+ <div className="nre-assertion-body">reason: {assertion.reason}</div>
150
+ ) : assertion.expected !== undefined || assertion.received !== undefined || assertion.evidence !== undefined ? (
151
+ <div className="nre-assertion-body">
152
+ {assertion.expected !== undefined ? <span>expected: {assertion.expected}</span> : null}
153
+ {assertion.received !== undefined ? <span>received: {assertion.received}</span> : null}
154
+ {assertion.evidence !== undefined ? <span>evidence: {assertion.evidence}</span> : null}
155
+ </div>
156
+ ) : null}
157
+ </div>
158
+ );
159
+ }
160
+
17
161
  export function AttemptSource({ data, className }: { data: AttemptSourceData | null; className?: string }): ReactElement | null {
18
162
  if (data === null) return null;
163
+ const firstAttentionLine = data.lines.find((line) => {
164
+ const tone = lineTone(line);
165
+ return tone === "bad" || tone === "warn" || tone === "na";
166
+ })?.line;
19
167
  return (
20
168
  <div className={cx("nre", "nre-attempt-source", className)}>
21
169
  <div className="nre-attempt-source-head">{data.sourcePath}</div>
22
170
  <div className="nre-attempt-source-lines">
23
171
  {data.lines.map((line) => {
24
- const hasAsserts = line.assertions.length > 0;
25
- const hasSends = line.sends.length > 0;
26
- const worstTone = hasAsserts
27
- ? line.assertions.some((a) => assertTone(a) === "bad")
28
- ? "bad"
29
- : line.assertions.some((a) => assertTone(a) === "warn")
30
- ? "warn"
31
- : line.assertions.some((a) => assertTone(a) === "na")
32
- ? "na"
33
- : "good"
34
- : undefined;
172
+ const tone = lineTone(line);
173
+ const interactive = line.assertions.length > 0 || line.sends.length > 0 || line.turns.length > 0;
174
+ const lineClass = cx(
175
+ "nre-source-line",
176
+ tone ? `nre-tone-${tone}` : undefined,
177
+ line.sends.length > 0 || line.turns.length > 0 ? "nre-source-line-send" : undefined,
178
+ );
179
+ if (!interactive) {
180
+ return (
181
+ <div key={line.line} className={lineClass}>
182
+ <LineSummary line={line} tone={tone} />
183
+ </div>
184
+ );
185
+ }
35
186
  return (
36
- <details
37
- key={line.line}
38
- className={cx("nre-source-line", worstTone ? `nre-tone-${worstTone}` : undefined, hasSends ? "nre-source-line-send" : undefined)}
39
- open={hasAsserts && (worstTone === "bad" || worstTone === "warn")}
40
- >
187
+ <details key={line.line} className={lineClass} open={line.line === firstAttentionLine}>
41
188
  <summary>
42
- <span className="nre-source-ln">{line.line}</span>
43
- <code className="nre-source-text">{line.text}</code>
44
- {hasSends ? <span className="nre-source-send-mark">↳ {line.sends.map((s) => s.label).join(", ")}</span> : null}
189
+ <LineSummary line={line} tone={tone} />
45
190
  </summary>
46
- {hasAsserts ? (
47
- <div className="nre-source-line-detail">
48
- {line.assertions.map((a, i) => (
49
- <div key={i} className={`nre-assertion-row nre-tone-${assertTone(a)}`}>
50
- <span className="nre-assertion-name">{a.name}</span>
51
- {a.outcome === "unavailable" ? (
52
- <span>{a.reason}</span>
53
- ) : (
54
- <>
55
- {a.expected !== undefined ? <span>expected: {a.expected}</span> : null}
56
- {a.received !== undefined ? <span>received: {a.received}</span> : null}
57
- </>
58
- )}
59
- </div>
60
- ))}
61
- </div>
62
- ) : null}
191
+ <div className="nre-source-line-detail">
192
+ {line.turns.map((turn, i) => (
193
+ <TurnDetail key={`${turn.label}-${i}`} turn={turn} />
194
+ ))}
195
+ {line.assertions.map((assertion, i) => (
196
+ <AssertionDetail key={i} assertion={assertion} />
197
+ ))}
198
+ </div>
63
199
  </details>
64
200
  );
65
201
  })}
@@ -75,6 +211,14 @@ export function AttemptSource({ data, className }: { data: AttemptSourceData | n
75
211
  ))}
76
212
  </div>
77
213
  ) : null}
214
+ {data.unlocatedTurns.length > 0 ? (
215
+ <div className="nre-attempt-source-unlocated">
216
+ <div className="nre-attempt-source-unmapped-head">Other conversation</div>
217
+ {data.unlocatedTurns.map((turn, i) => (
218
+ <TurnDetail key={`${turn.label}-${i}`} turn={turn} showMeta showSent />
219
+ ))}
220
+ </div>
221
+ ) : null}
78
222
  </div>
79
223
  );
80
224
  }
@@ -1,8 +1,9 @@
1
1
  // cases: docs/engineering/unit-tests/reports/cases.md
2
2
  // Attempt 详情组件族的单元测试:11 个叶子的非空/空证据矩阵、AttemptAssessment 的
3
3
  // source/assertions fallback、AttemptDetail 的内建顺序、spec/data 等价与 scope-input page
4
- // 报错、AttemptConversation 的 loc 分轮、断言区默认展开规则、AttemptTimeline 的默认折叠。
4
+ // 报错、AttemptConversation 的 loc 分轮、attemptSourceData 的 loc 投影、AttemptTimeline 的默认折叠。
5
5
  // 纯渲染,注入数据:直接构造 AttemptEvidence fixture,不 mock fetch(这些组件从不 fetch)。
6
+ // 样式与视觉交互(染色、布局、展开观感)不在本层:归 E2E 报告域(docs/engineering/e2e-ci/results.md)。
6
7
 
7
8
  import { renderToStaticMarkup } from "react-dom/server";
8
9
  import { describe, expect, it } from "vitest";
@@ -310,14 +311,20 @@ describe("AttemptAssessment / AttemptDetail(组合组件)", () => {
310
311
  await expect(resolveOnScopePage(<AttemptAssessment />)).rejects.toThrow(/attempt-input page/);
311
312
  });
312
313
 
313
- it("AttemptDetail:一级子节点类型序列与内建顺序逐项相同", () => {
314
+ it("AttemptDetail:有 source 时不重复 Conversation,无 source 时在 usage 后保留 fallback", () => {
314
315
  // AttemptDetail 自己是组合组件:resolve 会把它(以及嵌套的 AttemptAssessment)递归展开,
315
316
  // 所以这里直接检查它的 compose 函数产出的原始树(与「内建报告」测试检查 standard.tsx
316
317
  // 原始声明同一手法),不走完整 resolve——那样 AttemptAssessment 会被替换成它自己展开出的
317
318
  // <Col> 而不再是 AttemptAssessment 这个类型。
318
319
  const compose = composeOf(AttemptDetail)!;
319
- const tree = compose({}, {} as never) as unknown as { props: { children: Array<{ type: unknown }> } };
320
- expect(tree.props.children.map((c) => c.type)).toEqual([
320
+ const childTypes = (evidence: AttemptEvidence): unknown[] => {
321
+ const tree = compose({}, { page: { input: "attempt", evidence } } as never) as unknown as {
322
+ props: { children: Array<{ type: unknown } | null> };
323
+ };
324
+ return tree.props.children.filter((child): child is { type: unknown } => child !== null).map((child) => child.type);
325
+ };
326
+ const withoutSource = childTypes(evidenceOf());
327
+ expect(withoutSource).toEqual([
321
328
  AttemptSummary,
322
329
  AttemptAssessment,
323
330
  AttemptFixPrompt,
@@ -328,6 +335,30 @@ describe("AttemptAssessment / AttemptDetail(组合组件)", () => {
328
335
  AttemptTrace,
329
336
  AttemptDiff,
330
337
  ]);
338
+
339
+ const withSource = childTypes(
340
+ evidenceOf({
341
+ capabilities: { ...NO_CAPS, source: true },
342
+ evalSource: {
343
+ sourcePath: "evals/a.ts",
344
+ sourceSha256: "x",
345
+ lines: [{ line: 1, text: "", assertions: [], sends: [] }],
346
+ unmapped: [],
347
+ summary: {
348
+ totalAssertions: 0,
349
+ mappedAssertions: 0,
350
+ unmappedAssertions: 0,
351
+ passed: 0,
352
+ failed: 0,
353
+ gate: 0,
354
+ soft: 0,
355
+ totalLines: 1,
356
+ annotatedLines: 0,
357
+ },
358
+ },
359
+ }),
360
+ );
361
+ expect(withSource).toEqual(withoutSource.filter((type) => type !== AttemptConversation));
331
362
  });
332
363
  });
333
364
 
@@ -455,6 +486,53 @@ describe("AttemptConversation:标准事件流按 loc 分轮", () => {
455
486
  });
456
487
  });
457
488
 
489
+ // bug: memory/attempt-detail-components-shipped-without-styles.md
490
+ describe("attemptSourceData:标准事件流按 loc 投影回 send 行", () => {
491
+ it("send 行的 turns 携带 sentText 与按序归并的完整回复", () => {
492
+ const sourcePath = "evals/a.ts";
493
+ const data = attemptSourceData(
494
+ evidenceOf({
495
+ capabilities: { ...NO_CAPS, source: true, execution: true },
496
+ evalSource: {
497
+ sourcePath,
498
+ sourceSha256: "x",
499
+ lines: [
500
+ { line: 1, text: 'import { defineEval } from "niceeval";', assertions: [], sends: [] },
501
+ {
502
+ line: 2,
503
+ text: 'const reply = await t.send("hello");',
504
+ assertions: [],
505
+ sends: [{ label: "s1/t1", status: "completed" as const, durationMs: 120, loc: { file: sourcePath, line: 2 } }],
506
+ },
507
+ ],
508
+ unmapped: [],
509
+ summary: {
510
+ totalAssertions: 0,
511
+ mappedAssertions: 0,
512
+ unmappedAssertions: 0,
513
+ passed: 0,
514
+ failed: 0,
515
+ gate: 0,
516
+ soft: 0,
517
+ totalLines: 2,
518
+ annotatedLines: 1,
519
+ },
520
+ },
521
+ events: [
522
+ { type: "message", role: "user", text: "hello", loc: { file: sourcePath, line: 2 } },
523
+ { type: "message", role: "assistant", text: "assistant reply attached to the source line" },
524
+ ],
525
+ }),
526
+ )!;
527
+
528
+ expect(data.lines[1]!.turns[0]).toMatchObject({ label: "s1/t1", sentText: "hello" });
529
+ expect(data.lines[1]!.turns[0]!.replies).toEqual([
530
+ { kind: "assistant", text: "assistant reply attached to the source line" },
531
+ ]);
532
+ expect(data.lines[0]!.turns).toEqual([]);
533
+ });
534
+ });
535
+
458
536
  // ───────────────────────── AttemptTimeline:默认折叠 ─────────────────────────
459
537
 
460
538
  describe("AttemptTimeline:默认只显示主链,children 收合", () => {
@@ -18,6 +18,7 @@ import type {
18
18
  AttemptErrorData,
19
19
  AttemptFixPromptData,
20
20
  AttemptSourceData,
21
+ AttemptSourceTurn,
21
22
  AttemptSummaryData,
22
23
  AttemptTimelineData,
23
24
  AttemptTraceData,
@@ -71,7 +72,55 @@ export function attemptAssertionsData(evidence: AttemptEvidence): AttemptAsserti
71
72
  export function attemptSourceData(evidence: AttemptEvidence): AttemptSourceData | null {
72
73
  if (!evidence.capabilities.source || evidence.evalSource === null) return null;
73
74
  const { sourcePath, lines, unmapped, summary } = evidence.evalSource;
74
- return { locator: evidence.locator, sourcePath, lines, unmapped, summary };
75
+ const projectedLines = lines.map((line) => ({
76
+ ...line,
77
+ turns: line.sends.map<AttemptSourceTurn>((send) => ({
78
+ label: send.label,
79
+ status: send.status,
80
+ ...(send.durationMs === undefined ? {} : { durationMs: send.durationMs }),
81
+ sentText: "",
82
+ replies: [],
83
+ })),
84
+ }));
85
+ const usedTurns = new Map<number, number>();
86
+ const unlocatedTurns: AttemptSourceTurn[] = [];
87
+ const conversation = attemptConversationData(evidence);
88
+
89
+ for (const [roundIndex, round] of (conversation?.rounds ?? []).entries()) {
90
+ const status = round.replies.some(
91
+ (reply) =>
92
+ reply.kind === "error" ||
93
+ ((reply.kind === "tool" || reply.kind === "subagent") && reply.status === "failed"),
94
+ )
95
+ ? "failed"
96
+ : round.replies.some((reply) => reply.kind === "input")
97
+ ? "waiting"
98
+ : "completed";
99
+ const fallback: AttemptSourceTurn = {
100
+ label: `t${roundIndex + 1}`,
101
+ status,
102
+ sentText: round.sentText,
103
+ replies: round.replies,
104
+ };
105
+ const loc = round.loc;
106
+ if (!loc || loc.file !== sourcePath || loc.line < 1 || loc.line > projectedLines.length) {
107
+ unlocatedTurns.push(fallback);
108
+ continue;
109
+ }
110
+
111
+ const line = projectedLines[loc.line - 1]!;
112
+ const turnIndex = usedTurns.get(loc.line) ?? 0;
113
+ usedTurns.set(loc.line, turnIndex + 1);
114
+ const annotated = line.turns[turnIndex];
115
+ if (annotated) {
116
+ annotated.sentText = round.sentText;
117
+ annotated.replies = round.replies;
118
+ } else {
119
+ line.turns.push(fallback);
120
+ }
121
+ }
122
+
123
+ return { locator: evidence.locator, sourcePath, lines: projectedLines, unmapped, unlocatedTurns, summary };
75
124
  }
76
125
 
77
126
  // ───────────────────────── AttemptFixPrompt ─────────────────────────
@@ -107,6 +107,9 @@ export function attemptSourceText(data: AttemptSourceData | null, ctx: TextConte
107
107
  const command = evidenceCommand(ctx, data.locator, "--source");
108
108
  const headerParts = [`${data.sourcePath} · ${data.summary.annotatedLines}/${data.summary.totalLines} lines annotated`];
109
109
  if (command) headerParts.push(command);
110
+ const hasConversation = data.unlocatedTurns.length > 0 || data.lines.some((line) => line.turns.length > 0);
111
+ const executionCommand = hasConversation ? evidenceCommand(ctx, data.locator, "--execution") : null;
112
+ if (executionCommand) headerParts.push(executionCommand);
110
113
  // 源码锚由 assertionLine 自己按 a.loc 拼(与 AttemptAssertions 共用同一份逻辑);这里只负责
111
114
  // 挑出非 passed 的条目,不重复算锚点。
112
115
  const failed = data.lines.flatMap((line) => line.assertions.filter((a) => a.outcome !== "passed"));
@@ -261,13 +261,24 @@ export const AttemptAssertions = makeAttemptComponent<AttemptAssertionsData>({
261
261
 
262
262
  /** AnnotatedSourceLine(src/results/annotated-source.ts):一行源码 + 映射到这一行的断言 / send 标注。 */
263
263
  function annotatedSourceLineProblem(value: unknown, path: string): string | null {
264
- if (!isObject(value)) return `"${path}" must be an AnnotatedSourceLine { line, text, assertions, sends }`;
264
+ if (!isObject(value)) return `"${path}" must be an AttemptSourceLineData { line, text, assertions, sends, turns }`;
265
265
  if (typeof value.line !== "number") return `"${path}.line" must be a number`;
266
266
  if (typeof value.text !== "string") return `"${path}.text" must be a string`;
267
267
  const assertionsProblem = arrayProblem(value.assertions, `${path}.assertions`, assertionResultProblem);
268
268
  if (assertionsProblem !== null) return assertionsProblem;
269
269
  if (!Array.isArray(value.sends)) return `"${path}.sends" must be an array`;
270
- return null;
270
+ return arrayProblem(value.turns, `${path}.turns`, sourceTurnProblem);
271
+ }
272
+
273
+ function sourceTurnProblem(value: unknown, path: string): string | null {
274
+ if (!isObject(value)) return `"${path}" must be an AttemptSourceTurn`;
275
+ if (typeof value.label !== "string") return `"${path}.label" must be a string`;
276
+ if (value.status !== "completed" && value.status !== "failed" && value.status !== "waiting") {
277
+ return `"${path}.status" must be "completed", "failed", or "waiting"`;
278
+ }
279
+ if (value.durationMs !== undefined && typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
280
+ if (typeof value.sentText !== "string") return `"${path}.sentText" must be a string`;
281
+ return arrayProblem(value.replies, `${path}.replies`, conversationReplyProblem);
271
282
  }
272
283
 
273
284
  /** AnnotatedEvalSourceSummary(src/results/annotated-source.ts):全是计数字段。 */
@@ -297,6 +308,8 @@ export function validateSourceData(data: unknown): string | null {
297
308
  if (linesProblem !== null) return linesProblem;
298
309
  const unmappedProblem = arrayProblem(data.unmapped, "unmapped", assertionResultProblem);
299
310
  if (unmappedProblem !== null) return unmappedProblem;
311
+ const turnsProblem = arrayProblem(data.unlocatedTurns, "unlocatedTurns", sourceTurnProblem);
312
+ if (turnsProblem !== null) return turnsProblem;
300
313
  return sourceSummaryProblem(data.summary, "summary");
301
314
  }
302
315
 
@@ -566,18 +579,22 @@ export const AttemptAssessment = defineComponent((_props: Record<string, never>,
566
579
  });
567
580
  AttemptAssessment.displayName = "AttemptAssessment";
568
581
 
569
- /** 内建排列顺序;不产生新的 data 或渲染面,用户可以在自己的 attempt-input page 里重排这些叶子。 */
570
- export const AttemptDetail = defineComponent(() => (
571
- <Col>
572
- <AttemptSummary />
573
- <AttemptAssessment />
574
- <AttemptFixPrompt />
575
- <AttemptTimeline />
576
- <AttemptDiagnostics />
577
- <AttemptUsage />
578
- <AttemptConversation />
579
- <AttemptTrace />
580
- <AttemptDiff />
581
- </Col>
582
- ));
582
+ /** 内建排列顺序;有 source 时回复已按 loc 展开在 AttemptSource 行内,不再重复一份 round 卡。 */
583
+ export const AttemptDetail = defineComponent((_props: Record<string, never>, ctx) => {
584
+ const conversationLivesInSource =
585
+ ctx.page.input === "attempt" && ctx.page.evidence.capabilities.source && ctx.page.evidence.evalSource !== null;
586
+ return (
587
+ <Col>
588
+ <AttemptSummary />
589
+ <AttemptAssessment />
590
+ <AttemptFixPrompt />
591
+ <AttemptTimeline />
592
+ <AttemptDiagnostics />
593
+ <AttemptUsage />
594
+ {conversationLivesInSource ? null : <AttemptConversation />}
595
+ <AttemptTrace />
596
+ <AttemptDiff />
597
+ </Col>
598
+ );
599
+ });
583
600
  AttemptDetail.displayName = "AttemptDetail";
@@ -106,8 +106,9 @@ describe("validateSourceData", () => {
106
106
  const valid = {
107
107
  locator: "@1abcdef2",
108
108
  sourcePath: "eval.ts",
109
- lines: [{ line: 1, text: "t.send(...)", assertions: [], sends: [] }],
109
+ lines: [{ line: 1, text: "t.send(...)", assertions: [], sends: [], turns: [] }],
110
110
  unmapped: [],
111
+ unlocatedTurns: [],
111
112
  summary: validSummary,
112
113
  };
113
114
 
@@ -126,9 +127,25 @@ describe("validateSourceData", () => {
126
127
  });
127
128
 
128
129
  it("lines[i].assertions 嵌套断言结构错误报错", () => {
129
- const bad = { ...valid, lines: [{ line: 1, text: "x", assertions: [{ name: "eq" }], sends: [] }] };
130
+ const bad = { ...valid, lines: [{ line: 1, text: "x", assertions: [{ name: "eq" }], sends: [], turns: [] }] };
130
131
  expect(validateSourceData(bad)).toMatch(/"lines\[0\]\.assertions\[0\]\.severity"/);
131
132
  });
133
+
134
+ it("lines[i].turns[j].replies[k] 递归校验回复判别联合", () => {
135
+ const bad = {
136
+ ...valid,
137
+ lines: [
138
+ {
139
+ line: 1,
140
+ text: "x",
141
+ assertions: [],
142
+ sends: [],
143
+ turns: [{ label: "s1/t1", status: "completed", sentText: "go", replies: [{ kind: "assistant" }] }],
144
+ },
145
+ ],
146
+ };
147
+ expect(validateSourceData(bad)).toMatch(/"lines\[0\]\.turns\[0\]\.replies\[0\]\.text"/);
148
+ });
132
149
  });
133
150
 
134
151
  describe("validateTimelineData", () => {