niceeval 0.10.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +5 -1
- package/dist/report/components/attempt-detail/AttemptConversation.js +42 -2
- package/dist/report/components/attempt-detail/AttemptSource.js +83 -13
- package/dist/report/components/attempt-detail/compute.js +44 -1
- package/dist/report/components/attempt-detail/faces.js +4 -0
- package/dist/report/components/attempt-detail/index.d.ts +2 -2
- package/dist/report/components/attempt-detail/index.js +24 -4
- package/dist/report/components/entity-lists/ExperimentList.d.ts +1 -2
- package/dist/report/components/entity-lists/ExperimentList.js +6 -5
- package/dist/report/components/entity-lists/faces.d.ts +1 -1
- package/dist/report/components/entity-lists/faces.js +10 -9
- package/dist/report/components/entity-lists/index.d.ts +5 -7
- package/dist/report/components/entity-lists/index.js +7 -3
- package/dist/report/components/metric-views/MetricScatter.js +2 -38
- package/dist/report/index.d.ts +1 -1
- package/dist/report/model/format.d.ts +5 -5
- package/dist/report/model/format.js +34 -8
- package/dist/report/model/types.d.ts +16 -2
- package/dist/report/react/index.d.ts +1 -1
- package/dist/runner/types.d.ts +2 -1
- package/dist/sandbox/e2b.d.ts +0 -1
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +5 -5
- package/docs-site/zh/explanation/runner.mdx +6 -2
- package/docs-site/zh/reference/cli.mdx +2 -2
- package/docs-site/zh/reference/report-components.mdx +3 -1
- package/docs-site/zh/tutorials/experiments.mdx +1 -2
- package/docs-site/zh/tutorials/viewing-results.mdx +12 -9
- package/docs-site/zh/tutorials/write-experiment.mdx +1 -3
- package/package.json +3 -4
- package/src/cli.ts +4 -2
- package/src/o11y/prices.json +176 -99
- package/src/report/assets/styles.css +822 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +55 -7
- package/src/report/components/attempt-detail/AttemptSource.tsx +186 -42
- package/src/report/components/attempt-detail/attempt-components.test.tsx +82 -4
- package/src/report/components/attempt-detail/compute.ts +50 -1
- package/src/report/components/attempt-detail/faces.ts +3 -0
- package/src/report/components/attempt-detail/index.tsx +33 -16
- package/src/report/components/attempt-detail/validate.test.ts +19 -2
- package/src/report/components/entity-lists/ExperimentList.tsx +13 -10
- package/src/report/components/entity-lists/faces.ts +10 -9
- package/src/report/components/entity-lists/index.tsx +7 -10
- package/src/report/components/metric-views/MetricScatter.tsx +2 -38
- package/src/report/components/render.test.tsx +19 -9
- package/src/report/index.ts +2 -0
- package/src/report/model/format.ts +33 -8
- package/src/report/model/types.ts +18 -2
- package/src/report/react/index.tsx +2 -0
- package/src/report/runtime/dual-render.test.tsx +43 -10
- package/src/runner/types.ts +2 -1
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +38 -2
- package/src/view/view-report.test.ts +3 -1
|
@@ -3,8 +3,40 @@
|
|
|
3
3
|
|
|
4
4
|
import type { ReactElement, ReactNode } from "react";
|
|
5
5
|
import type { AttemptConversationData, AttemptConversationReply, AttemptConversationRound } from "../../model/types.ts";
|
|
6
|
+
import type { JsonValue, ToolName } from "../../../types.ts";
|
|
6
7
|
import { cx } from "../shared.ts";
|
|
7
8
|
|
|
9
|
+
const TOOL_VERB: Partial<Record<ToolName, string>> = {
|
|
10
|
+
shell: "Bash",
|
|
11
|
+
file_read: "Read",
|
|
12
|
+
file_write: "Write",
|
|
13
|
+
file_edit: "Edit",
|
|
14
|
+
web_fetch: "Fetch",
|
|
15
|
+
web_search: "Search",
|
|
16
|
+
glob: "Glob",
|
|
17
|
+
grep: "Grep",
|
|
18
|
+
list_dir: "List",
|
|
19
|
+
agent_task: "Task",
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
function compact(value: JsonValue | undefined, max = 140): string {
|
|
23
|
+
if (value === undefined || value === null) return "";
|
|
24
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
25
|
+
const oneLine = text.replace(/\s+/g, " ").trim();
|
|
26
|
+
return oneLine.length > max ? `${oneLine.slice(0, max)}…` : oneLine;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function toolPrimaryArg(input: JsonValue): string {
|
|
30
|
+
if (typeof input === "string") return compact(input, 80);
|
|
31
|
+
if (input === null || Array.isArray(input) || typeof input !== "object") return "";
|
|
32
|
+
for (const key of ["command", "cmd", "path", "file", "file_path", "pattern", "query", "url", "prompt", "description"]) {
|
|
33
|
+
const value = input[key];
|
|
34
|
+
if (typeof value === "string" && value) return compact(value, 80);
|
|
35
|
+
if (key === "command" && Array.isArray(value)) return compact(value.filter((item) => typeof item === "string").join(" "), 80);
|
|
36
|
+
}
|
|
37
|
+
return compact(input, 80);
|
|
38
|
+
}
|
|
39
|
+
|
|
8
40
|
function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
|
|
9
41
|
switch (reply.kind) {
|
|
10
42
|
case "assistant":
|
|
@@ -37,11 +69,20 @@ function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
|
|
|
37
69
|
</div>
|
|
38
70
|
);
|
|
39
71
|
case "tool":
|
|
72
|
+
const verb = (reply.tool ? TOOL_VERB[reply.tool] : undefined) ?? reply.name;
|
|
73
|
+
const arg = toolPrimaryArg(reply.input);
|
|
74
|
+
const preview = compact(reply.output);
|
|
40
75
|
return (
|
|
41
76
|
<details className="nre-conv-tool">
|
|
42
77
|
<summary>
|
|
43
|
-
{reply.
|
|
44
|
-
{
|
|
78
|
+
<span className={cx("nre-conv-tool-dot", reply.status ? `nre-conv-tool-${reply.status}` : "nre-conv-tool-pending")} />
|
|
79
|
+
<span className="nre-conv-tool-name" title={arg ? `${verb}(${arg})` : verb}>
|
|
80
|
+
{arg ? `${verb}(${arg})` : verb}
|
|
81
|
+
</span>
|
|
82
|
+
<span className="nre-conv-tool-preview">
|
|
83
|
+
{reply.status ?? "pending"}
|
|
84
|
+
{preview ? ` · ${preview}` : ""}
|
|
85
|
+
</span>
|
|
45
86
|
</summary>
|
|
46
87
|
<pre className="nre-conv-tool-io">{JSON.stringify(reply.input, null, 2)}</pre>
|
|
47
88
|
{reply.output !== undefined ? <pre className="nre-conv-tool-io">{JSON.stringify(reply.output, null, 2)}</pre> : null}
|
|
@@ -71,6 +112,17 @@ function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
|
|
|
71
112
|
}
|
|
72
113
|
}
|
|
73
114
|
|
|
115
|
+
/** AttemptSource 复用同一份回复 renderer,把一轮执行挂回对应的 send 源码行。 */
|
|
116
|
+
export function ConversationReplies({ replies }: { replies: AttemptConversationReply[] }): ReactElement {
|
|
117
|
+
return (
|
|
118
|
+
<div className="nre-conv-replies">
|
|
119
|
+
{replies.map((reply, i) => (
|
|
120
|
+
<ReplyRow key={i} reply={reply} />
|
|
121
|
+
))}
|
|
122
|
+
</div>
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
74
126
|
function RoundCard({ round, index }: { round: AttemptConversationRound; index: number }): ReactElement {
|
|
75
127
|
return (
|
|
76
128
|
<div className="nre-conv-round">
|
|
@@ -83,11 +135,7 @@ function RoundCard({ round, index }: { round: AttemptConversationRound; index: n
|
|
|
83
135
|
) : null}
|
|
84
136
|
</div>
|
|
85
137
|
{round.sentText ? <div className="nre-conv-sent">{round.sentText}</div> : null}
|
|
86
|
-
<
|
|
87
|
-
{round.replies.map((reply, i) => (
|
|
88
|
-
<ReplyRow key={i} reply={reply} />
|
|
89
|
-
))}
|
|
90
|
-
</div>
|
|
138
|
+
<ConversationReplies replies={round.replies} />
|
|
91
139
|
</div>
|
|
92
140
|
);
|
|
93
141
|
}
|
|
@@ -1,12 +1,38 @@
|
|
|
1
|
-
// AttemptSource
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// 边界拆分,同一批事实不在两个组件里各展一份。
|
|
1
|
+
// AttemptSource:GitHub diff 式带标注 eval 源码。send / assertion 行按状态着色,点击
|
|
2
|
+
// 原生 details 在调用点展开完整回复与 assertion 细节。没有 source 时零输出
|
|
3
|
+
// (docs/feature/reports/library/attempt-detail.md)。
|
|
5
4
|
|
|
6
|
-
import type { ReactElement } from "react";
|
|
7
|
-
import type { AttemptSourceData } from "../../model/types.ts";
|
|
5
|
+
import type { ReactElement, ReactNode } from "react";
|
|
6
|
+
import type { AttemptSourceData, AttemptSourceLineData, AttemptSourceTurn } from "../../model/types.ts";
|
|
8
7
|
import type { AssertionResult } from "../../../types.ts";
|
|
9
8
|
import { cx } from "../shared.ts";
|
|
9
|
+
import { ConversationReplies } from "./AttemptConversation.tsx";
|
|
10
|
+
|
|
11
|
+
const TS_HL_RE =
|
|
12
|
+
/(\/\/[^\n]*)|(\/\*[^]*?\*\/)|(`(?:\\.|[^`\\])*`|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|\b(import|from|export|default|const|let|var|async|await|function|return|if|else|for|of|in|new|class|extends|typeof|void|true|false|null|undefined)\b|\b(\d[\d_.]*)\b|([A-Za-z_$][\w$]*)(?=\s*\()/g;
|
|
13
|
+
|
|
14
|
+
/** 逐行零依赖 TS 高亮;token class 是稳定的 web 展示语义,不改源码文本。 */
|
|
15
|
+
function highlightTs(line: string): ReactNode[] {
|
|
16
|
+
const out: ReactNode[] = [];
|
|
17
|
+
let last = 0;
|
|
18
|
+
let i = 0;
|
|
19
|
+
let match: RegExpExecArray | null;
|
|
20
|
+
TS_HL_RE.lastIndex = 0;
|
|
21
|
+
while ((match = TS_HL_RE.exec(line))) {
|
|
22
|
+
if (match.index > last) out.push(line.slice(last, match.index));
|
|
23
|
+
const tokenClass =
|
|
24
|
+
match[1] || match[2] ? "tok-comment" : match[3] ? "tok-str" : match[4] ? "tok-kw" : match[5] ? "tok-num" : "tok-fn";
|
|
25
|
+
out.push(
|
|
26
|
+
<span key={i++} className={tokenClass}>
|
|
27
|
+
{match[0]}
|
|
28
|
+
</span>,
|
|
29
|
+
);
|
|
30
|
+
last = match.index + match[0].length;
|
|
31
|
+
if (match[0].length === 0) TS_HL_RE.lastIndex++;
|
|
32
|
+
}
|
|
33
|
+
if (last < line.length) out.push(line.slice(last));
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
10
36
|
|
|
11
37
|
function assertTone(a: AssertionResult): "good" | "warn" | "bad" | "na" {
|
|
12
38
|
if (a.outcome === "unavailable") return "na";
|
|
@@ -14,52 +40,162 @@ function assertTone(a: AssertionResult): "good" | "warn" | "bad" | "na" {
|
|
|
14
40
|
return a.severity === "soft" ? "warn" : "bad";
|
|
15
41
|
}
|
|
16
42
|
|
|
43
|
+
function lineTone(line: AttemptSourceLineData): "good" | "warn" | "bad" | "na" | undefined {
|
|
44
|
+
if (line.assertions.length === 0) return undefined;
|
|
45
|
+
if (line.assertions.some((a) => assertTone(a) === "bad")) return "bad";
|
|
46
|
+
if (line.assertions.some((a) => assertTone(a) === "warn")) return "warn";
|
|
47
|
+
if (line.assertions.some((a) => assertTone(a) === "na")) return "na";
|
|
48
|
+
return "good";
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function formatDuration(ms: number): string {
|
|
52
|
+
return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function thresholdScores(assertions: AssertionResult[]): string[] {
|
|
56
|
+
const scores: string[] = [];
|
|
57
|
+
for (const assertion of assertions) {
|
|
58
|
+
if (assertion.outcome !== "unavailable" && assertion.threshold !== undefined) {
|
|
59
|
+
scores.push(`${assertion.score}/${assertion.threshold}`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return scores;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** 行号位标记(与产品站示例卡同语言:图标顶替行号,不加独立状态列);内联 SVG,零图标依赖。 */
|
|
66
|
+
const MARK_ICONS: Record<"send" | "good" | "bad" | "warn" | "na", ReactElement> = {
|
|
67
|
+
send: <path d="M21 15a4 4 0 0 1-4 4H8l-5 3V7a4 4 0 0 1 4-4h10a4 4 0 0 1 4 4z" />,
|
|
68
|
+
good: (
|
|
69
|
+
<>
|
|
70
|
+
<circle cx="12" cy="12" r="10" />
|
|
71
|
+
<path d="m9 12 2 2 4-4" />
|
|
72
|
+
</>
|
|
73
|
+
),
|
|
74
|
+
bad: (
|
|
75
|
+
<>
|
|
76
|
+
<circle cx="12" cy="12" r="10" />
|
|
77
|
+
<path d="m15 9-6 6" />
|
|
78
|
+
<path d="m9 9 6 6" />
|
|
79
|
+
</>
|
|
80
|
+
),
|
|
81
|
+
warn: (
|
|
82
|
+
<>
|
|
83
|
+
<circle cx="12" cy="12" r="10" />
|
|
84
|
+
<line x1="12" x2="12" y1="8" y2="12" />
|
|
85
|
+
<line x1="12" x2="12.01" y1="16" y2="16" />
|
|
86
|
+
</>
|
|
87
|
+
),
|
|
88
|
+
na: (
|
|
89
|
+
<>
|
|
90
|
+
<circle cx="12" cy="12" r="10" />
|
|
91
|
+
<path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3" />
|
|
92
|
+
<path d="M12 17h.01" />
|
|
93
|
+
</>
|
|
94
|
+
),
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
function LineNo({ line, tone, send }: { line: number; tone: ReturnType<typeof lineTone>; send: boolean }): ReactElement {
|
|
98
|
+
const kind = tone ?? (send ? "send" : null);
|
|
99
|
+
if (kind === null) return <span className="nre-source-ln">{line}</span>;
|
|
100
|
+
const label = kind === "bad" ? "failed" : kind === "warn" ? "soft failed" : kind === "good" ? "passed" : kind === "na" ? "unavailable" : "send";
|
|
101
|
+
return (
|
|
102
|
+
<span className="nre-source-ln nre-source-ln-mark" role="img" aria-label={label} title={label}>
|
|
103
|
+
<svg viewBox="0 0 24 24" aria-hidden="true">{MARK_ICONS[kind]}</svg>
|
|
104
|
+
</span>
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function LineSummary({ line, tone }: { line: AttemptSourceLineData; tone: ReturnType<typeof lineTone> }): ReactElement {
|
|
109
|
+
const interactive = line.assertions.length > 0 || line.sends.length > 0 || line.turns.length > 0;
|
|
110
|
+
const scores = thresholdScores(line.assertions);
|
|
111
|
+
return (
|
|
112
|
+
<span className="nre-source-line-summary">
|
|
113
|
+
<LineNo line={line.line} tone={tone} send={line.sends.length > 0 || line.turns.length > 0} />
|
|
114
|
+
<code className="nre-source-text">{highlightTs(line.text)}</code>
|
|
115
|
+
<span className="nre-source-line-meta">
|
|
116
|
+
{scores.length > 0 ? <span className="nre-source-score-badge">{scores.join(", ")}</span> : null}
|
|
117
|
+
{interactive ? <span className="nre-source-chevron">›</span> : null}
|
|
118
|
+
</span>
|
|
119
|
+
</span>
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function TurnDetail({ turn, showMeta = false, showSent = false }: { turn: AttemptSourceTurn; showMeta?: boolean; showSent?: boolean }): ReactElement {
|
|
124
|
+
return (
|
|
125
|
+
<div className={cx("nre-source-turn", `nre-source-turn-${turn.status}`)}>
|
|
126
|
+
{showMeta ? (
|
|
127
|
+
<div className="nre-source-turn-head">
|
|
128
|
+
<span>{turn.label}</span>
|
|
129
|
+
<span>{turn.status}</span>
|
|
130
|
+
{turn.durationMs === undefined ? null : <span>{formatDuration(turn.durationMs)}</span>}
|
|
131
|
+
</div>
|
|
132
|
+
) : null}
|
|
133
|
+
{showSent && turn.sentText ? <div className="nre-conv-sent">{turn.sentText}</div> : null}
|
|
134
|
+
<ConversationReplies replies={turn.replies} />
|
|
135
|
+
</div>
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function AssertionDetail({ assertion }: { assertion: AssertionResult }): ReactElement {
|
|
140
|
+
return (
|
|
141
|
+
<div className={`nre-assertion-row nre-tone-${assertTone(assertion)}`}>
|
|
142
|
+
<div className="nre-source-assertion-head">
|
|
143
|
+
<span className="nre-assertion-badge">{assertion.outcome}</span>
|
|
144
|
+
<span className="nre-assertion-name">{assertion.name}</span>
|
|
145
|
+
{assertion.outcome !== "unavailable" ? <span className="nre-source-assertion-score">{assertion.score}</span> : null}
|
|
146
|
+
</div>
|
|
147
|
+
{assertion.detail ? <div className="nre-assertion-detail">{assertion.detail}</div> : null}
|
|
148
|
+
{assertion.outcome === "unavailable" ? (
|
|
149
|
+
<div className="nre-assertion-body">reason: {assertion.reason}</div>
|
|
150
|
+
) : assertion.expected !== undefined || assertion.received !== undefined || assertion.evidence !== undefined ? (
|
|
151
|
+
<div className="nre-assertion-body">
|
|
152
|
+
{assertion.expected !== undefined ? <span>expected: {assertion.expected}</span> : null}
|
|
153
|
+
{assertion.received !== undefined ? <span>received: {assertion.received}</span> : null}
|
|
154
|
+
{assertion.evidence !== undefined ? <span>evidence: {assertion.evidence}</span> : null}
|
|
155
|
+
</div>
|
|
156
|
+
) : null}
|
|
157
|
+
</div>
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
17
161
|
export function AttemptSource({ data, className }: { data: AttemptSourceData | null; className?: string }): ReactElement | null {
|
|
18
162
|
if (data === null) return null;
|
|
163
|
+
const firstAttentionLine = data.lines.find((line) => {
|
|
164
|
+
const tone = lineTone(line);
|
|
165
|
+
return tone === "bad" || tone === "warn" || tone === "na";
|
|
166
|
+
})?.line;
|
|
19
167
|
return (
|
|
20
168
|
<div className={cx("nre", "nre-attempt-source", className)}>
|
|
21
169
|
<div className="nre-attempt-source-head">{data.sourcePath}</div>
|
|
22
170
|
<div className="nre-attempt-source-lines">
|
|
23
171
|
{data.lines.map((line) => {
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
172
|
+
const tone = lineTone(line);
|
|
173
|
+
const interactive = line.assertions.length > 0 || line.sends.length > 0 || line.turns.length > 0;
|
|
174
|
+
const lineClass = cx(
|
|
175
|
+
"nre-source-line",
|
|
176
|
+
tone ? `nre-tone-${tone}` : undefined,
|
|
177
|
+
line.sends.length > 0 || line.turns.length > 0 ? "nre-source-line-send" : undefined,
|
|
178
|
+
);
|
|
179
|
+
if (!interactive) {
|
|
180
|
+
return (
|
|
181
|
+
<div key={line.line} className={lineClass}>
|
|
182
|
+
<LineSummary line={line} tone={tone} />
|
|
183
|
+
</div>
|
|
184
|
+
);
|
|
185
|
+
}
|
|
35
186
|
return (
|
|
36
|
-
<details
|
|
37
|
-
key={line.line}
|
|
38
|
-
className={cx("nre-source-line", worstTone ? `nre-tone-${worstTone}` : undefined, hasSends ? "nre-source-line-send" : undefined)}
|
|
39
|
-
open={hasAsserts && (worstTone === "bad" || worstTone === "warn")}
|
|
40
|
-
>
|
|
187
|
+
<details key={line.line} className={lineClass} open={line.line === firstAttentionLine}>
|
|
41
188
|
<summary>
|
|
42
|
-
<
|
|
43
|
-
<code className="nre-source-text">{line.text}</code>
|
|
44
|
-
{hasSends ? <span className="nre-source-send-mark">↳ {line.sends.map((s) => s.label).join(", ")}</span> : null}
|
|
189
|
+
<LineSummary line={line} tone={tone} />
|
|
45
190
|
</summary>
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
{
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
<>
|
|
55
|
-
{a.expected !== undefined ? <span>expected: {a.expected}</span> : null}
|
|
56
|
-
{a.received !== undefined ? <span>received: {a.received}</span> : null}
|
|
57
|
-
</>
|
|
58
|
-
)}
|
|
59
|
-
</div>
|
|
60
|
-
))}
|
|
61
|
-
</div>
|
|
62
|
-
) : null}
|
|
191
|
+
<div className="nre-source-line-detail">
|
|
192
|
+
{line.turns.map((turn, i) => (
|
|
193
|
+
<TurnDetail key={`${turn.label}-${i}`} turn={turn} />
|
|
194
|
+
))}
|
|
195
|
+
{line.assertions.map((assertion, i) => (
|
|
196
|
+
<AssertionDetail key={i} assertion={assertion} />
|
|
197
|
+
))}
|
|
198
|
+
</div>
|
|
63
199
|
</details>
|
|
64
200
|
);
|
|
65
201
|
})}
|
|
@@ -75,6 +211,14 @@ export function AttemptSource({ data, className }: { data: AttemptSourceData | n
|
|
|
75
211
|
))}
|
|
76
212
|
</div>
|
|
77
213
|
) : null}
|
|
214
|
+
{data.unlocatedTurns.length > 0 ? (
|
|
215
|
+
<div className="nre-attempt-source-unlocated">
|
|
216
|
+
<div className="nre-attempt-source-unmapped-head">Other conversation</div>
|
|
217
|
+
{data.unlocatedTurns.map((turn, i) => (
|
|
218
|
+
<TurnDetail key={`${turn.label}-${i}`} turn={turn} showMeta showSent />
|
|
219
|
+
))}
|
|
220
|
+
</div>
|
|
221
|
+
) : null}
|
|
78
222
|
</div>
|
|
79
223
|
);
|
|
80
224
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
2
|
// Attempt 详情组件族的单元测试:11 个叶子的非空/空证据矩阵、AttemptAssessment 的
|
|
3
3
|
// source/assertions fallback、AttemptDetail 的内建顺序、spec/data 等价与 scope-input page
|
|
4
|
-
// 报错、AttemptConversation 的 loc
|
|
4
|
+
// 报错、AttemptConversation 的 loc 分轮、attemptSourceData 的 loc 投影、AttemptTimeline 的默认折叠。
|
|
5
5
|
// 纯渲染,注入数据:直接构造 AttemptEvidence fixture,不 mock fetch(这些组件从不 fetch)。
|
|
6
|
+
// 样式与视觉交互(染色、布局、展开观感)不在本层:归 E2E 报告域(docs/engineering/e2e-ci/results.md)。
|
|
6
7
|
|
|
7
8
|
import { renderToStaticMarkup } from "react-dom/server";
|
|
8
9
|
import { describe, expect, it } from "vitest";
|
|
@@ -310,14 +311,20 @@ describe("AttemptAssessment / AttemptDetail(组合组件)", () => {
|
|
|
310
311
|
await expect(resolveOnScopePage(<AttemptAssessment />)).rejects.toThrow(/attempt-input page/);
|
|
311
312
|
});
|
|
312
313
|
|
|
313
|
-
it("AttemptDetail
|
|
314
|
+
it("AttemptDetail:有 source 时不重复 Conversation,无 source 时在 usage 后保留 fallback", () => {
|
|
314
315
|
// AttemptDetail 自己是组合组件:resolve 会把它(以及嵌套的 AttemptAssessment)递归展开,
|
|
315
316
|
// 所以这里直接检查它的 compose 函数产出的原始树(与「内建报告」测试检查 standard.tsx
|
|
316
317
|
// 原始声明同一手法),不走完整 resolve——那样 AttemptAssessment 会被替换成它自己展开出的
|
|
317
318
|
// <Col> 而不再是 AttemptAssessment 这个类型。
|
|
318
319
|
const compose = composeOf(AttemptDetail)!;
|
|
319
|
-
const
|
|
320
|
-
|
|
320
|
+
const childTypes = (evidence: AttemptEvidence): unknown[] => {
|
|
321
|
+
const tree = compose({}, { page: { input: "attempt", evidence } } as never) as unknown as {
|
|
322
|
+
props: { children: Array<{ type: unknown } | null> };
|
|
323
|
+
};
|
|
324
|
+
return tree.props.children.filter((child): child is { type: unknown } => child !== null).map((child) => child.type);
|
|
325
|
+
};
|
|
326
|
+
const withoutSource = childTypes(evidenceOf());
|
|
327
|
+
expect(withoutSource).toEqual([
|
|
321
328
|
AttemptSummary,
|
|
322
329
|
AttemptAssessment,
|
|
323
330
|
AttemptFixPrompt,
|
|
@@ -328,6 +335,30 @@ describe("AttemptAssessment / AttemptDetail(组合组件)", () => {
|
|
|
328
335
|
AttemptTrace,
|
|
329
336
|
AttemptDiff,
|
|
330
337
|
]);
|
|
338
|
+
|
|
339
|
+
const withSource = childTypes(
|
|
340
|
+
evidenceOf({
|
|
341
|
+
capabilities: { ...NO_CAPS, source: true },
|
|
342
|
+
evalSource: {
|
|
343
|
+
sourcePath: "evals/a.ts",
|
|
344
|
+
sourceSha256: "x",
|
|
345
|
+
lines: [{ line: 1, text: "", assertions: [], sends: [] }],
|
|
346
|
+
unmapped: [],
|
|
347
|
+
summary: {
|
|
348
|
+
totalAssertions: 0,
|
|
349
|
+
mappedAssertions: 0,
|
|
350
|
+
unmappedAssertions: 0,
|
|
351
|
+
passed: 0,
|
|
352
|
+
failed: 0,
|
|
353
|
+
gate: 0,
|
|
354
|
+
soft: 0,
|
|
355
|
+
totalLines: 1,
|
|
356
|
+
annotatedLines: 0,
|
|
357
|
+
},
|
|
358
|
+
},
|
|
359
|
+
}),
|
|
360
|
+
);
|
|
361
|
+
expect(withSource).toEqual(withoutSource.filter((type) => type !== AttemptConversation));
|
|
331
362
|
});
|
|
332
363
|
});
|
|
333
364
|
|
|
@@ -455,6 +486,53 @@ describe("AttemptConversation:标准事件流按 loc 分轮", () => {
|
|
|
455
486
|
});
|
|
456
487
|
});
|
|
457
488
|
|
|
489
|
+
// bug: memory/attempt-detail-components-shipped-without-styles.md
|
|
490
|
+
describe("attemptSourceData:标准事件流按 loc 投影回 send 行", () => {
|
|
491
|
+
it("send 行的 turns 携带 sentText 与按序归并的完整回复", () => {
|
|
492
|
+
const sourcePath = "evals/a.ts";
|
|
493
|
+
const data = attemptSourceData(
|
|
494
|
+
evidenceOf({
|
|
495
|
+
capabilities: { ...NO_CAPS, source: true, execution: true },
|
|
496
|
+
evalSource: {
|
|
497
|
+
sourcePath,
|
|
498
|
+
sourceSha256: "x",
|
|
499
|
+
lines: [
|
|
500
|
+
{ line: 1, text: 'import { defineEval } from "niceeval";', assertions: [], sends: [] },
|
|
501
|
+
{
|
|
502
|
+
line: 2,
|
|
503
|
+
text: 'const reply = await t.send("hello");',
|
|
504
|
+
assertions: [],
|
|
505
|
+
sends: [{ label: "s1/t1", status: "completed" as const, durationMs: 120, loc: { file: sourcePath, line: 2 } }],
|
|
506
|
+
},
|
|
507
|
+
],
|
|
508
|
+
unmapped: [],
|
|
509
|
+
summary: {
|
|
510
|
+
totalAssertions: 0,
|
|
511
|
+
mappedAssertions: 0,
|
|
512
|
+
unmappedAssertions: 0,
|
|
513
|
+
passed: 0,
|
|
514
|
+
failed: 0,
|
|
515
|
+
gate: 0,
|
|
516
|
+
soft: 0,
|
|
517
|
+
totalLines: 2,
|
|
518
|
+
annotatedLines: 1,
|
|
519
|
+
},
|
|
520
|
+
},
|
|
521
|
+
events: [
|
|
522
|
+
{ type: "message", role: "user", text: "hello", loc: { file: sourcePath, line: 2 } },
|
|
523
|
+
{ type: "message", role: "assistant", text: "assistant reply attached to the source line" },
|
|
524
|
+
],
|
|
525
|
+
}),
|
|
526
|
+
)!;
|
|
527
|
+
|
|
528
|
+
expect(data.lines[1]!.turns[0]).toMatchObject({ label: "s1/t1", sentText: "hello" });
|
|
529
|
+
expect(data.lines[1]!.turns[0]!.replies).toEqual([
|
|
530
|
+
{ kind: "assistant", text: "assistant reply attached to the source line" },
|
|
531
|
+
]);
|
|
532
|
+
expect(data.lines[0]!.turns).toEqual([]);
|
|
533
|
+
});
|
|
534
|
+
});
|
|
535
|
+
|
|
458
536
|
// ───────────────────────── AttemptTimeline:默认折叠 ─────────────────────────
|
|
459
537
|
|
|
460
538
|
describe("AttemptTimeline:默认只显示主链,children 收合", () => {
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
AttemptErrorData,
|
|
19
19
|
AttemptFixPromptData,
|
|
20
20
|
AttemptSourceData,
|
|
21
|
+
AttemptSourceTurn,
|
|
21
22
|
AttemptSummaryData,
|
|
22
23
|
AttemptTimelineData,
|
|
23
24
|
AttemptTraceData,
|
|
@@ -71,7 +72,55 @@ export function attemptAssertionsData(evidence: AttemptEvidence): AttemptAsserti
|
|
|
71
72
|
export function attemptSourceData(evidence: AttemptEvidence): AttemptSourceData | null {
|
|
72
73
|
if (!evidence.capabilities.source || evidence.evalSource === null) return null;
|
|
73
74
|
const { sourcePath, lines, unmapped, summary } = evidence.evalSource;
|
|
74
|
-
|
|
75
|
+
const projectedLines = lines.map((line) => ({
|
|
76
|
+
...line,
|
|
77
|
+
turns: line.sends.map<AttemptSourceTurn>((send) => ({
|
|
78
|
+
label: send.label,
|
|
79
|
+
status: send.status,
|
|
80
|
+
...(send.durationMs === undefined ? {} : { durationMs: send.durationMs }),
|
|
81
|
+
sentText: "",
|
|
82
|
+
replies: [],
|
|
83
|
+
})),
|
|
84
|
+
}));
|
|
85
|
+
const usedTurns = new Map<number, number>();
|
|
86
|
+
const unlocatedTurns: AttemptSourceTurn[] = [];
|
|
87
|
+
const conversation = attemptConversationData(evidence);
|
|
88
|
+
|
|
89
|
+
for (const [roundIndex, round] of (conversation?.rounds ?? []).entries()) {
|
|
90
|
+
const status = round.replies.some(
|
|
91
|
+
(reply) =>
|
|
92
|
+
reply.kind === "error" ||
|
|
93
|
+
((reply.kind === "tool" || reply.kind === "subagent") && reply.status === "failed"),
|
|
94
|
+
)
|
|
95
|
+
? "failed"
|
|
96
|
+
: round.replies.some((reply) => reply.kind === "input")
|
|
97
|
+
? "waiting"
|
|
98
|
+
: "completed";
|
|
99
|
+
const fallback: AttemptSourceTurn = {
|
|
100
|
+
label: `t${roundIndex + 1}`,
|
|
101
|
+
status,
|
|
102
|
+
sentText: round.sentText,
|
|
103
|
+
replies: round.replies,
|
|
104
|
+
};
|
|
105
|
+
const loc = round.loc;
|
|
106
|
+
if (!loc || loc.file !== sourcePath || loc.line < 1 || loc.line > projectedLines.length) {
|
|
107
|
+
unlocatedTurns.push(fallback);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const line = projectedLines[loc.line - 1]!;
|
|
112
|
+
const turnIndex = usedTurns.get(loc.line) ?? 0;
|
|
113
|
+
usedTurns.set(loc.line, turnIndex + 1);
|
|
114
|
+
const annotated = line.turns[turnIndex];
|
|
115
|
+
if (annotated) {
|
|
116
|
+
annotated.sentText = round.sentText;
|
|
117
|
+
annotated.replies = round.replies;
|
|
118
|
+
} else {
|
|
119
|
+
line.turns.push(fallback);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return { locator: evidence.locator, sourcePath, lines: projectedLines, unmapped, unlocatedTurns, summary };
|
|
75
124
|
}
|
|
76
125
|
|
|
77
126
|
// ───────────────────────── AttemptFixPrompt ─────────────────────────
|
|
@@ -107,6 +107,9 @@ export function attemptSourceText(data: AttemptSourceData | null, ctx: TextConte
|
|
|
107
107
|
const command = evidenceCommand(ctx, data.locator, "--source");
|
|
108
108
|
const headerParts = [`${data.sourcePath} · ${data.summary.annotatedLines}/${data.summary.totalLines} lines annotated`];
|
|
109
109
|
if (command) headerParts.push(command);
|
|
110
|
+
const hasConversation = data.unlocatedTurns.length > 0 || data.lines.some((line) => line.turns.length > 0);
|
|
111
|
+
const executionCommand = hasConversation ? evidenceCommand(ctx, data.locator, "--execution") : null;
|
|
112
|
+
if (executionCommand) headerParts.push(executionCommand);
|
|
110
113
|
// 源码锚由 assertionLine 自己按 a.loc 拼(与 AttemptAssertions 共用同一份逻辑);这里只负责
|
|
111
114
|
// 挑出非 passed 的条目,不重复算锚点。
|
|
112
115
|
const failed = data.lines.flatMap((line) => line.assertions.filter((a) => a.outcome !== "passed"));
|
|
@@ -261,13 +261,24 @@ export const AttemptAssertions = makeAttemptComponent<AttemptAssertionsData>({
|
|
|
261
261
|
|
|
262
262
|
/** AnnotatedSourceLine(src/results/annotated-source.ts):一行源码 + 映射到这一行的断言 / send 标注。 */
|
|
263
263
|
function annotatedSourceLineProblem(value: unknown, path: string): string | null {
|
|
264
|
-
if (!isObject(value)) return `"${path}" must be an
|
|
264
|
+
if (!isObject(value)) return `"${path}" must be an AttemptSourceLineData { line, text, assertions, sends, turns }`;
|
|
265
265
|
if (typeof value.line !== "number") return `"${path}.line" must be a number`;
|
|
266
266
|
if (typeof value.text !== "string") return `"${path}.text" must be a string`;
|
|
267
267
|
const assertionsProblem = arrayProblem(value.assertions, `${path}.assertions`, assertionResultProblem);
|
|
268
268
|
if (assertionsProblem !== null) return assertionsProblem;
|
|
269
269
|
if (!Array.isArray(value.sends)) return `"${path}.sends" must be an array`;
|
|
270
|
-
return
|
|
270
|
+
return arrayProblem(value.turns, `${path}.turns`, sourceTurnProblem);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function sourceTurnProblem(value: unknown, path: string): string | null {
|
|
274
|
+
if (!isObject(value)) return `"${path}" must be an AttemptSourceTurn`;
|
|
275
|
+
if (typeof value.label !== "string") return `"${path}.label" must be a string`;
|
|
276
|
+
if (value.status !== "completed" && value.status !== "failed" && value.status !== "waiting") {
|
|
277
|
+
return `"${path}.status" must be "completed", "failed", or "waiting"`;
|
|
278
|
+
}
|
|
279
|
+
if (value.durationMs !== undefined && typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
|
|
280
|
+
if (typeof value.sentText !== "string") return `"${path}.sentText" must be a string`;
|
|
281
|
+
return arrayProblem(value.replies, `${path}.replies`, conversationReplyProblem);
|
|
271
282
|
}
|
|
272
283
|
|
|
273
284
|
/** AnnotatedEvalSourceSummary(src/results/annotated-source.ts):全是计数字段。 */
|
|
@@ -297,6 +308,8 @@ export function validateSourceData(data: unknown): string | null {
|
|
|
297
308
|
if (linesProblem !== null) return linesProblem;
|
|
298
309
|
const unmappedProblem = arrayProblem(data.unmapped, "unmapped", assertionResultProblem);
|
|
299
310
|
if (unmappedProblem !== null) return unmappedProblem;
|
|
311
|
+
const turnsProblem = arrayProblem(data.unlocatedTurns, "unlocatedTurns", sourceTurnProblem);
|
|
312
|
+
if (turnsProblem !== null) return turnsProblem;
|
|
300
313
|
return sourceSummaryProblem(data.summary, "summary");
|
|
301
314
|
}
|
|
302
315
|
|
|
@@ -566,18 +579,22 @@ export const AttemptAssessment = defineComponent((_props: Record<string, never>,
|
|
|
566
579
|
});
|
|
567
580
|
AttemptAssessment.displayName = "AttemptAssessment";
|
|
568
581
|
|
|
569
|
-
/**
|
|
570
|
-
export const AttemptDetail = defineComponent(() =>
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
<
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
582
|
+
/** 内建排列顺序;有 source 时回复已按 loc 展开在 AttemptSource 行内,不再重复一份 round 卡。 */
|
|
583
|
+
export const AttemptDetail = defineComponent((_props: Record<string, never>, ctx) => {
|
|
584
|
+
const conversationLivesInSource =
|
|
585
|
+
ctx.page.input === "attempt" && ctx.page.evidence.capabilities.source && ctx.page.evidence.evalSource !== null;
|
|
586
|
+
return (
|
|
587
|
+
<Col>
|
|
588
|
+
<AttemptSummary />
|
|
589
|
+
<AttemptAssessment />
|
|
590
|
+
<AttemptFixPrompt />
|
|
591
|
+
<AttemptTimeline />
|
|
592
|
+
<AttemptDiagnostics />
|
|
593
|
+
<AttemptUsage />
|
|
594
|
+
{conversationLivesInSource ? null : <AttemptConversation />}
|
|
595
|
+
<AttemptTrace />
|
|
596
|
+
<AttemptDiff />
|
|
597
|
+
</Col>
|
|
598
|
+
);
|
|
599
|
+
});
|
|
583
600
|
AttemptDetail.displayName = "AttemptDetail";
|
|
@@ -106,8 +106,9 @@ describe("validateSourceData", () => {
|
|
|
106
106
|
const valid = {
|
|
107
107
|
locator: "@1abcdef2",
|
|
108
108
|
sourcePath: "eval.ts",
|
|
109
|
-
lines: [{ line: 1, text: "t.send(...)", assertions: [], sends: [] }],
|
|
109
|
+
lines: [{ line: 1, text: "t.send(...)", assertions: [], sends: [], turns: [] }],
|
|
110
110
|
unmapped: [],
|
|
111
|
+
unlocatedTurns: [],
|
|
111
112
|
summary: validSummary,
|
|
112
113
|
};
|
|
113
114
|
|
|
@@ -126,9 +127,25 @@ describe("validateSourceData", () => {
|
|
|
126
127
|
});
|
|
127
128
|
|
|
128
129
|
it("lines[i].assertions 嵌套断言结构错误报错", () => {
|
|
129
|
-
const bad = { ...valid, lines: [{ line: 1, text: "x", assertions: [{ name: "eq" }], sends: [] }] };
|
|
130
|
+
const bad = { ...valid, lines: [{ line: 1, text: "x", assertions: [{ name: "eq" }], sends: [], turns: [] }] };
|
|
130
131
|
expect(validateSourceData(bad)).toMatch(/"lines\[0\]\.assertions\[0\]\.severity"/);
|
|
131
132
|
});
|
|
133
|
+
|
|
134
|
+
it("lines[i].turns[j].replies[k] 递归校验回复判别联合", () => {
|
|
135
|
+
const bad = {
|
|
136
|
+
...valid,
|
|
137
|
+
lines: [
|
|
138
|
+
{
|
|
139
|
+
line: 1,
|
|
140
|
+
text: "x",
|
|
141
|
+
assertions: [],
|
|
142
|
+
sends: [],
|
|
143
|
+
turns: [{ label: "s1/t1", status: "completed", sentText: "go", replies: [{ kind: "assistant" }] }],
|
|
144
|
+
},
|
|
145
|
+
],
|
|
146
|
+
};
|
|
147
|
+
expect(validateSourceData(bad)).toMatch(/"lines\[0\]\.turns\[0\]\.replies\[0\]\.text"/);
|
|
148
|
+
});
|
|
132
149
|
});
|
|
133
150
|
|
|
134
151
|
describe("validateTimelineData", () => {
|