niceeval 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +77 -45
- package/dist/agents/types.d.ts +28 -6
- package/dist/i18n/en.d.ts +2 -0
- package/dist/i18n/zh-CN.d.ts +2 -0
- package/dist/report/built-in/index.d.ts +3 -2
- package/dist/report/built-in/index.js +7 -8
- package/dist/report/built-in/standard.d.ts +1 -0
- package/dist/report/built-in/standard.js +30 -0
- package/dist/report/components.d.ts +72 -6
- package/dist/report/components.js +159 -10
- package/dist/report/compute.d.ts +31 -4
- package/dist/report/compute.js +131 -12
- package/dist/report/index.d.ts +4 -4
- package/dist/report/index.js +3 -2
- package/dist/report/locale.d.ts +39 -1
- package/dist/report/locale.js +69 -0
- package/dist/report/react/AttemptList.d.ts +3 -1
- package/dist/report/react/AttemptList.js +3 -3
- package/dist/report/react/CopyFixPrompt.d.ts +12 -0
- package/dist/report/react/CopyFixPrompt.js +12 -0
- package/dist/report/react/HeroCard.d.ts +13 -0
- package/dist/report/react/HeroCard.js +35 -0
- package/dist/report/react/PoweredBy.d.ts +5 -0
- package/dist/report/react/PoweredBy.js +7 -0
- package/dist/report/react/ScopeWarnings.d.ts +12 -0
- package/dist/report/react/ScopeWarnings.js +18 -0
- package/dist/report/react/TraceWaterfall.d.ts +14 -0
- package/dist/report/react/TraceWaterfall.js +22 -0
- package/dist/report/react/index.d.ts +6 -1
- package/dist/report/react/index.js +6 -0
- package/dist/report/report.d.ts +22 -6
- package/dist/report/report.js +66 -53
- package/dist/report/scope-warnings.d.ts +28 -0
- package/dist/report/scope-warnings.js +101 -0
- package/dist/report/text/faces.d.ts +19 -1
- package/dist/report/text/faces.js +61 -0
- package/dist/report/tree.js +6 -1
- package/dist/report/types.d.ts +45 -10
- package/dist/report/web.d.ts +5 -4
- package/dist/report/web.js +7 -22
- package/dist/results/select.d.ts +15 -2
- package/dist/results/select.js +75 -10
- package/dist/results/types.d.ts +26 -11
- package/dist/runner/fingerprint.d.ts +3 -3
- package/dist/runner/sandbox-selection.d.ts +12 -0
- package/dist/runner/types.d.ts +18 -6
- package/dist/sandbox/types.d.ts +12 -0
- package/docs-site/zh/explanation/evals.mdx +2 -1
- package/docs-site/zh/explanation/experiment.mdx +2 -0
- package/docs-site/zh/how-to/custom-reports.mdx +22 -6
- package/docs-site/zh/how-to/publish-report.mdx +6 -8
- package/docs-site/zh/how-to/viewing-results.mdx +3 -3
- package/docs-site/zh/how-to/write-experiment.mdx +40 -1
- package/docs-site/zh/reference/builtin-agents.mdx +40 -3
- package/docs-site/zh/reference/cli.mdx +1 -2
- package/docs-site/zh/reference/define-eval.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +10 -5
- package/docs-site/zh/reference/report-components.mdx +2 -2
- package/docs-site/zh/reference/results-data.mdx +2 -4
- package/package.json +2 -1
- package/src/agents/bub.ts +13 -1
- package/src/agents/claude-code.test.ts +43 -1
- package/src/agents/claude-code.ts +32 -14
- package/src/agents/codex.test.ts +168 -1
- package/src/agents/codex.ts +51 -15
- package/src/agents/mcp.ts +31 -0
- package/src/agents/post-setup.ts +33 -0
- package/src/agents/types.ts +28 -7
- package/src/cli.ts +9 -11
- package/src/context/context.ts +11 -5
- package/src/define.ts +3 -0
- package/src/i18n/en.ts +5 -2
- package/src/i18n/zh-CN.ts +5 -1
- package/src/index.ts +1 -0
- package/src/report/built-in/index.tsx +8 -7
- package/src/report/built-in/standard.tsx +59 -0
- package/src/report/components.tsx +231 -17
- package/src/report/compute.ts +146 -21
- package/src/report/dual-render.test.tsx +139 -12
- package/src/report/index.ts +20 -1
- package/src/report/locale.ts +83 -1
- package/src/report/react/AttemptList.tsx +13 -1
- package/src/report/react/CopyFixPrompt.tsx +37 -0
- package/src/report/react/HeroCard.tsx +59 -0
- package/src/report/react/PoweredBy.tsx +20 -0
- package/src/report/react/ScopeWarnings.tsx +74 -0
- package/src/report/react/TraceWaterfall.tsx +78 -0
- package/src/report/react/enhance.js +14 -0
- package/src/report/react/index.tsx +11 -0
- package/src/report/react/styles.css +187 -7
- package/src/report/report.test.ts +2 -20
- package/src/report/report.ts +97 -62
- package/src/report/scope-warnings.ts +155 -0
- package/src/report/site-components.test.tsx +526 -0
- package/src/report/text/faces.ts +66 -0
- package/src/report/tree.ts +8 -1
- package/src/report/types.ts +51 -11
- package/src/report/web.ts +7 -40
- package/src/results/copy.ts +15 -78
- package/src/results/host-equivalence.test.ts +5 -1
- package/src/results/open.ts +5 -4
- package/src/results/publish.ts +4 -146
- package/src/results/results.test.ts +86 -9
- package/src/results/select.ts +78 -10
- package/src/results/types.ts +27 -7
- package/src/runner/attempt.ts +10 -9
- package/src/runner/discover.test.ts +9 -1
- package/src/runner/discover.ts +3 -3
- package/src/runner/fingerprint.ts +9 -4
- package/src/runner/ledger.test.ts +30 -1
- package/src/runner/ledger.ts +26 -4
- package/src/runner/run.ts +5 -1
- package/src/runner/sandbox-selection.test.ts +131 -0
- package/src/runner/sandbox-selection.ts +110 -0
- package/src/runner/types.ts +19 -2
- package/src/sandbox/types.ts +6 -0
- package/src/show/index.ts +17 -10
- package/src/show/render.ts +11 -11
- package/src/show/report-host.test.ts +32 -15
- package/src/show/report-host.ts +5 -4
- package/src/show/show.test.ts +140 -3
- package/src/view/app/App.test.tsx +78 -17
- package/src/view/app/App.tsx +17 -78
- package/src/view/app/components/CopyControls.tsx +4 -42
- package/src/view/app/i18n.ts +5 -227
- package/src/view/app/lib/rows.ts +3 -21
- package/src/view/app/shared.ts +1 -3
- package/src/view/app/types.ts +2 -2
- package/src/view/artifact-serving.test.ts +1 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.ts +2 -13
- package/src/view/index.ts +1 -12
- package/src/view/server.ts +0 -2
- package/src/view/shared/types.ts +11 -6
- package/src/view/site-parity.test.ts +1 -1
- package/src/view/site.ts +1 -1
- package/src/view/styles.css +6 -266
- package/src/view/view-report.test.ts +64 -27
- package/src/view/app/components/LazyArtifact.tsx +0 -51
- package/src/view/app/components/SkippedRunsBanner.tsx +0 -140
- package/src/view/app/pages/AttemptsPage.tsx +0 -80
- package/src/view/app/pages/TracesPage.tsx +0 -35
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// ScopeWarnings 的聚合层:把 Scope 警告按「下一步动作」组织成组,web / text 两面共用
|
|
2
|
+
// (docs/feature/reports/library/site-components.md「聚合轴是动作,不是发生顺序」)。
|
|
3
|
+
// message 是完整叙述的单源,这里只组织、不改写;徽标 / 组头文案按 kind 表登记的模板
|
|
4
|
+
// (docs/feature/results/library.md「警告 kind 全集」)经 locale 词典渲染,未登记的 kind
|
|
5
|
+
// 回退为单独成组、逐条 message 原样。
|
|
6
|
+
|
|
7
|
+
import type { ScopeWarning } from "../results/types.ts";
|
|
8
|
+
import { gapParts } from "../results/select.ts";
|
|
9
|
+
import { localeText, type ReportLocale, type ReportMessageKey } from "./locale.ts";
|
|
10
|
+
|
|
11
|
+
/** kind 表登记的类别:integrity(选中集合的分母可能不对)组排在 freshness(可能过期)之前。 */
|
|
12
|
+
export type WarningCategory = "integrity" | "freshness";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* 聚合层的宽松形态:kind 表新增 kind 时(如 unreadable-snapshot 先于类型落地)
|
|
16
|
+
* 不假设字段全集,读结构化字段前逐个判形。
|
|
17
|
+
*/
|
|
18
|
+
interface AnyWarning {
|
|
19
|
+
kind: string;
|
|
20
|
+
message: string;
|
|
21
|
+
command?: string;
|
|
22
|
+
experimentId?: string;
|
|
23
|
+
[key: string]: unknown;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ScopeWarningGroup {
|
|
27
|
+
category: WarningCategory;
|
|
28
|
+
/** 实验组为 experimentId;kind 组为登记的组头文案(含条数);未登记 kind 用 kind 原文。 */
|
|
29
|
+
title: string;
|
|
30
|
+
/** 每条警告一枚、与 warnings 同序;未登记徽标模板的成员不出徽标。 */
|
|
31
|
+
badges: readonly { kind: string; text: string }[];
|
|
32
|
+
/** 组内命令去重后恰一条时归组头(复制即推进整组);多条或零条为 null,命令随明细逐条走。 */
|
|
33
|
+
headCommand: string | null;
|
|
34
|
+
/** 原始条目(明细层,message 单源)。 */
|
|
35
|
+
warnings: readonly ScopeWarning[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface GroupedScopeWarnings {
|
|
39
|
+
/** 分类计数汇总行,任何组数下都产出;web 面用作外层折叠块的 <summary>,text 面只在多组时打印。 */
|
|
40
|
+
summary: string;
|
|
41
|
+
groups: readonly ScopeWarningGroup[];
|
|
42
|
+
/** 警告总条数 ≤ 3 时组级明细默认展开(web 面第二层 <details> 的 open;阈值是行为契约,无开关)。 */
|
|
43
|
+
detailsOpen: boolean;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const CATEGORY: Record<string, WarningCategory> = {
|
|
47
|
+
"partial-coverage": "integrity",
|
|
48
|
+
"unfinished-snapshot": "integrity",
|
|
49
|
+
"unreadable-snapshot": "integrity",
|
|
50
|
+
"stale-snapshot": "freshness",
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
/** 实验作用域且登记了徽标模板的 kind 才进实验组;其余(含未登记 kind)按 kind 聚合。 */
|
|
54
|
+
const EXPERIMENT_KINDS = new Set(["partial-coverage", "stale-snapshot", "unfinished-snapshot"]);
|
|
55
|
+
|
|
56
|
+
function pluralText(
|
|
57
|
+
locale: ReportLocale,
|
|
58
|
+
base: "warnings.summary.experiments" | "warnings.group.unreadableSnapshot" | "warnings.details",
|
|
59
|
+
n: number,
|
|
60
|
+
): string {
|
|
61
|
+
return localeText(locale, `${base}.${n === 1 ? "one" : "other"}` as ReportMessageKey, { n });
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** 明细折叠块的标签(「N 条原始警告」)。 */
|
|
65
|
+
export function warningDetailsLabel(locale: ReportLocale, n: number): string {
|
|
66
|
+
return pluralText(locale, "warnings.details", n);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function gapText(locale: ReportLocale, fromIso: string, toIso: string): string {
|
|
70
|
+
const { n, unit } = gapParts(fromIso, toIso);
|
|
71
|
+
return localeText(locale, `warnings.gap.${unit}.${n === 1 ? "one" : "other"}` as ReportMessageKey, { n });
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function badgeText(w: AnyWarning, locale: ReportLocale): string | null {
|
|
75
|
+
switch (w.kind) {
|
|
76
|
+
case "partial-coverage":
|
|
77
|
+
return localeText(locale, "warnings.badge.partialCoverage", {
|
|
78
|
+
covered: String(w.covered),
|
|
79
|
+
total: String(w.total),
|
|
80
|
+
});
|
|
81
|
+
case "stale-snapshot":
|
|
82
|
+
return localeText(locale, "warnings.badge.staleSnapshot", {
|
|
83
|
+
gap: gapText(locale, String(w.startedAt), String(w.latestStartedAt)),
|
|
84
|
+
});
|
|
85
|
+
case "unfinished-snapshot":
|
|
86
|
+
return localeText(locale, "warnings.badge.unfinishedSnapshot");
|
|
87
|
+
default:
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** 组内命令去重:恰一条时它就是「复制即推进整组」的组头命令。 */
|
|
93
|
+
function dedupeCommand(members: readonly AnyWarning[]): string | null {
|
|
94
|
+
const commands = new Set(members.map((w) => w.command).filter((c): c is string => typeof c === "string" && c !== ""));
|
|
95
|
+
return commands.size === 1 ? [...commands][0] : null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function groupCategory(members: readonly AnyWarning[]): WarningCategory {
|
|
99
|
+
return members.some((w) => (CATEGORY[w.kind] ?? "integrity") === "integrity") ? "integrity" : "freshness";
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function groupScopeWarnings(input: readonly ScopeWarning[], locale: ReportLocale): GroupedScopeWarnings {
|
|
103
|
+
const warnings = input as readonly AnyWarning[];
|
|
104
|
+
const byExperiment = new Map<string, AnyWarning[]>();
|
|
105
|
+
const byKind = new Map<string, AnyWarning[]>();
|
|
106
|
+
for (const w of warnings) {
|
|
107
|
+
if (EXPERIMENT_KINDS.has(w.kind) && typeof w.experimentId === "string") {
|
|
108
|
+
const members = byExperiment.get(w.experimentId) ?? [];
|
|
109
|
+
members.push(w);
|
|
110
|
+
byExperiment.set(w.experimentId, members);
|
|
111
|
+
} else {
|
|
112
|
+
const members = byKind.get(w.kind) ?? [];
|
|
113
|
+
members.push(w);
|
|
114
|
+
byKind.set(w.kind, members);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const groups: ScopeWarningGroup[] = [];
|
|
119
|
+
for (const [experimentId, members] of byExperiment) {
|
|
120
|
+
groups.push({
|
|
121
|
+
category: groupCategory(members),
|
|
122
|
+
title: experimentId,
|
|
123
|
+
badges: members
|
|
124
|
+
.map((w) => ({ kind: w.kind, text: badgeText(w, locale) }))
|
|
125
|
+
.filter((b): b is { kind: string; text: string } => b.text !== null),
|
|
126
|
+
headCommand: dedupeCommand(members),
|
|
127
|
+
warnings: members as unknown as ScopeWarning[],
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
for (const [kind, members] of byKind) {
|
|
131
|
+
groups.push({
|
|
132
|
+
category: CATEGORY[kind] ?? "integrity",
|
|
133
|
+
title:
|
|
134
|
+
kind === "unreadable-snapshot" ? pluralText(locale, "warnings.group.unreadableSnapshot", members.length) : kind,
|
|
135
|
+
badges: [],
|
|
136
|
+
headCommand: dedupeCommand(members),
|
|
137
|
+
warnings: members as unknown as ScopeWarning[],
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
// 稳定排序:integrity 在前,同类别保持首次出现顺序。
|
|
141
|
+
const rank = (c: WarningCategory) => (c === "integrity" ? 0 : 1);
|
|
142
|
+
groups.sort((a, b) => rank(a.category) - rank(b.category));
|
|
143
|
+
|
|
144
|
+
const parts: string[] = [];
|
|
145
|
+
if (byExperiment.size > 0) parts.push(pluralText(locale, "warnings.summary.experiments", byExperiment.size));
|
|
146
|
+
for (const [kind, members] of byKind) {
|
|
147
|
+
parts.push(
|
|
148
|
+
kind === "unreadable-snapshot"
|
|
149
|
+
? pluralText(locale, "warnings.group.unreadableSnapshot", members.length)
|
|
150
|
+
: `${kind} ×${members.length}`,
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return { summary: parts.join(" · "), groups, detailsOpen: warnings.length <= 3 };
|
|
155
|
+
}
|