niceeval 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/INDEX.md +77 -45
  2. package/dist/agents/types.d.ts +28 -6
  3. package/dist/i18n/en.d.ts +2 -0
  4. package/dist/i18n/zh-CN.d.ts +2 -0
  5. package/dist/report/built-in/index.d.ts +3 -2
  6. package/dist/report/built-in/index.js +7 -8
  7. package/dist/report/built-in/standard.d.ts +1 -0
  8. package/dist/report/built-in/standard.js +30 -0
  9. package/dist/report/components.d.ts +72 -6
  10. package/dist/report/components.js +159 -10
  11. package/dist/report/compute.d.ts +31 -4
  12. package/dist/report/compute.js +131 -12
  13. package/dist/report/index.d.ts +4 -4
  14. package/dist/report/index.js +3 -2
  15. package/dist/report/locale.d.ts +39 -1
  16. package/dist/report/locale.js +69 -0
  17. package/dist/report/react/AttemptList.d.ts +3 -1
  18. package/dist/report/react/AttemptList.js +3 -3
  19. package/dist/report/react/CopyFixPrompt.d.ts +12 -0
  20. package/dist/report/react/CopyFixPrompt.js +12 -0
  21. package/dist/report/react/HeroCard.d.ts +13 -0
  22. package/dist/report/react/HeroCard.js +35 -0
  23. package/dist/report/react/PoweredBy.d.ts +5 -0
  24. package/dist/report/react/PoweredBy.js +7 -0
  25. package/dist/report/react/ScopeWarnings.d.ts +12 -0
  26. package/dist/report/react/ScopeWarnings.js +18 -0
  27. package/dist/report/react/TraceWaterfall.d.ts +14 -0
  28. package/dist/report/react/TraceWaterfall.js +22 -0
  29. package/dist/report/react/index.d.ts +6 -1
  30. package/dist/report/react/index.js +6 -0
  31. package/dist/report/report.d.ts +22 -6
  32. package/dist/report/report.js +66 -53
  33. package/dist/report/scope-warnings.d.ts +28 -0
  34. package/dist/report/scope-warnings.js +101 -0
  35. package/dist/report/text/faces.d.ts +19 -1
  36. package/dist/report/text/faces.js +61 -0
  37. package/dist/report/tree.js +6 -1
  38. package/dist/report/types.d.ts +45 -10
  39. package/dist/report/web.d.ts +5 -4
  40. package/dist/report/web.js +7 -22
  41. package/dist/results/select.d.ts +15 -2
  42. package/dist/results/select.js +75 -10
  43. package/dist/results/types.d.ts +26 -11
  44. package/dist/runner/fingerprint.d.ts +3 -3
  45. package/dist/runner/sandbox-selection.d.ts +12 -0
  46. package/dist/runner/types.d.ts +18 -6
  47. package/dist/sandbox/types.d.ts +12 -0
  48. package/docs-site/zh/explanation/evals.mdx +2 -1
  49. package/docs-site/zh/explanation/experiment.mdx +2 -0
  50. package/docs-site/zh/how-to/custom-reports.mdx +22 -6
  51. package/docs-site/zh/how-to/publish-report.mdx +6 -8
  52. package/docs-site/zh/how-to/viewing-results.mdx +3 -3
  53. package/docs-site/zh/how-to/write-experiment.mdx +40 -1
  54. package/docs-site/zh/reference/builtin-agents.mdx +40 -3
  55. package/docs-site/zh/reference/cli.mdx +1 -2
  56. package/docs-site/zh/reference/define-eval.mdx +8 -0
  57. package/docs-site/zh/reference/official-adapters.mdx +10 -5
  58. package/docs-site/zh/reference/report-components.mdx +2 -2
  59. package/docs-site/zh/reference/results-data.mdx +2 -4
  60. package/package.json +2 -1
  61. package/src/agents/bub.ts +13 -1
  62. package/src/agents/claude-code.test.ts +43 -1
  63. package/src/agents/claude-code.ts +32 -14
  64. package/src/agents/codex.test.ts +168 -1
  65. package/src/agents/codex.ts +51 -15
  66. package/src/agents/mcp.ts +31 -0
  67. package/src/agents/post-setup.ts +33 -0
  68. package/src/agents/types.ts +28 -7
  69. package/src/cli.ts +9 -11
  70. package/src/context/context.ts +11 -5
  71. package/src/define.ts +3 -0
  72. package/src/i18n/en.ts +5 -2
  73. package/src/i18n/zh-CN.ts +5 -1
  74. package/src/index.ts +1 -0
  75. package/src/report/built-in/index.tsx +8 -7
  76. package/src/report/built-in/standard.tsx +59 -0
  77. package/src/report/components.tsx +231 -17
  78. package/src/report/compute.ts +146 -21
  79. package/src/report/dual-render.test.tsx +139 -12
  80. package/src/report/index.ts +20 -1
  81. package/src/report/locale.ts +83 -1
  82. package/src/report/react/AttemptList.tsx +13 -1
  83. package/src/report/react/CopyFixPrompt.tsx +37 -0
  84. package/src/report/react/HeroCard.tsx +59 -0
  85. package/src/report/react/PoweredBy.tsx +20 -0
  86. package/src/report/react/ScopeWarnings.tsx +74 -0
  87. package/src/report/react/TraceWaterfall.tsx +78 -0
  88. package/src/report/react/enhance.js +14 -0
  89. package/src/report/react/index.tsx +11 -0
  90. package/src/report/react/styles.css +187 -7
  91. package/src/report/report.test.ts +2 -20
  92. package/src/report/report.ts +97 -62
  93. package/src/report/scope-warnings.ts +155 -0
  94. package/src/report/site-components.test.tsx +526 -0
  95. package/src/report/text/faces.ts +66 -0
  96. package/src/report/tree.ts +8 -1
  97. package/src/report/types.ts +51 -11
  98. package/src/report/web.ts +7 -40
  99. package/src/results/copy.ts +15 -78
  100. package/src/results/host-equivalence.test.ts +5 -1
  101. package/src/results/open.ts +5 -4
  102. package/src/results/publish.ts +4 -146
  103. package/src/results/results.test.ts +86 -9
  104. package/src/results/select.ts +78 -10
  105. package/src/results/types.ts +27 -7
  106. package/src/runner/attempt.ts +10 -9
  107. package/src/runner/discover.test.ts +9 -1
  108. package/src/runner/discover.ts +3 -3
  109. package/src/runner/fingerprint.ts +9 -4
  110. package/src/runner/ledger.test.ts +30 -1
  111. package/src/runner/ledger.ts +26 -4
  112. package/src/runner/run.ts +5 -1
  113. package/src/runner/sandbox-selection.test.ts +131 -0
  114. package/src/runner/sandbox-selection.ts +110 -0
  115. package/src/runner/types.ts +19 -2
  116. package/src/sandbox/types.ts +6 -0
  117. package/src/show/index.ts +17 -10
  118. package/src/show/render.ts +11 -11
  119. package/src/show/report-host.test.ts +32 -15
  120. package/src/show/report-host.ts +5 -4
  121. package/src/show/show.test.ts +140 -3
  122. package/src/view/app/App.test.tsx +78 -17
  123. package/src/view/app/App.tsx +17 -78
  124. package/src/view/app/components/CopyControls.tsx +4 -42
  125. package/src/view/app/i18n.ts +5 -227
  126. package/src/view/app/lib/rows.ts +3 -21
  127. package/src/view/app/shared.ts +1 -3
  128. package/src/view/app/types.ts +2 -2
  129. package/src/view/artifact-serving.test.ts +1 -1
  130. package/src/view/client-dist/app.css +1 -1
  131. package/src/view/client-dist/app.js +20 -20
  132. package/src/view/data.ts +2 -13
  133. package/src/view/index.ts +1 -12
  134. package/src/view/server.ts +0 -2
  135. package/src/view/shared/types.ts +11 -6
  136. package/src/view/site-parity.test.ts +1 -1
  137. package/src/view/site.ts +1 -1
  138. package/src/view/styles.css +6 -266
  139. package/src/view/view-report.test.ts +64 -27
  140. package/src/view/app/components/LazyArtifact.tsx +0 -51
  141. package/src/view/app/components/SkippedRunsBanner.tsx +0 -140
  142. package/src/view/app/pages/AttemptsPage.tsx +0 -80
  143. package/src/view/app/pages/TracesPage.tsx +0 -35
@@ -0,0 +1,101 @@
1
+ // ScopeWarnings 的聚合层:把 Scope 警告按「下一步动作」组织成组,web / text 两面共用
2
+ // (docs/feature/reports/library/site-components.md「聚合轴是动作,不是发生顺序」)。
3
+ // message 是完整叙述的单源,这里只组织、不改写;徽标 / 组头文案按 kind 表登记的模板
4
+ // (docs/feature/results/library.md「警告 kind 全集」)经 locale 词典渲染,未登记的 kind
5
+ // 回退为单独成组、逐条 message 原样。
6
+ import { gapParts } from "../results/select.js";
7
+ import { localeText } from "./locale.js";
8
+ const CATEGORY = {
9
+ "partial-coverage": "integrity",
10
+ "unfinished-snapshot": "integrity",
11
+ "unreadable-snapshot": "integrity",
12
+ "stale-snapshot": "freshness",
13
+ };
14
+ /** 实验作用域且登记了徽标模板的 kind 才进实验组;其余(含未登记 kind)按 kind 聚合。 */
15
+ const EXPERIMENT_KINDS = new Set(["partial-coverage", "stale-snapshot", "unfinished-snapshot"]);
16
+ function pluralText(locale, base, n) {
17
+ return localeText(locale, `${base}.${n === 1 ? "one" : "other"}`, { n });
18
+ }
19
+ /** 明细折叠块的标签(「N 条原始警告」)。 */
20
+ export function warningDetailsLabel(locale, n) {
21
+ return pluralText(locale, "warnings.details", n);
22
+ }
23
+ function gapText(locale, fromIso, toIso) {
24
+ const { n, unit } = gapParts(fromIso, toIso);
25
+ return localeText(locale, `warnings.gap.${unit}.${n === 1 ? "one" : "other"}`, { n });
26
+ }
27
+ function badgeText(w, locale) {
28
+ switch (w.kind) {
29
+ case "partial-coverage":
30
+ return localeText(locale, "warnings.badge.partialCoverage", {
31
+ covered: String(w.covered),
32
+ total: String(w.total),
33
+ });
34
+ case "stale-snapshot":
35
+ return localeText(locale, "warnings.badge.staleSnapshot", {
36
+ gap: gapText(locale, String(w.startedAt), String(w.latestStartedAt)),
37
+ });
38
+ case "unfinished-snapshot":
39
+ return localeText(locale, "warnings.badge.unfinishedSnapshot");
40
+ default:
41
+ return null;
42
+ }
43
+ }
44
+ /** 组内命令去重:恰一条时它就是「复制即推进整组」的组头命令。 */
45
+ function dedupeCommand(members) {
46
+ const commands = new Set(members.map((w) => w.command).filter((c) => typeof c === "string" && c !== ""));
47
+ return commands.size === 1 ? [...commands][0] : null;
48
+ }
49
+ function groupCategory(members) {
50
+ return members.some((w) => (CATEGORY[w.kind] ?? "integrity") === "integrity") ? "integrity" : "freshness";
51
+ }
52
+ export function groupScopeWarnings(input, locale) {
53
+ const warnings = input;
54
+ const byExperiment = new Map();
55
+ const byKind = new Map();
56
+ for (const w of warnings) {
57
+ if (EXPERIMENT_KINDS.has(w.kind) && typeof w.experimentId === "string") {
58
+ const members = byExperiment.get(w.experimentId) ?? [];
59
+ members.push(w);
60
+ byExperiment.set(w.experimentId, members);
61
+ }
62
+ else {
63
+ const members = byKind.get(w.kind) ?? [];
64
+ members.push(w);
65
+ byKind.set(w.kind, members);
66
+ }
67
+ }
68
+ const groups = [];
69
+ for (const [experimentId, members] of byExperiment) {
70
+ groups.push({
71
+ category: groupCategory(members),
72
+ title: experimentId,
73
+ badges: members
74
+ .map((w) => ({ kind: w.kind, text: badgeText(w, locale) }))
75
+ .filter((b) => b.text !== null),
76
+ headCommand: dedupeCommand(members),
77
+ warnings: members,
78
+ });
79
+ }
80
+ for (const [kind, members] of byKind) {
81
+ groups.push({
82
+ category: CATEGORY[kind] ?? "integrity",
83
+ title: kind === "unreadable-snapshot" ? pluralText(locale, "warnings.group.unreadableSnapshot", members.length) : kind,
84
+ badges: [],
85
+ headCommand: dedupeCommand(members),
86
+ warnings: members,
87
+ });
88
+ }
89
+ // 稳定排序:integrity 在前,同类别保持首次出现顺序。
90
+ const rank = (c) => (c === "integrity" ? 0 : 1);
91
+ groups.sort((a, b) => rank(a.category) - rank(b.category));
92
+ const parts = [];
93
+ if (byExperiment.size > 0)
94
+ parts.push(pluralText(locale, "warnings.summary.experiments", byExperiment.size));
95
+ for (const [kind, members] of byKind) {
96
+ parts.push(kind === "unreadable-snapshot"
97
+ ? pluralText(locale, "warnings.group.unreadableSnapshot", members.length)
98
+ : `${kind} ×${members.length}`);
99
+ }
100
+ return { summary: parts.join(" · "), groups, detailsOpen: warnings.length <= 3 };
101
+ }
@@ -1,4 +1,5 @@
1
- import type { AttemptListItem, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentListItem, LineData, MatrixData, MetricCell, ScatterData, ScopeSummaryData, ScoreboardData, TableData, VerdictTally } from "../types.ts";
1
+ import type { AttemptListItem, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentListItem, HeroData, LineData, MatrixData, MetricCell, ScatterData, ScopeSummaryData, ScopeWarning, ScoreboardData, TableData, TraceWaterfallRow, VerdictTally } from "../types.ts";
2
+ import type { LocalizedText } from "../locale.ts";
2
3
  import type { TextContext } from "../tree.ts";
3
4
  import { type ReportLocale } from "../locale.ts";
4
5
  /** 格子的文本形态:缺数据 —,覆盖不全带 samples/total 角标;display 按 locale 解析。 */
@@ -25,3 +26,20 @@ export declare function deltaText(data: DeltaData, ctx: TextContext): string;
25
26
  export declare function experimentListText(items: readonly ExperimentListItem[], ctx: TextContext, relativeTo?: string): string;
26
27
  export declare function evalListText(items: readonly EvalListItem[], ctx: TextContext): string;
27
28
  export declare function attemptListText(items: readonly AttemptListItem[], total: number | undefined, ctx: TextContext): string;
29
+ /**
30
+ * HeroCard 的 text 面:标题行 + meta 行(最后运行时间;空范围为内置「暂无运行」文案;
31
+ * 多快照时标注合成来源),不含品牌行(品牌行是纯 web 件,text 面零输出)。
32
+ */
33
+ export declare function heroCardText(title: LocalizedText, data: HeroData, ctx: TextContext): string;
34
+ /**
35
+ * ScopeWarnings 的 text 面:按动作聚合(../scope-warnings.ts,与 web 面共用),同构但不折叠——
36
+ * 多组时首行 "! <分类计数汇总>";每组一行组头 "! <标题> — <徽标> → <组头命令>",其下缩进
37
+ * 逐条原样打印 message(已以下一步收尾,不截断掉尾段)。空警告集零输出。
38
+ */
39
+ export declare function scopeWarningsText(warnings: readonly ScopeWarning[], ctx: TextContext): string;
40
+ /**
41
+ * TraceWaterfall 的 text 面:每 attempt 一行——locator、总耗时(缺 trace 如实显示缺失)、
42
+ * 顶层 span 计数与失败标记,行尾是可复制的 `--timing` 下钻命令(经宿主注入的
43
+ * attemptCommand 通道拼出,携带宿主上下文)。attempt 有选择器,索引终结于可执行命令。
44
+ */
45
+ export declare function traceWaterfallText(rows: readonly TraceWaterfallRow[], ctx: TextContext): string;
@@ -3,6 +3,7 @@
3
3
  // 零 react、零 IO、纯同步 —— 这是 text 宿主不需要 react-dom 的那一半。
4
4
  // chrome 文案(注脚、verdict 词、截断提示)经 ctx.locale 查 locale 字典;
5
5
  // 数据 display 是 LocalizedText,按 LocalizedText 回退规则取值。
6
+ import { groupScopeWarnings } from "../scope-warnings.js";
6
7
  import { experimentDisplayName, fitFailureSummary, formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD, verdictMark, } from "../format.js";
7
8
  import { countText, localeText, resolveLocalizedText, resolveMetricLabel, } from "../locale.js";
8
9
  import { indentBlock, padDisplay, stringWidth, textBar, wrapDisplay } from "./layout.js";
@@ -608,3 +609,63 @@ export function attemptListText(items, total, ctx) {
608
609
  blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
609
610
  return blocks.join("\n\n");
610
611
  }
612
+ // ───────────────────────── 站点组件(HeroCard / ScopeWarnings / TraceWaterfall)─────────────────────────
613
+ /**
614
+ * HeroCard 的 text 面:标题行 + meta 行(最后运行时间;空范围为内置「暂无运行」文案;
615
+ * 多快照时标注合成来源),不含品牌行(品牌行是纯 web 件,text 面零输出)。
616
+ */
617
+ export function heroCardText(title, data, ctx) {
618
+ const locale = ctx.locale;
619
+ const meta = data.latestStartedAt === null
620
+ ? localeText(locale, "hero.noRuns")
621
+ : [
622
+ localeText(locale, "hero.lastRun", { time: formatDateTimeMinute(data.latestStartedAt) }),
623
+ ...(data.snapshots > 1 ? [localeText(locale, "hero.composedSnapshots", { n: data.snapshots })] : []),
624
+ ].join(" · ");
625
+ return `${resolveLocalizedText(title, locale)}\n${meta}`;
626
+ }
627
+ /**
628
+ * ScopeWarnings 的 text 面:按动作聚合(../scope-warnings.ts,与 web 面共用),同构但不折叠——
629
+ * 多组时首行 "! <分类计数汇总>";每组一行组头 "! <标题> — <徽标> → <组头命令>",其下缩进
630
+ * 逐条原样打印 message(已以下一步收尾,不截断掉尾段)。空警告集零输出。
631
+ */
632
+ export function scopeWarningsText(warnings, ctx) {
633
+ if (warnings.length === 0)
634
+ return "";
635
+ const { summary, groups } = groupScopeWarnings(warnings, ctx.locale);
636
+ const lines = [];
637
+ // 汇总行只在多组时打印;单组时组头即汇总,不另起一行(web 面则恒以汇总行作外层 <summary>)。
638
+ if (groups.length > 1)
639
+ lines.push(`! ${summary}`);
640
+ for (const group of groups) {
641
+ const badges = group.badges.length > 0 ? ` — ${group.badges.map((b) => b.text).join(" · ")}` : "";
642
+ const command = group.headCommand !== null ? ` → ${group.headCommand}` : "";
643
+ lines.push(`! ${group.title}${badges}${command}`);
644
+ for (const w of group.warnings)
645
+ lines.push(`! ${w.message}`);
646
+ }
647
+ return lines.join("\n");
648
+ }
649
+ /**
650
+ * TraceWaterfall 的 text 面:每 attempt 一行——locator、总耗时(缺 trace 如实显示缺失)、
651
+ * 顶层 span 计数与失败标记,行尾是可复制的 `--timing` 下钻命令(经宿主注入的
652
+ * attemptCommand 通道拼出,携带宿主上下文)。attempt 有选择器,索引终结于可执行命令。
653
+ */
654
+ export function traceWaterfallText(rows, ctx) {
655
+ const locale = ctx.locale;
656
+ if (rows.length === 0)
657
+ return localeText(locale, "traceWaterfall.empty");
658
+ return rows
659
+ .map((row) => {
660
+ const failedSpans = row.spans.filter((span) => span.failed).length;
661
+ const parts = [
662
+ row.locator,
663
+ row.evalId,
664
+ row.durationMs === null ? localeText(locale, "traceWaterfall.noTrace") : formatDurationMs(row.durationMs),
665
+ countText(locale, "traceWaterfall.spans", row.spans.length),
666
+ ...(failedSpans > 0 ? [`✗ ${countText(locale, "traceWaterfall.failedSpans", failedSpans)}`] : []),
667
+ ];
668
+ return `${parts.join(" · ")} ${ctx.attemptCommand(row.locator)} --timing`;
669
+ })
670
+ .join("\n");
671
+ }
@@ -187,7 +187,12 @@ async function resolveNode(node, state, path) {
187
187
  return node;
188
188
  }
189
189
  if (Array.isArray(node)) {
190
- return Promise.all(node.map((child) => resolveNode(child, state, path)));
190
+ const resolved = await Promise.all(node.map((child) => resolveNode(child, state, path)));
191
+ // resolve 重建的 children 数组对 React 是动态列表(JSX 静态 children 的免 key 待遇随重建
192
+ // 丢失);声明序即身份,给缺 key 的元素补声明位 key,免得 web 面渲染刷 key 警告。
193
+ return resolved.map((child, i) => isReportElement(child) && (child.key === undefined || child.key === null)
194
+ ? { ...child, key: `.nre-${i}` }
195
+ : child);
191
196
  }
192
197
  if (!isReportElement(node))
193
198
  return node;
@@ -275,6 +275,51 @@ export interface ScopeSummaryData {
275
275
  /** costUSD 按 attempt 求和;缺失成本不伪造为 0。 */
276
276
  totalCostUSD: MetricCell;
277
277
  }
278
+ /**
279
+ * `HeroCard` 的数据(docs/feature/reports/library/site-components.md):站点标题区的
280
+ * 运行 meta——最后运行时间与快照合成来源。标题不在 data 里,它是站点声明与 Scope 的合成物,
281
+ * 经 `HeroCardProps.title` 传入。
282
+ */
283
+ export interface HeroData {
284
+ /** Scope 中最新快照的开始时间;空 Scope 为 null,不编造当前时间。 */
285
+ latestStartedAt: string | null;
286
+ /** 贡献当前水位的快照数;大于 1 时 web 面标注「由 N 次运行合成」。 */
287
+ snapshots: number;
288
+ }
289
+ /**
290
+ * `CopyFixPrompt` 的数据:resolve 期算好的修复 prompt 全文与参与的失败数
291
+ * (docs/feature/reports/library/site-components.md)。
292
+ */
293
+ export interface CopyFixPromptData {
294
+ /** 修复 prompt 全文;失败逐条含 eval id、主失败摘要与 attempt 下钻命令。 */
295
+ prompt: string;
296
+ /** 参与 prompt 的失败 attempt 数(verdict 为 failed / errored)。 */
297
+ failures: number;
298
+ }
299
+ /** `TraceWaterfall` 一行里的一个顶层 span 摘要(canonical OTel 字段归一后的形态)。 */
300
+ export interface TraceSpanSummary {
301
+ name: string;
302
+ /** 归一后的语义角色;turn 归入 agent,未识别落 other。 */
303
+ kind: "agent" | "model" | "tool" | "other";
304
+ /** 相对该 attempt trace 起点的偏移(毫秒)。 */
305
+ startOffsetMs: number;
306
+ durationMs: number;
307
+ /** span status 为 error 时 true(web 面失败标记的来源)。 */
308
+ failed: boolean;
309
+ }
310
+ /**
311
+ * `TraceWaterfall` 一行 = 一次 attempt 的执行时间瀑布摘要。只画被测 agent 的原始 span
312
+ * (trace.json);runner 生命周期节点(`result.phases`)不进瀑布,组合视图归 attempt 详情。
313
+ */
314
+ export interface TraceWaterfallRow {
315
+ experimentId: string;
316
+ evalId: string;
317
+ locator: AttemptLocator;
318
+ /** trace.json 缺失或为空时 null;行照常出现,证据位置如实显示缺失,不猜值。 */
319
+ durationMs: number | null;
320
+ /** 顶层 span 摘要,按 startOffsetMs 升序。 */
321
+ spans: readonly TraceSpanSummary[];
322
+ }
278
323
  /** 一个可比组的数据;三个子块都只消费本组快照,不能含其它父目录的引用。 */
279
324
  export interface ExperimentComparisonGroupData {
280
325
  /** experiment id 的完整父路径;根目录 experiment 使用完整 id。 */
@@ -359,13 +404,3 @@ export interface ExperimentListItem {
359
404
  lastRunAt: string;
360
405
  evalRows: ExperimentListEvalRow[];
361
406
  }
362
- /** 三个实体列表共用的计算选项。 */
363
- export interface EntityListDataOptions {
364
- /**
365
- * 展示层遮蔽:只改写这次组件数据中的自由文本——条目本身与任何嵌套 attempt 条目的
366
- * `failureSummary`;身份与分类字段(experimentId、evalId、locator、数值指标)不经它。
367
- * 只作用于这次计算产出的组件数据,不改盘上或任何导出目录里的 artifact;
368
- * 发布 artifact 的脱敏用 copySnapshots({ redact })。
369
- */
370
- redact?: (text: string) => string;
371
- }
@@ -12,14 +12,15 @@ export interface StaticHtmlOptions {
12
12
  }
13
13
  /**
14
14
  * web 宿主的装载语义:选页 → resolve(组合展开 + spec 取数,唯一的 await 边界)→
15
- * 树校验(与 text 宿主同一遍)→ 静态渲染 web 面;Scope 有挑选警告时在报告顶部前置
16
- * 一块警告 HTML(宿主是 warning 的唯一呈现者,组件数据不复制 warning)。
15
+ * 树校验(与 text 宿主同一遍)→ 静态渲染 web 面。宿主不在报告树外另设警告通道——
16
+ * 挑选警告的呈现件是 `ScopeWarnings` 组件,内建报告每页都放它,自定义报告放不放是
17
+ * 作者义务(docs/feature/reports/architecture.md「Scope 是计算入口」)。
17
18
  */
18
19
  export declare function renderReportToStaticHtml(definition: ReportDefinition, ctx: ReportHostContext, options?: StaticHtmlOptions): Promise<string>;
19
20
  /**
20
21
  * 渲染一页报告树的 web 面(宿主逐页调用;页选择归宿主):resolve → validate → 静态渲染。
21
- * Scope 有挑选警告时在页顶前置警告块(带 command 的警告渲染为可复制命令)——宿主是
22
- * warning 的唯一呈现者,组件数据不复制 warning。ctx.report 是宿主规范化后的声明。
22
+ * 挑选警告由页内的 `ScopeWarnings` 组件呈现,宿主不前置任何树外块。
23
+ * ctx.report 是宿主规范化后的声明。
23
24
  */
24
25
  export declare function renderReportTreeToStaticHtml(tree: import("./tree.ts").ReportNode, ctx: {
25
26
  scope: Scope;
@@ -1,26 +1,15 @@
1
1
  // web 宿主(view --report)的装载入口:同一棵树走 web 面,renderToStaticMarkup 吐静态
2
2
  // HTML 烘进查看器的报告槽。只有这一侧真正 import react-dom(import 边界即运行时边界),
3
3
  // 所以本文件不从 niceeval/report 的入口 re-export —— 宿主与测试按源路径 import。
4
- import * as React from "react";
5
4
  import { renderToStaticMarkup } from "react-dom/server";
6
5
  import { resolveReportTree, runWithWebContext, validateReportTree, ResolveMemo, } from "./tree.js";
7
6
  import { DEFAULT_REPORT_LOCALE } from "./locale.js";
8
7
  import { buildReportMeta, pickReportPage } from "./report.js";
9
- /**
10
- * 挑选警告的 HTML 形态:宿主级前置块(`.nre nre-report-warnings` 外壳内一个
11
- * `ul.nre-warnings` + `li.nre-warning[data-kind]`,复用 styles.css 已有样式)。
12
- * 带 `command` 的警告把命令渲染为可复制块(`.nre-warning-command`);无 command 的
13
- * 只显示 message,不硬造动作。经 renderToStaticMarkup 走 React,文本自动转义。
14
- */
15
- function renderScopeWarningsHtml(scope) {
16
- return renderToStaticMarkup(React.createElement("div", { className: "nre nre-report-warnings" }, React.createElement("ul", { className: "nre-warnings" }, scope.warnings.map((w, i) => React.createElement("li", { key: i, className: "nre-warning", "data-kind": w.kind }, w.message, "command" in w && w.command
17
- ? React.createElement("code", { className: "nre-warning-command", "data-nre-copy": w.command }, w.command)
18
- : null)))));
19
- }
20
8
  /**
21
9
  * web 宿主的装载语义:选页 → resolve(组合展开 + spec 取数,唯一的 await 边界)→
22
- * 树校验(与 text 宿主同一遍)→ 静态渲染 web 面;Scope 有挑选警告时在报告顶部前置
23
- * 一块警告 HTML(宿主是 warning 的唯一呈现者,组件数据不复制 warning)。
10
+ * 树校验(与 text 宿主同一遍)→ 静态渲染 web 面。宿主不在报告树外另设警告通道——
11
+ * 挑选警告的呈现件是 `ScopeWarnings` 组件,内建报告每页都放它,自定义报告放不放是
12
+ * 作者义务(docs/feature/reports/architecture.md「Scope 是计算入口」)。
24
13
  */
25
14
  export async function renderReportToStaticHtml(definition, ctx, options) {
26
15
  const page = pickReportPage(definition, options?.pageId);
@@ -36,14 +25,12 @@ export async function renderReportToStaticHtml(definition, ctx, options) {
36
25
  attemptHref: options?.attemptHref ?? ((locator) => `#/attempt/${locator}`),
37
26
  locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
38
27
  };
39
- const body = runWithWebContext(webCtx, () => renderToStaticMarkup(resolved));
40
- const warnings = ctx.scope.warnings.length > 0 ? renderScopeWarningsHtml(ctx.scope) : "";
41
- return warnings + body;
28
+ return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved));
42
29
  }
43
30
  /**
44
31
  * 渲染一页报告树的 web 面(宿主逐页调用;页选择归宿主):resolve → validate → 静态渲染。
45
- * Scope 有挑选警告时在页顶前置警告块(带 command 的警告渲染为可复制命令)——宿主是
46
- * warning 的唯一呈现者,组件数据不复制 warning。ctx.report 是宿主规范化后的声明。
32
+ * 挑选警告由页内的 `ScopeWarnings` 组件呈现,宿主不前置任何树外块。
33
+ * ctx.report 是宿主规范化后的声明。
47
34
  */
48
35
  export async function renderReportTreeToStaticHtml(tree, ctx, options) {
49
36
  const resolved = await resolveReportTree(tree, {
@@ -57,7 +44,5 @@ export async function renderReportTreeToStaticHtml(tree, ctx, options) {
57
44
  attemptHref: options?.attemptHref ?? ((locator) => `#/attempt/${locator}`),
58
45
  locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
59
46
  };
60
- const body = runWithWebContext(webCtx, () => renderToStaticMarkup(resolved));
61
- const warnings = ctx.scope.warnings.length > 0 ? renderScopeWarningsHtml(ctx.scope) : "";
62
- return warnings + body;
47
+ return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved));
63
48
  }
@@ -1,7 +1,12 @@
1
1
  import type { AttemptHandle, DedupeWarning, Experiment, Results, Scope, ScopeWarning, Snapshot } from "./types.ts";
2
2
  import type { ExperimentRunInfo, JsonValue } from "../types.ts";
3
- /** Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。 */
4
- export declare function selectLatest(experiments: Experiment[], opts?: {
3
+ /**
4
+ * Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。
5
+ * 收整个 `Results` 而不是裸 `Experiment[]`,是为了同时取 `skipped` / `root` 生成
6
+ * `unreadable-snapshot` 警告(非实验作用域,不受 `opts.experiments` 过滤 —— 那些落盘
7
+ * 本来就没能解析出 experimentId,没有前缀可过滤)。
8
+ */
9
+ export declare function selectLatest(results: Pick<Results, "experiments" | "skipped" | "root">, opts?: {
5
10
  experiments?: string | string[];
6
11
  }): Scope;
7
12
  /** selectCurrentResults 的范围输入:experiment id 前缀与 eval id 前缀,都可缺省。 */
@@ -65,3 +70,11 @@ export declare function dedupeAttempts(attempts: AttemptHandle[]): {
65
70
  export declare function isNewerSnapshot(a: Snapshot, b: Snapshot): boolean;
66
71
  /** experiment id 分段前缀过滤(--experiment / latest({ experiments }) 同一语义);包内使用,不进公共 barrel。 */
67
72
  export declare function filterExperiments(experiments: Experiment[], filter?: string | string[]): Experiment[];
73
+ /**
74
+ * stale 警告的人话时距:选粒度最大的单位,四舍五入。结构化形态是单源——message 的英文时距
75
+ * 与 ScopeWarnings 徽标的本地化时距都从这里出,阈值不写两份。
76
+ */
77
+ export declare function gapParts(fromIso: string, toIso: string): {
78
+ n: number;
79
+ unit: "second" | "minute" | "hour" | "day";
80
+ };
@@ -4,9 +4,14 @@
4
4
  // 选择器必须诚实:残缺、落后、未收尾都被算出来,以结构化 warnings 随 Scope 走 ——
5
5
  // 渲染与否在消费方(message 是渲染好的英文句子,以下一步收尾),但缺口不静默。
6
6
  import { evalPrefixPredicate } from "../shared/aggregate.js";
7
- /** Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。 */
8
- export function selectLatest(experiments, opts) {
9
- const selected = filterExperiments(experiments, opts?.experiments);
7
+ /**
8
+ * Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。
9
+ * 收整个 `Results` 而不是裸 `Experiment[]`,是为了同时取 `skipped` / `root` 生成
10
+ * `unreadable-snapshot` 警告(非实验作用域,不受 `opts.experiments` 过滤 —— 那些落盘
11
+ * 本来就没能解析出 experimentId,没有前缀可过滤)。
12
+ */
13
+ export function selectLatest(results, opts) {
14
+ const selected = filterExperiments(results.experiments, opts?.experiments);
10
15
  const snapshots = selected.map((exp) => exp.latest);
11
16
  const warnings = [];
12
17
  // stale 的基准:Scope 中最新的落盘(无阈值,如实触发;要阈值消费方按字段自比)。
@@ -52,6 +57,7 @@ export function selectLatest(experiments, opts) {
52
57
  });
53
58
  }
54
59
  }
60
+ warnings.push(...unreadableSnapshotWarnings(results.skipped, results.root));
55
61
  return makeScope("latest-snapshots", snapshots, warnings);
56
62
  }
57
63
  /** 一个快照的可比性配置投影;pairsByFlag 与 experimentListData 复用同一字段集。 */
@@ -188,8 +194,61 @@ export function selectCurrentResults(results, scope = {}) {
188
194
  });
189
195
  }
190
196
  }
197
+ warnings.push(...unreadableSnapshotWarnings(results.skipped, results.root));
191
198
  return makeScope("current-evals", snapshots, warnings);
192
199
  }
200
+ /**
201
+ * `results.skipped` 里每一条不可读落盘 → 一条 `unreadable-snapshot` ScopeWarning。
202
+ * 非实验作用域(没有 experimentId 字段):`latest()` / `current()` 都原样带上全部
203
+ * `skipped` 条目,不受 `opts.experiments` 前缀过滤影响(那些落盘本来就没能解析出
204
+ * experimentId,没有前缀可比);`makeScope().filter()` 按「非实验作用域的警告保留」
205
+ * 规则自动放行,不需要额外分支。
206
+ */
207
+ function unreadableSnapshotWarnings(skipped, root) {
208
+ return skipped.map((s) => {
209
+ switch (s.reason) {
210
+ case "incompatible-version": {
211
+ const producer = s.producer;
212
+ const schemaText = s.schemaVersion !== undefined ? ` (schemaVersion ${s.schemaVersion})` : "";
213
+ if (producer?.name === "niceeval" && producer.version) {
214
+ const command = `npx niceeval@${producer.version} show --results ${root}`;
215
+ return {
216
+ kind: "unreadable-snapshot",
217
+ dir: s.dir,
218
+ reason: s.reason,
219
+ message: `snapshot at "${s.dir}" was written by niceeval ${producer.version}${schemaText} and cannot be read by this version; run \`${command}\` to open it`,
220
+ command,
221
+ };
222
+ }
223
+ const writtenBy = producer?.name
224
+ ? `${producer.name}${producer.version ? ` ${producer.version}` : ""}`
225
+ : "an incompatible tool version";
226
+ return {
227
+ kind: "unreadable-snapshot",
228
+ dir: s.dir,
229
+ reason: s.reason,
230
+ message: `snapshot at "${s.dir}" was written by ${writtenBy}${schemaText} and cannot be read by this version; open it with the tool version that produced it`,
231
+ };
232
+ }
233
+ case "malformed": {
234
+ const detail = s.detail ? ` (${s.detail})` : "";
235
+ return {
236
+ kind: "unreadable-snapshot",
237
+ dir: s.dir,
238
+ reason: s.reason,
239
+ message: `snapshot at "${s.dir}" is malformed${detail} and was skipped; inspect snapshot.json in that directory for corrupted JSON or a missing required field`,
240
+ };
241
+ }
242
+ case "incomplete":
243
+ return {
244
+ kind: "unreadable-snapshot",
245
+ dir: s.dir,
246
+ reason: s.reason,
247
+ message: `snapshot at "${s.dir}" has attempt data but no snapshot.json (likely interrupted before metadata was written) and was skipped; inspect ${s.dir} — completed attempts remain on disk for manual review`,
248
+ };
249
+ }
250
+ });
251
+ }
193
252
  /**
194
253
  * Scope 构造:attempts 按口径物化(快照 attempts 的平铺);filter 只删不换 —— 快照删减,
195
254
  * attempts 随之同步修剪,warnings 修剪规则是「experimentId 不在幸存快照中的丢弃,
@@ -261,18 +320,24 @@ export function filterExperiments(experiments, filter) {
261
320
  const prefixes = (Array.isArray(filter) ? filter : [filter]).map((p) => p.replace(/\/+$/, ""));
262
321
  return experiments.filter((exp) => prefixes.some((p) => exp.id === p || exp.id.startsWith(p + "/")));
263
322
  }
264
- /** stale 警告的人话时距:选粒度最大的单位,四舍五入。 */
265
- function humanizeGap(fromIso, toIso) {
323
+ /**
324
+ * stale 警告的人话时距:选粒度最大的单位,四舍五入。结构化形态是单源——message 的英文时距
325
+ * 与 ScopeWarnings 徽标的本地化时距都从这里出,阈值不写两份。
326
+ */
327
+ export function gapParts(fromIso, toIso) {
266
328
  const ms = Math.max(0, Date.parse(toIso) - Date.parse(fromIso));
267
329
  const seconds = Math.round(ms / 1000);
268
330
  if (seconds < 90)
269
- return `${seconds} second${seconds === 1 ? "" : "s"}`;
331
+ return { n: seconds, unit: "second" };
270
332
  const minutes = Math.round(seconds / 60);
271
333
  if (minutes < 90)
272
- return `${minutes} minute${minutes === 1 ? "" : "s"}`;
334
+ return { n: minutes, unit: "minute" };
273
335
  const hours = Math.round(minutes / 60);
274
336
  if (hours < 36)
275
- return `${hours} hour${hours === 1 ? "" : "s"}`;
276
- const days = Math.round(hours / 24);
277
- return `${days} day${days === 1 ? "" : "s"}`;
337
+ return { n: hours, unit: "hour" };
338
+ return { n: Math.round(hours / 24), unit: "day" };
339
+ }
340
+ function humanizeGap(fromIso, toIso) {
341
+ const { n, unit } = gapParts(fromIso, toIso);
342
+ return `${n} ${unit}${n === 1 ? "" : "s"}`;
278
343
  }
@@ -33,13 +33,6 @@ export interface SnapshotMeta {
33
33
  completedAt?: string;
34
34
  /** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
35
35
  knownEvalIds?: string[];
36
- /**
37
- * 发布拷贝的自描述标记:copySnapshots 补记,消毒函数 → "applied"、redact: false → "none";
38
- * 本地事实根没有此字段。只声明流程,不证明无秘密;view --out 据此分级防呆。
39
- */
40
- publish?: {
41
- redaction: "applied" | "none";
42
- };
43
36
  /** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
44
37
  name?: LocalizedText;
45
38
  }
@@ -117,10 +110,6 @@ export interface Snapshot {
117
110
  dir: string;
118
111
  /** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
119
112
  knownEvalIds?: string[];
120
- /** 发布拷贝的自描述标记(见 SnapshotMeta.publish);本地事实根没有此字段。 */
121
- publish?: {
122
- redaction: "applied" | "none";
123
- };
124
113
  }
125
114
  /** 一个实验的全部历史:同一 experiment id 的历次快照归在一起。 */
126
115
  export interface Experiment {
@@ -153,6 +142,12 @@ export interface SkippedDir {
153
142
  }
154
143
  /** openResults 的返回:experiments 分层;skipped 不静默丢。 */
155
144
  export interface Results {
145
+ /**
146
+ * 结果根目录的绝对路径(`openResults()` 入参解析后的原样值,不论传入的是结果根、
147
+ * 实验目录、快照目录还是某个 snapshot.json)。`unreadable-snapshot` 警告拼版本化
148
+ * `command`(`npx niceeval@<version> show --results <root>`)时取它。
149
+ */
150
+ root: string;
156
151
  /** 每个实验一项,挂着自己的全部历史(id 字典序)。 */
157
152
  experiments: Experiment[];
158
153
  skipped: SkippedDir[];
@@ -229,6 +224,26 @@ export type ScopeWarning = {
229
224
  message: string;
230
225
  /** 一条可复制即跑的推进命令:`niceeval exp <experimentId>`。 */
231
226
  command: string;
227
+ } | {
228
+ /**
229
+ * 扫描结果根遇到的不可读快照:schema 不兼容、JSON 损坏 / 必需字段错误(malformed)、
230
+ * attempt 已写入但缺 `snapshot.json`(incomplete)。该快照被跳过,不挡其余结果
231
+ * (非 niceeval JSON 静默忽略,不产生这个 kind)。非实验作用域(没有 experimentId
232
+ * 字段) —— `Scope.filter()` 修剪时恒保留。
233
+ */
234
+ kind: "unreadable-snapshot";
235
+ /** 该快照目录的绝对路径。 */
236
+ dir: string;
237
+ /** 与 `SkippedDir.reason` 同一取值集,原样透传。 */
238
+ reason: "incompatible-version" | "malformed" | "incomplete";
239
+ message: string;
240
+ /**
241
+ * 只有 reason 为 `incompatible-version` 且能确定是 niceeval 自己产出(`producer.name
242
+ * === "niceeval"` 且带 `producer.version`)时给出:`npx niceeval@<version> show --results
243
+ * <root>`。第三方 producer、版本信息缺失,或 reason 为 malformed / incomplete 时省略——
244
+ * 这些情况没有单条命令能解决,message 改给定位动作。
245
+ */
246
+ command?: string;
232
247
  };
233
248
  /** dedupeAttempts 的警告:身份键缺 startedAt,宁可不去重也不误删。 */
234
249
  export interface DedupeWarning {
@@ -1,11 +1,11 @@
1
- import type { DiscoveredEval, EvalResult } from "../types.ts";
1
+ import type { DiscoveredEval, EvalResult, SandboxOption } from "../types.ts";
2
2
  import type { AgentRun } from "./types.ts";
3
3
  export declare function cacheKey(run: AgentRun, evalId: string): string;
4
4
  /**
5
5
  * @param sourceCache 按 sourcePath 缓存文件内容:一个矩阵(实验 × eval)会对同一批源文件
6
6
  * 反复算指纹,不带缓存会在任何 attempt 起跑前做 E×N 次重复文件读。
7
7
  */
8
- export declare function computeFingerprint(evalDef: DiscoveredEval, run: AgentRun, sourceCache?: Map<string, Promise<string>>): Promise<string>;
8
+ export declare function computeFingerprint(evalDef: DiscoveredEval, run: AgentRun, sourceCache?: Map<string, Promise<string>>, configSandbox?: SandboxOption): Promise<string>;
9
9
  export interface CarryPlan {
10
10
  /** `cacheKey(run, evalId)` → 本次规划出的指纹,供调用方按同一口径判断"这条要不要携入"。 */
11
11
  plannedFingerprints: Map<string, string>;
@@ -20,4 +20,4 @@ export interface CarryPlan {
20
20
  * 不一致,live 表格就会显示"还在等名额",而 run.ts 其实已经把它筛掉、根本不会调度这个 attempt
21
21
  * (见 memory 的 live-carry-row-shows-waiting-forever)。
22
22
  */
23
- export declare function planCarry(evals: DiscoveredEval[], agentRuns: AgentRun[], priorResults: EvalResult[] | undefined): Promise<CarryPlan>;
23
+ export declare function planCarry(evals: DiscoveredEval[], agentRuns: AgentRun[], priorResults: EvalResult[] | undefined, configSandbox?: SandboxOption): Promise<CarryPlan>;
@@ -0,0 +1,12 @@
1
+ import type { DiscoveredEval, SandboxOption, SandboxRunInfo } from "../types.ts";
2
+ import type { AgentRun } from "./types.ts";
3
+ /** 该 eval 实际起步的 spec:未声明 environment 用基础 spec;声明了则查表派生并缓存。 */
4
+ export declare function sandboxForEval(run: AgentRun, evalDef: DiscoveredEval, fallback?: SandboxOption): SandboxOption | undefined;
5
+ /** 在 dry-run / carry / concurrency / attempt 展开之前一次性查表;全部缺项一次穷举,不等到花费发生后才出现。 */
6
+ export declare function prepareRunSandboxes(evals: DiscoveredEval[], runs: AgentRun[], fallback?: SandboxOption): void;
7
+ /** ExperimentRunInfo 的 sandbox 投影:顶层恒为基础 spec;sandboxByEval 只含声明了 environment 的选中 eval。 */
8
+ export declare function sandboxProjection(run: AgentRun, fallback?: SandboxOption): {
9
+ sandbox?: SandboxRunInfo;
10
+ sandboxByEval?: Record<string, SandboxRunInfo>;
11
+ };
12
+ export declare function resolvedSandboxRecommendedConcurrency(evals: DiscoveredEval[], runs: AgentRun[], fallback?: SandboxOption): number;