niceeval 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/INDEX.md +77 -45
  2. package/dist/agents/types.d.ts +28 -6
  3. package/dist/i18n/en.d.ts +2 -0
  4. package/dist/i18n/zh-CN.d.ts +2 -0
  5. package/dist/report/built-in/index.d.ts +3 -2
  6. package/dist/report/built-in/index.js +7 -8
  7. package/dist/report/built-in/standard.d.ts +1 -0
  8. package/dist/report/built-in/standard.js +30 -0
  9. package/dist/report/components.d.ts +72 -6
  10. package/dist/report/components.js +159 -10
  11. package/dist/report/compute.d.ts +31 -4
  12. package/dist/report/compute.js +131 -12
  13. package/dist/report/index.d.ts +4 -4
  14. package/dist/report/index.js +3 -2
  15. package/dist/report/locale.d.ts +39 -1
  16. package/dist/report/locale.js +69 -0
  17. package/dist/report/react/AttemptList.d.ts +3 -1
  18. package/dist/report/react/AttemptList.js +3 -3
  19. package/dist/report/react/CopyFixPrompt.d.ts +12 -0
  20. package/dist/report/react/CopyFixPrompt.js +12 -0
  21. package/dist/report/react/HeroCard.d.ts +13 -0
  22. package/dist/report/react/HeroCard.js +35 -0
  23. package/dist/report/react/PoweredBy.d.ts +5 -0
  24. package/dist/report/react/PoweredBy.js +7 -0
  25. package/dist/report/react/ScopeWarnings.d.ts +12 -0
  26. package/dist/report/react/ScopeWarnings.js +18 -0
  27. package/dist/report/react/TraceWaterfall.d.ts +14 -0
  28. package/dist/report/react/TraceWaterfall.js +22 -0
  29. package/dist/report/react/index.d.ts +6 -1
  30. package/dist/report/react/index.js +6 -0
  31. package/dist/report/report.d.ts +22 -6
  32. package/dist/report/report.js +66 -53
  33. package/dist/report/scope-warnings.d.ts +28 -0
  34. package/dist/report/scope-warnings.js +101 -0
  35. package/dist/report/text/faces.d.ts +19 -1
  36. package/dist/report/text/faces.js +61 -0
  37. package/dist/report/tree.js +6 -1
  38. package/dist/report/types.d.ts +45 -10
  39. package/dist/report/web.d.ts +5 -4
  40. package/dist/report/web.js +7 -22
  41. package/dist/results/select.d.ts +15 -2
  42. package/dist/results/select.js +75 -10
  43. package/dist/results/types.d.ts +26 -11
  44. package/dist/runner/fingerprint.d.ts +3 -3
  45. package/dist/runner/sandbox-selection.d.ts +12 -0
  46. package/dist/runner/types.d.ts +18 -6
  47. package/dist/sandbox/types.d.ts +12 -0
  48. package/docs-site/zh/explanation/evals.mdx +2 -1
  49. package/docs-site/zh/explanation/experiment.mdx +2 -0
  50. package/docs-site/zh/how-to/custom-reports.mdx +22 -6
  51. package/docs-site/zh/how-to/publish-report.mdx +6 -8
  52. package/docs-site/zh/how-to/viewing-results.mdx +3 -3
  53. package/docs-site/zh/how-to/write-experiment.mdx +40 -1
  54. package/docs-site/zh/reference/builtin-agents.mdx +40 -3
  55. package/docs-site/zh/reference/cli.mdx +1 -2
  56. package/docs-site/zh/reference/define-eval.mdx +8 -0
  57. package/docs-site/zh/reference/official-adapters.mdx +10 -5
  58. package/docs-site/zh/reference/report-components.mdx +2 -2
  59. package/docs-site/zh/reference/results-data.mdx +2 -4
  60. package/package.json +2 -1
  61. package/src/agents/bub.ts +13 -1
  62. package/src/agents/claude-code.test.ts +43 -1
  63. package/src/agents/claude-code.ts +32 -14
  64. package/src/agents/codex.test.ts +168 -1
  65. package/src/agents/codex.ts +51 -15
  66. package/src/agents/mcp.ts +31 -0
  67. package/src/agents/post-setup.ts +33 -0
  68. package/src/agents/types.ts +28 -7
  69. package/src/cli.ts +9 -11
  70. package/src/context/context.ts +11 -5
  71. package/src/define.ts +3 -0
  72. package/src/i18n/en.ts +5 -2
  73. package/src/i18n/zh-CN.ts +5 -1
  74. package/src/index.ts +1 -0
  75. package/src/report/built-in/index.tsx +8 -7
  76. package/src/report/built-in/standard.tsx +59 -0
  77. package/src/report/components.tsx +231 -17
  78. package/src/report/compute.ts +146 -21
  79. package/src/report/dual-render.test.tsx +139 -12
  80. package/src/report/index.ts +20 -1
  81. package/src/report/locale.ts +83 -1
  82. package/src/report/react/AttemptList.tsx +13 -1
  83. package/src/report/react/CopyFixPrompt.tsx +37 -0
  84. package/src/report/react/HeroCard.tsx +59 -0
  85. package/src/report/react/PoweredBy.tsx +20 -0
  86. package/src/report/react/ScopeWarnings.tsx +74 -0
  87. package/src/report/react/TraceWaterfall.tsx +78 -0
  88. package/src/report/react/enhance.js +14 -0
  89. package/src/report/react/index.tsx +11 -0
  90. package/src/report/react/styles.css +187 -7
  91. package/src/report/report.test.ts +2 -20
  92. package/src/report/report.ts +97 -62
  93. package/src/report/scope-warnings.ts +155 -0
  94. package/src/report/site-components.test.tsx +526 -0
  95. package/src/report/text/faces.ts +66 -0
  96. package/src/report/tree.ts +8 -1
  97. package/src/report/types.ts +51 -11
  98. package/src/report/web.ts +7 -40
  99. package/src/results/copy.ts +15 -78
  100. package/src/results/host-equivalence.test.ts +5 -1
  101. package/src/results/open.ts +5 -4
  102. package/src/results/publish.ts +4 -146
  103. package/src/results/results.test.ts +86 -9
  104. package/src/results/select.ts +78 -10
  105. package/src/results/types.ts +27 -7
  106. package/src/runner/attempt.ts +10 -9
  107. package/src/runner/discover.test.ts +9 -1
  108. package/src/runner/discover.ts +3 -3
  109. package/src/runner/fingerprint.ts +9 -4
  110. package/src/runner/ledger.test.ts +30 -1
  111. package/src/runner/ledger.ts +26 -4
  112. package/src/runner/run.ts +5 -1
  113. package/src/runner/sandbox-selection.test.ts +131 -0
  114. package/src/runner/sandbox-selection.ts +110 -0
  115. package/src/runner/types.ts +19 -2
  116. package/src/sandbox/types.ts +6 -0
  117. package/src/show/index.ts +17 -10
  118. package/src/show/render.ts +11 -11
  119. package/src/show/report-host.test.ts +32 -15
  120. package/src/show/report-host.ts +5 -4
  121. package/src/show/show.test.ts +140 -3
  122. package/src/view/app/App.test.tsx +78 -17
  123. package/src/view/app/App.tsx +17 -78
  124. package/src/view/app/components/CopyControls.tsx +4 -42
  125. package/src/view/app/i18n.ts +5 -227
  126. package/src/view/app/lib/rows.ts +3 -21
  127. package/src/view/app/shared.ts +1 -3
  128. package/src/view/app/types.ts +2 -2
  129. package/src/view/artifact-serving.test.ts +1 -1
  130. package/src/view/client-dist/app.css +1 -1
  131. package/src/view/client-dist/app.js +20 -20
  132. package/src/view/data.ts +2 -13
  133. package/src/view/index.ts +1 -12
  134. package/src/view/server.ts +0 -2
  135. package/src/view/shared/types.ts +11 -6
  136. package/src/view/site-parity.test.ts +1 -1
  137. package/src/view/site.ts +1 -1
  138. package/src/view/styles.css +6 -266
  139. package/src/view/view-report.test.ts +64 -27
  140. package/src/view/app/components/LazyArtifact.tsx +0 -51
  141. package/src/view/app/components/SkippedRunsBanner.tsx +0 -140
  142. package/src/view/app/pages/AttemptsPage.tsx +0 -80
  143. package/src/view/app/pages/TracesPage.tsx +0 -35
@@ -1,51 +0,0 @@
1
- import { useEffect, useState } from "react";
2
- import type { LazyArtifactType, T } from "../shared.ts";
3
- import { artifactUrl } from "../lib/artifact-url.ts";
4
- import { asEvents, asSpans } from "../lib/guards.ts";
5
- import { Trace } from "./Trace.tsx";
6
- import { Transcript } from "./Transcript.tsx";
7
-
8
- export function LazyArtifact({ type, src, autoLoad = false, t }: { type: LazyArtifactType; src: string; autoLoad?: boolean; t: T }) {
9
- const [open, setOpen] = useState(autoLoad);
10
- const [loaded, setLoaded] = useState(false);
11
- const [content, setContent] = useState<unknown>(null);
12
- const [error, setError] = useState("");
13
-
14
- const load = async () => {
15
- if (loaded) return;
16
- setLoaded(true);
17
- try {
18
- const resp = await fetch(artifactUrl(src));
19
- if (!resp.ok) throw new Error("HTTP " + resp.status);
20
- const body = await resp.json();
21
- setContent(body);
22
- setError("");
23
- } catch (e) {
24
- setLoaded(false);
25
- setError(`${t("trace.loadFailed")} ${String(e)}`);
26
- }
27
- };
28
-
29
- useEffect(() => {
30
- if (autoLoad) void load();
31
- }, []);
32
-
33
- return (
34
- <details
35
- className="trace-details"
36
- open={open}
37
- onToggle={(e) => {
38
- const isOpen = e.currentTarget.open;
39
- setOpen(isOpen);
40
- if (isOpen) void load();
41
- }}
42
- >
43
- <summary>{type === "transcript" ? t("trace.transcript") : t("trace.timing")}</summary>
44
- <div className="trace-slot">
45
- {error ? <div className="trace-span-meta">{error}</div> : !content ? <div className="trace-span-meta">{t("trace.loading")}</div> : null}
46
- {content && type === "transcript" ? <Transcript events={asEvents(content) ?? []} t={t} /> : null}
47
- {content && type === "trace" ? <Trace spans={asSpans(content) ?? []} t={t} /> : null}
48
- </div>
49
- </details>
50
- );
51
- }
@@ -1,140 +0,0 @@
1
- import { useState } from "react";
2
- import { AlertCircle, ChevronRight, Copy, Check } from "lucide-react";
3
- import type { T } from "../shared.ts";
4
- import type { MessageKey } from "../i18n.ts";
5
- import type { SkippedRunNotice } from "../types.ts";
6
- import { copyText } from "./CopyControls.tsx";
7
-
8
- // 折叠前默认可见的条目数(跨分组累计);超出的部分收进「展开其余」。
9
- const DEFAULT_VISIBLE = 3;
10
-
11
- type GroupKind = "incompatible" | "incompatibleForeign" | "malformed" | "incomplete";
12
-
13
- interface SkippedGroup {
14
- key: string;
15
- kind: GroupKind;
16
- messageKey: MessageKey;
17
- vars: Record<string, string | number>;
18
- items: SkippedRunNotice[];
19
- }
20
-
21
- /** 同一原因(且描述文案完全相同)的失败 run 归成一组,组内条目只保留目录名 + 命令,
22
- * 避免把完整原因句子重复 N 遍——这正是原实现把首屏挤满的根因。 */
23
- function groupSkippedRuns(runs: SkippedRunNotice[]): SkippedGroup[] {
24
- const groups = new Map<string, SkippedGroup>();
25
- for (const run of runs) {
26
- let kind: GroupKind;
27
- let key: string;
28
- let messageKey: MessageKey;
29
- let vars: Record<string, string | number>;
30
-
31
- if (run.reason === "malformed") {
32
- kind = "malformed";
33
- vars = { detail: run.detail ?? "?" };
34
- key = `malformed:${vars.detail}`;
35
- messageKey = "banner.skipped.malformed";
36
- } else if (run.reason === "incomplete") {
37
- kind = "incomplete";
38
- vars = {};
39
- key = "incomplete";
40
- messageKey = "banner.skipped.incomplete";
41
- } else if (run.producerName && run.producerName !== "niceeval") {
42
- kind = "incompatibleForeign";
43
- vars = { name: run.producerName, version: run.producerVersion ?? "?", schemaVersion: run.schemaVersion ?? "?" };
44
- key = `foreign:${vars.name}:${vars.version}:${vars.schemaVersion}`;
45
- messageKey = "banner.skipped.incompatibleForeign";
46
- } else {
47
- kind = "incompatible";
48
- vars = { producer: run.producerVersion ?? "?", schemaVersion: run.schemaVersion ?? "?" };
49
- key = `incompatible:${vars.producer}:${vars.schemaVersion}`;
50
- messageKey = "banner.skipped.incompatible";
51
- }
52
-
53
- const existing = groups.get(key);
54
- if (existing) {
55
- existing.items.push(run);
56
- } else {
57
- groups.set(key, { key, kind, messageKey, vars, items: [run] });
58
- }
59
- }
60
- return [...groups.values()];
61
- }
62
-
63
- function CopyCommandButton({ command, t }: { command: string; t: T }) {
64
- const [copied, setCopied] = useState(false);
65
- return (
66
- <button
67
- type="button"
68
- className={`ib-copy${copied ? " is-copied" : ""}`}
69
- title={t("banner.copyCommand")}
70
- onClick={async () => {
71
- await copyText(command);
72
- setCopied(true);
73
- setTimeout(() => setCopied(false), 1200);
74
- }}
75
- >
76
- {copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
77
- </button>
78
- );
79
- }
80
-
81
- export function SkippedRunsBanner({ skippedRuns, t }: { skippedRuns: SkippedRunNotice[]; t: T }) {
82
- const [expanded, setExpanded] = useState(false);
83
- if (skippedRuns.length === 0) return null;
84
-
85
- const groups = groupSkippedRuns(skippedRuns);
86
- const total = skippedRuns.length;
87
- let budget = expanded ? total : DEFAULT_VISIBLE;
88
- const hiddenCount = expanded ? 0 : Math.max(0, total - DEFAULT_VISIBLE);
89
-
90
- return (
91
- <section className="incompatible-banner" role="alert">
92
- <div className="ib-head">
93
- <AlertCircle className="ib-icon" aria-hidden="true" />
94
- <b>{t("banner.skippedTitle", { count: total })}</b>
95
- </div>
96
- <div className="ib-groups">
97
- {groups.map((group) => {
98
- const visibleItems = group.items.slice(0, Math.max(0, budget));
99
- budget -= visibleItems.length;
100
- const hiddenInGroup = group.items.length - visibleItems.length;
101
- return (
102
- <div className="ib-group" key={group.key}>
103
- <div className="ib-group-head">
104
- <span className="ib-group-count">{group.items.length}</span>
105
- <span className="ib-group-desc">{t(group.messageKey, group.vars)}</span>
106
- </div>
107
- {visibleItems.length > 0 && (
108
- <ul>
109
- {visibleItems.map((item) => (
110
- <li key={item.dir}>
111
- <span className="ib-dir">{item.dir}</span>
112
- {item.command && <CopyCommandButton command={item.command} t={t} />}
113
- </li>
114
- ))}
115
- {hiddenInGroup > 0 && !expanded && (
116
- <li className="ib-more" aria-hidden="true">
117
- +{hiddenInGroup}
118
- </li>
119
- )}
120
- </ul>
121
- )}
122
- </div>
123
- );
124
- })}
125
- </div>
126
- {hiddenCount > 0 && (
127
- <button type="button" className="ib-toggle" onClick={() => setExpanded(true)}>
128
- <ChevronRight aria-hidden="true" />
129
- {t("banner.expandRest", { count: hiddenCount })}
130
- </button>
131
- )}
132
- {expanded && total > DEFAULT_VISIBLE && (
133
- <button type="button" className="ib-toggle" onClick={() => setExpanded(false)}>
134
- <ChevronRight className="is-open" aria-hidden="true" />
135
- {t("banner.collapse")}
136
- </button>
137
- )}
138
- </section>
139
- );
140
- }
@@ -1,80 +0,0 @@
1
- import { useState } from "react";
2
- import type { RowRun, T } from "../shared.ts";
3
- import { verdictClass, verdictLabel } from "../lib/verdict.ts";
4
- import { formatCost, formatDateTime, formatDuration, formatTokens, totalTokens } from "../lib/format.ts";
5
-
6
- /** 全部历史 attempt 打平成一张表;列表在 App 里由 flattenAttempts(全部快照,已跨快照去重)算好。 */
7
- export function AttemptsView({ attempts, t }: { attempts: RowRun[]; t: T }) {
8
- const [query, setQuery] = useState("");
9
- const allRuns = attempts;
10
- const filtered = allRuns.filter((r: RowRun) => {
11
- const q = query.trim().toLowerCase();
12
- return !q || `${r.id} ${r.rowLabel} ${r.rowAgent} ${r.rowModel || ""}`.toLowerCase().includes(q);
13
- });
14
- return (
15
- <section id="tab-attempts">
16
- <div className="section-head">
17
- <h2>{t("section.attempts")}</h2>
18
- <div className="controls">
19
- <input
20
- className="search"
21
- type="search"
22
- placeholder={t("search.attempts")}
23
- autoComplete="off"
24
- value={query}
25
- onChange={(e) => setQuery(e.target.value)}
26
- />
27
- </div>
28
- </div>
29
- {!allRuns.length ? (
30
- <div className="empty">{t("empty.attempts")}</div>
31
- ) : (
32
- <div className="table-wrap">
33
- <table>
34
- <thead>
35
- <tr>
36
- <th>{t("table.evalId")}</th>
37
- <th>{t("table.experiment")}</th>
38
- <th>{t("table.verdict")}</th>
39
- <th>{t("table.agent")}</th>
40
- <th>{t("table.model")}</th>
41
- <th>{t("metric.duration")}</th>
42
- <th>{t("table.tokens")}</th>
43
- <th>{t("table.estCost")}</th>
44
- <th>{t("table.ranAt")}</th>
45
- </tr>
46
- </thead>
47
- <tbody>
48
- {filtered.length ? (
49
- filtered.map((r: RowRun) => {
50
- const verdict = r.verdict;
51
- return (
52
- <tr key={`${r.id}-${r.rowLabel}-${r.attempt}`}>
53
- <td>
54
- <span className="name">{r.id}</span>
55
- </td>
56
- <td>{r.rowLabel}</td>
57
- <td className={verdictClass(verdict)}>{verdictLabel(verdict, t)}</td>
58
- <td>{r.rowAgent}</td>
59
- <td>{r.rowModel || t("config.default")}</td>
60
- <td className="num">{formatDuration(r.durationMs)}</td>
61
- <td className="num">{formatTokens(totalTokens(r.usage))}</td>
62
- <td className="num">{formatCost(r.estimatedCostUSD)}</td>
63
- <td className="num">{r.startedAt ? formatDateTime(r.startedAt) : "-"}</td>
64
- </tr>
65
- );
66
- })
67
- ) : (
68
- <tr>
69
- <td colSpan={9} style={{ textAlign: "center", color: "var(--muted)" }}>
70
- {t("empty.attemptsFilter")}
71
- </td>
72
- </tr>
73
- )}
74
- </tbody>
75
- </table>
76
- </div>
77
- )}
78
- </section>
79
- );
80
- }
@@ -1,35 +0,0 @@
1
- import type { RowRun, T } from "../shared.ts";
2
- import { verdictClass, verdictLabel } from "../lib/verdict.ts";
3
- import { formatDuration } from "../lib/format.ts";
4
- import { LazyArtifact } from "../components/LazyArtifact.tsx";
5
-
6
- /** 全部历史 attempt 的 trace/transcript 视图;列表在 App 里由 flattenAttempts 算好。 */
7
- export function TracesView({ attempts, t }: { attempts: RowRun[]; t: T }) {
8
- const traceable = attempts.filter((r: RowRun) => r.hasEvents || r.hasTrace);
9
- return (
10
- <section id="tab-traces">
11
- <div className="section-head">
12
- <h2>{t("section.traces")}</h2>
13
- </div>
14
- {!traceable.length ? (
15
- <div className="empty">{t("empty.traces")}</div>
16
- ) : (
17
- traceable.map((r: RowRun) => {
18
- const verdict = r.verdict;
19
- return (
20
- <div className="traces-entry" key={`${r.id}-${r.rowLabel}-${r.attempt}`}>
21
- <div className="traces-entry-head">
22
- <span className={`${verdictClass(verdict)} traces-verdict`}>{verdictLabel(verdict, t)}</span>
23
- <span className="eval-id">{r.id}</span>
24
- <span className="traces-exp">{r.rowLabel}</span>
25
- <span className="num traces-dur">{formatDuration(r.durationMs)}</span>
26
- </div>
27
- {r.hasEvents && r.artifactBase ? <LazyArtifact type="transcript" src={`${r.artifactBase}/events.json`} t={t} /> : null}
28
- {r.hasTrace && r.artifactBase ? <LazyArtifact type="trace" src={`${r.artifactBase}/trace.json`} t={t} /> : null}
29
- </div>
30
- );
31
- })
32
- )}
33
- </section>
34
- );
35
- }