niceeval 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/README.md +4 -2
- package/README.zh.md +4 -2
- package/dist/agents/types.d.ts +2 -2
- package/dist/o11y/types.d.ts +3 -3
- package/dist/report/assets/colors.d.ts +1 -1
- package/dist/report/assets/colors.js +1 -1
- package/dist/report/components/attempt-detail/AttemptSource.js +1 -2
- package/dist/report/components/attempt-detail/compute.d.ts +4 -4
- package/dist/report/components/attempt-detail/compute.js +6 -6
- package/dist/report/components/attempt-detail/faces.d.ts +1 -1
- package/dist/report/components/attempt-detail/faces.js +3 -3
- package/dist/report/components/entity-lists/AttemptList.d.ts +3 -3
- package/dist/report/components/entity-lists/AttemptList.js +3 -3
- package/dist/report/components/entity-lists/ExperimentList.js +2 -2
- package/dist/report/components/entity-lists/compute.js +1 -1
- package/dist/report/components/entity-lists/faces.js +1 -1
- package/dist/report/components/entity-lists/index.d.ts +1 -1
- package/dist/report/components/entity-lists/index.js +1 -1
- package/dist/report/components/metric-views/chart-math.d.ts +2 -2
- package/dist/report/components/metric-views/chart-math.js +2 -2
- package/dist/report/components/metric-views/compute.d.ts +2 -3
- package/dist/report/components/metric-views/compute.js +3 -4
- package/dist/report/components/shared.d.ts +1 -1
- package/dist/report/components/site-components/compute.d.ts +5 -5
- package/dist/report/components/site-components/compute.js +6 -6
- package/dist/report/components/site-components/index.d.ts +7 -7
- package/dist/report/components/site-components/index.js +7 -7
- package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
- package/dist/report/components/site-components/scope-warnings.js +3 -3
- package/dist/report/components/site-components/snapshot-diagnostics.js +1 -1
- package/dist/report/components/summaries/compute.d.ts +1 -1
- package/dist/report/components/summaries/compute.js +2 -2
- package/dist/report/components/summaries/faces.js +1 -1
- package/dist/report/components/summaries/index.d.ts +1 -1
- package/dist/report/components/summaries/index.js +1 -1
- package/dist/report/index.js +1 -1
- package/dist/report/model/aggregate.d.ts +2 -2
- package/dist/report/model/aggregate.js +2 -2
- package/dist/report/model/format.d.ts +2 -2
- package/dist/report/model/format.js +2 -2
- package/dist/report/model/metrics.js +3 -3
- package/dist/report/model/types.d.ts +11 -12
- package/dist/results/locator.js +1 -1
- package/dist/results/select.d.ts +4 -4
- package/dist/results/select.js +5 -5
- package/dist/results/types.d.ts +8 -8
- package/dist/runner/types.d.ts +14 -14
- package/dist/sandbox/resolve.d.ts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +21 -0
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/report-components.mdx +156 -74
- package/docs-site/zh/tutorials/custom-reports.mdx +112 -22
- package/docs-site/zh/tutorials/sandbox-agent.mdx +3 -4
- package/docs-site/zh/tutorials/sandbox-providers.mdx +15 -15
- package/docs-site/zh/tutorials/theming.mdx +169 -0
- package/package.json +2 -1
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/bub-install-spec.ts +26 -8
- package/src/agents/bub.ts +40 -15
- package/src/agents/builtin.ts +11 -1
- package/src/agents/coding-cli-versions.ts +67 -0
- package/src/agents/hermes.ts +235 -0
- package/src/agents/index.ts +4 -0
- package/src/agents/langgraph.test.ts +1 -1
- package/src/agents/manifest.ts +1 -1
- package/src/agents/openai-compat.test.ts +1 -1
- package/src/agents/openclaw.ts +103 -24
- package/src/agents/opencode.ts +183 -0
- package/src/agents/sdk-streams.test.ts +1 -1
- package/src/agents/shared.ts +13 -2
- package/src/agents/types.ts +2 -2
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/o11y/cost.ts +1 -1
- package/src/o11y/parsers/bub.test.ts +1 -1
- package/src/o11y/parsers/bub.ts +1 -1
- package/src/o11y/parsers/codex.test.ts +1 -1
- package/src/o11y/parsers/codex.ts +1 -1
- package/src/o11y/parsers/hermes.ts +198 -0
- package/src/o11y/parsers/openclaw.ts +25 -3
- package/src/o11y/parsers/opencode.ts +295 -0
- package/src/o11y/types.ts +3 -3
- package/src/report/assets/colors.ts +1 -1
- package/src/report/assets/styles.css +1 -1
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptConversation.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiff.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptError.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptSource.tsx +2 -3
- package/src/report/components/attempt-detail/AttemptSummary.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTrace.tsx +1 -1
- package/src/report/components/attempt-detail/UsageTable.tsx +1 -1
- package/src/report/components/attempt-detail/attempt-components.test.tsx +1 -1
- package/src/report/components/attempt-detail/compute.ts +6 -6
- package/src/report/components/attempt-detail/faces.ts +3 -3
- package/src/report/components/attempt-detail/index.tsx +1 -1
- package/src/report/components/entity-lists/AttemptList.tsx +3 -3
- package/src/report/components/entity-lists/ExperimentList.tsx +2 -2
- package/src/report/components/entity-lists/compute.ts +1 -1
- package/src/report/components/entity-lists/faces.ts +1 -1
- package/src/report/components/entity-lists/index.tsx +1 -1
- package/src/report/components/metric-views/DeltaTable.tsx +1 -2
- package/src/report/components/metric-views/StabilityMatrix.tsx +1 -1
- package/src/report/components/metric-views/chart-math.test.ts +1 -1
- package/src/report/components/metric-views/chart-math.ts +2 -2
- package/src/report/components/metric-views/compute.ts +3 -4
- package/src/report/components/shared.ts +1 -1
- package/src/report/components/site-components/CopyFixPrompt.tsx +1 -1
- package/src/report/components/site-components/HeroCard.tsx +1 -1
- package/src/report/components/site-components/PoweredBy.tsx +1 -1
- package/src/report/components/site-components/ScopeWarnings.tsx +1 -1
- package/src/report/components/site-components/SnapshotDiagnostics.tsx +1 -1
- package/src/report/components/site-components/TraceWaterfall.tsx +1 -1
- package/src/report/components/site-components/compute.ts +6 -6
- package/src/report/components/site-components/index.tsx +7 -7
- package/src/report/components/site-components/scope-warnings.ts +3 -3
- package/src/report/components/site-components/snapshot-diagnostics.ts +1 -1
- package/src/report/components/summaries/ScopeSummary.tsx +2 -2
- package/src/report/components/summaries/compute.ts +2 -2
- package/src/report/components/summaries/faces.ts +1 -1
- package/src/report/components/summaries/index.tsx +1 -1
- package/src/report/index.ts +1 -1
- package/src/report/model/aggregate.ts +2 -2
- package/src/report/model/format.ts +2 -2
- package/src/report/model/metrics.ts +3 -3
- package/src/report/model/types.ts +12 -13
- package/src/results/annotated-source.test.ts +1 -1
- package/src/results/attempt-evidence.test.ts +1 -1
- package/src/results/copy.ts +3 -3
- package/src/results/format.ts +3 -3
- package/src/results/host-equivalence.test.ts +6 -6
- package/src/results/index.ts +1 -1
- package/src/results/locator.test.ts +2 -2
- package/src/results/locator.ts +1 -1
- package/src/results/open.ts +2 -2
- package/src/results/publish.ts +1 -1
- package/src/results/results.test.ts +6 -323
- package/src/results/select.test.ts +438 -0
- package/src/results/select.ts +6 -6
- package/src/results/skipped-notice.ts +1 -1
- package/src/results/truncate.ts +2 -2
- package/src/results/types.ts +9 -9
- package/src/results/writer.ts +4 -4
- package/src/runner/attempt.test.ts +2 -2
- package/src/runner/attempt.ts +6 -6
- package/src/runner/eval-source.test.ts +1 -1
- package/src/runner/eval-source.ts +1 -1
- package/src/runner/reporters/artifacts.ts +1 -1
- package/src/runner/run.ts +2 -2
- package/src/runner/timing.ts +1 -1
- package/src/runner/types.ts +14 -14
- package/src/sandbox/docker-agent-image.ts +36 -0
- package/src/sandbox/e2b-agent-template.test.ts +6 -21
- package/src/sandbox/e2b-agent-template.ts +44 -11
- package/src/sandbox/index.ts +8 -0
- package/src/sandbox/official-baselines.test.ts +175 -0
- package/src/sandbox/resolve.ts +1 -1
- package/src/sandbox/types.ts +1 -1
- package/src/scoring/diff.ts +1 -1
- package/src/scoring/types.ts +1 -1
- package/src/shared/facts.ts +1 -1
- package/src/show/index.ts +1 -1
- package/src/show/render.ts +1 -1
- package/src/show/show.test.ts +3 -3
- package/src/view/data.test.ts +2 -2
- package/src/view/data.ts +1 -1
- package/src/view/view-report.test.ts +1 -1
|
@@ -19,7 +19,7 @@ export interface HeroProps {
|
|
|
19
19
|
* 合成物,没有单独的 spec 等价形。web 面渲染 hero 标题(h1)、按渲染 locale 格式化的运行
|
|
20
20
|
* meta(latestStartedAt 为 null 时内置「暂无运行」文案)与品牌行(等同 PoweredBy,恒含、
|
|
21
21
|
* 无拆除 prop);text 面输出标题行与 meta 行,不含品牌行
|
|
22
|
-
* (docs/feature/reports/
|
|
22
|
+
* (docs/feature/reports/components/site/hero-card.md)。
|
|
23
23
|
*/
|
|
24
24
|
export declare const HeroCard: ReportComponent<HeroCardProps>;
|
|
25
25
|
/** `HeroCard` 的 props:标题 + `heroData()` 的产物,只有 data 形态。 */
|
|
@@ -32,14 +32,14 @@ export interface HeroCardProps {
|
|
|
32
32
|
* `Hero`:页首的站点标题区——标题、最后运行时间、快照合成来源,恒含品牌行。官方组合组件,
|
|
33
33
|
* 与手写 `<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} />`
|
|
34
34
|
* 严格等价、没有私有能力;读 `ctx.report` 意味着输出跟随站点,要站点无关的标题区直接用
|
|
35
|
-
* `HeroCard` 显式传值(docs/feature/reports/
|
|
35
|
+
* `HeroCard` 显式传值(docs/feature/reports/components/site/hero.md)。
|
|
36
36
|
*/
|
|
37
37
|
export declare const Hero: ReportComponent<HeroProps>;
|
|
38
38
|
/**
|
|
39
39
|
* `PoweredBy`:唯一的品牌件,无 props 双面组件。web 面渲染指向 niceeval 官网的一行品牌色
|
|
40
40
|
* 小字(`utm_source=report&utm_medium=powered-by`,`rel` 仅 `noopener` 以保留 Referer);
|
|
41
41
|
* text 面零输出。没有任何配置——品牌契约是「提供一个组件,不给开关」:不想要品牌就不用
|
|
42
|
-
* 这些组件、自己写替代组件(docs/feature/reports/
|
|
42
|
+
* 这些组件、自己写替代组件(docs/feature/reports/components/site/powered-by.md)。
|
|
43
43
|
*/
|
|
44
44
|
export declare const PoweredBy: ReportComponent<Record<never, never>>;
|
|
45
45
|
/** `ScopeWarnings` 的 props:spec 形态取宿主 Scope 的 warnings,data 形态收 `ScopeWarning[]`。 */
|
|
@@ -49,7 +49,7 @@ export type ScopeWarningsProps = DataProps<readonly ScopeWarning[], Record<never
|
|
|
49
49
|
* 按「下一步动作」聚合渲染(带 experimentId 的按实验聚合、非实验作用域按 kind 聚合;
|
|
50
50
|
* 实验作用域组在前、非实验作用域组在后);web 面组头带去重后的可复制命令、明细收原生
|
|
51
51
|
* `<details>`(总条数 ≤ 3 默认展开),text 面同构但不折叠。空警告集与裸 `Snapshot[]` 输入
|
|
52
|
-
* 两面零输出(docs/feature/reports/
|
|
52
|
+
* 两面零输出(docs/feature/reports/components/site/sample-warnings.md)。
|
|
53
53
|
*/
|
|
54
54
|
export declare const ScopeWarnings: ReportComponent<ScopeWarningsProps>;
|
|
55
55
|
/** `SnapshotDiagnostics` 的 props:spec 形态从宿主 `Scope | Snapshot[]` 投影,data 形态收 `SnapshotDiagnosticsData`。 */
|
|
@@ -60,7 +60,7 @@ export type SnapshotDiagnosticsProps = DataProps<SnapshotDiagnosticsData, Record
|
|
|
60
60
|
* web 面整个区域是默认折叠的原生 `<details>`,`<summary>` 恒可见汇总涉及的 experiment 数、
|
|
61
61
|
* Snapshot 数、记录数(按 count 计数)与最高严重度;单诊断快照退化成一行,不摆空壳层级;
|
|
62
62
|
* text 面同构但不折叠。空诊断集两面零输出
|
|
63
|
-
* (docs/feature/reports/
|
|
63
|
+
* (docs/feature/reports/components/site/run-diagnostics.md)。
|
|
64
64
|
*/
|
|
65
65
|
export declare const SnapshotDiagnostics: ReportComponent<SnapshotDiagnosticsProps>;
|
|
66
66
|
/** `CopyFixPrompt` 的 props:spec 形态无选项,data 形态收 `copyFixPromptData()` 的产物。 */
|
|
@@ -69,7 +69,7 @@ export type CopyFixPromptProps = DataProps<CopyFixPromptData, Record<never, neve
|
|
|
69
69
|
* `CopyFixPrompt`:把当前范围的全部失败整理成一段可交给 coding agent 的修复 prompt。
|
|
70
70
|
* prompt 在 resolve 阶段算好、烘进静态 HTML,无 JS 时在折叠块里完整可读,「复制」是增强层
|
|
71
71
|
* 行为;`failures` 为 0 时两面零输出;text 面恒零输出——终端里的等价能力是 `show` 的
|
|
72
|
-
* attempt 下钻命令本身(docs/feature/reports/
|
|
72
|
+
* attempt 下钻命令本身(docs/feature/reports/components/site/copy-fix-prompt.md)。
|
|
73
73
|
*/
|
|
74
74
|
export declare const CopyFixPrompt: ReportComponent<CopyFixPromptProps>;
|
|
75
75
|
/** `TraceWaterfall` 的 props:spec 形态无选项,data 形态收 `traceWaterfallData()` 的产物。 */
|
|
@@ -81,6 +81,6 @@ export type TraceWaterfallProps = DataProps<readonly TraceWaterfallRow[], Record
|
|
|
81
81
|
* 的原始 span(agent / model / tool)。web 面静态渲染顶层 span 分解条(失败 span 带失败
|
|
82
82
|
* 标记),行链接 attempt 详情;text 面每 attempt 一行(locator、总耗时、span 计数与失败
|
|
83
83
|
* 标记)+ 可复制的 `--timing` 下钻命令。trace 缺失的行照常出现并如实显示缺失;runner
|
|
84
|
-
* 生命周期节点不进瀑布(docs/feature/reports/
|
|
84
|
+
* 生命周期节点不进瀑布(docs/feature/reports/components/site/trace-waterfall.md)。
|
|
85
85
|
*/
|
|
86
86
|
export declare const TraceWaterfall: ReportComponent<TraceWaterfallProps>;
|
|
@@ -149,7 +149,7 @@ const assertHeroData = (data) => {
|
|
|
149
149
|
* 合成物,没有单独的 spec 等价形。web 面渲染 hero 标题(h1)、按渲染 locale 格式化的运行
|
|
150
150
|
* meta(latestStartedAt 为 null 时内置「暂无运行」文案)与品牌行(等同 PoweredBy,恒含、
|
|
151
151
|
* 无拆除 prop);text 面输出标题行与 meta 行,不含品牌行
|
|
152
|
-
* (docs/feature/reports/
|
|
152
|
+
* (docs/feature/reports/components/site/hero-card.md)。
|
|
153
153
|
*/
|
|
154
154
|
export const HeroCard = defineComponent({
|
|
155
155
|
web: (props, ctx) => {
|
|
@@ -166,7 +166,7 @@ HeroCard.displayName = "HeroCard";
|
|
|
166
166
|
* `Hero`:页首的站点标题区——标题、最后运行时间、快照合成来源,恒含品牌行。官方组合组件,
|
|
167
167
|
* 与手写 `<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} />`
|
|
168
168
|
* 严格等价、没有私有能力;读 `ctx.report` 意味着输出跟随站点,要站点无关的标题区直接用
|
|
169
|
-
* `HeroCard` 显式传值(docs/feature/reports/
|
|
169
|
+
* `HeroCard` 显式传值(docs/feature/reports/components/site/hero.md)。
|
|
170
170
|
*/
|
|
171
171
|
export const Hero = defineComponent(async ({ title, className }, ctx) => (_jsx(HeroCard, { title: title ?? ctx.report.title, data: await heroData(ctx.scope), className: className })));
|
|
172
172
|
Hero.displayName = "Hero";
|
|
@@ -174,7 +174,7 @@ Hero.displayName = "Hero";
|
|
|
174
174
|
* `PoweredBy`:唯一的品牌件,无 props 双面组件。web 面渲染指向 niceeval 官网的一行品牌色
|
|
175
175
|
* 小字(`utm_source=report&utm_medium=powered-by`,`rel` 仅 `noopener` 以保留 Referer);
|
|
176
176
|
* text 面零输出。没有任何配置——品牌契约是「提供一个组件,不给开关」:不想要品牌就不用
|
|
177
|
-
* 这些组件、自己写替代组件(docs/feature/reports/
|
|
177
|
+
* 这些组件、自己写替代组件(docs/feature/reports/components/site/powered-by.md)。
|
|
178
178
|
*/
|
|
179
179
|
export const PoweredBy = defineComponent({
|
|
180
180
|
web: () => _jsx(PoweredByWeb, {}),
|
|
@@ -186,7 +186,7 @@ PoweredBy.displayName = "PoweredBy";
|
|
|
186
186
|
* 按「下一步动作」聚合渲染(带 experimentId 的按实验聚合、非实验作用域按 kind 聚合;
|
|
187
187
|
* 实验作用域组在前、非实验作用域组在后);web 面组头带去重后的可复制命令、明细收原生
|
|
188
188
|
* `<details>`(总条数 ≤ 3 默认展开),text 面同构但不折叠。空警告集与裸 `Snapshot[]` 输入
|
|
189
|
-
* 两面零输出(docs/feature/reports/
|
|
189
|
+
* 两面零输出(docs/feature/reports/components/site/sample-warnings.md)。
|
|
190
190
|
*/
|
|
191
191
|
export const ScopeWarnings = makeDataComponent({
|
|
192
192
|
name: "ScopeWarnings",
|
|
@@ -204,7 +204,7 @@ export const ScopeWarnings = makeDataComponent({
|
|
|
204
204
|
* web 面整个区域是默认折叠的原生 `<details>`,`<summary>` 恒可见汇总涉及的 experiment 数、
|
|
205
205
|
* Snapshot 数、记录数(按 count 计数)与最高严重度;单诊断快照退化成一行,不摆空壳层级;
|
|
206
206
|
* text 面同构但不折叠。空诊断集两面零输出
|
|
207
|
-
* (docs/feature/reports/
|
|
207
|
+
* (docs/feature/reports/components/site/run-diagnostics.md)。
|
|
208
208
|
*/
|
|
209
209
|
export const SnapshotDiagnostics = makeDataComponent({
|
|
210
210
|
name: "SnapshotDiagnostics",
|
|
@@ -220,7 +220,7 @@ export const SnapshotDiagnostics = makeDataComponent({
|
|
|
220
220
|
* `CopyFixPrompt`:把当前范围的全部失败整理成一段可交给 coding agent 的修复 prompt。
|
|
221
221
|
* prompt 在 resolve 阶段算好、烘进静态 HTML,无 JS 时在折叠块里完整可读,「复制」是增强层
|
|
222
222
|
* 行为;`failures` 为 0 时两面零输出;text 面恒零输出——终端里的等价能力是 `show` 的
|
|
223
|
-
* attempt 下钻命令本身(docs/feature/reports/
|
|
223
|
+
* attempt 下钻命令本身(docs/feature/reports/components/site/copy-fix-prompt.md)。
|
|
224
224
|
*/
|
|
225
225
|
export const CopyFixPrompt = makeDataComponent({
|
|
226
226
|
name: "CopyFixPrompt",
|
|
@@ -237,7 +237,7 @@ export const CopyFixPrompt = makeDataComponent({
|
|
|
237
237
|
* 的原始 span(agent / model / tool)。web 面静态渲染顶层 span 分解条(失败 span 带失败
|
|
238
238
|
* 标记),行链接 attempt 详情;text 面每 attempt 一行(locator、总耗时、span 计数与失败
|
|
239
239
|
* 标记)+ 可复制的 `--timing` 下钻命令。trace 缺失的行照常出现并如实显示缺失;runner
|
|
240
|
-
* 生命周期节点不进瀑布(docs/feature/reports/
|
|
240
|
+
* 生命周期节点不进瀑布(docs/feature/reports/components/site/trace-waterfall.md)。
|
|
241
241
|
*/
|
|
242
242
|
export const TraceWaterfall = makeDataComponent({
|
|
243
243
|
name: "TraceWaterfall",
|
|
@@ -27,6 +27,6 @@ export declare function warningDetailsLabel(locale: ReportLocale, n: number): st
|
|
|
27
27
|
* 其余(含未登记 kind)按 kind 聚合。组排序:实验作用域组在前(按实验 id 字典序),
|
|
28
28
|
* 非实验作用域组在后(按 kind);类别两档制(integrity / freshness)已随 stale-snapshot /
|
|
29
29
|
* partial-coverage 一并删除——三种 warning kind 都是同一类完整性事实,不再需要档位区分
|
|
30
|
-
* (docs/feature/reports/
|
|
30
|
+
* (docs/feature/reports/components/site/sample-warnings.md「聚合轴是动作,不是发生顺序」)。
|
|
31
31
|
*/
|
|
32
32
|
export declare function groupScopeWarnings(input: readonly ScopeWarning[], locale: ReportLocale): GroupedScopeWarnings;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// ScopeWarnings 的聚合层:把 Scope 警告按「下一步动作」组织成组,web / text 两面共用
|
|
2
|
-
// (docs/feature/reports/
|
|
2
|
+
// (docs/feature/reports/components/site/sample-warnings.md「聚合轴是动作,不是发生顺序」)。
|
|
3
3
|
// message 是完整叙述的单源,这里只组织、不改写;徽标 / 组头文案按 kind 表登记的模板
|
|
4
|
-
// (docs/feature/
|
|
4
|
+
// (docs/feature/record/library.md「警告 kind 全集」)经 locale 词典渲染,未登记的 kind
|
|
5
5
|
// 回退为单独成组、逐条 message 原样。
|
|
6
6
|
import { localeText } from "../../model/locale.js";
|
|
7
7
|
/** 实验作用域且登记了徽标模板的 kind 才进实验组;其余(含未登记 kind)按 kind 聚合。 */
|
|
@@ -31,7 +31,7 @@ function dedupeCommand(members) {
|
|
|
31
31
|
* 其余(含未登记 kind)按 kind 聚合。组排序:实验作用域组在前(按实验 id 字典序),
|
|
32
32
|
* 非实验作用域组在后(按 kind);类别两档制(integrity / freshness)已随 stale-snapshot /
|
|
33
33
|
* partial-coverage 一并删除——三种 warning kind 都是同一类完整性事实,不再需要档位区分
|
|
34
|
-
* (docs/feature/reports/
|
|
34
|
+
* (docs/feature/reports/components/site/sample-warnings.md「聚合轴是动作,不是发生顺序」)。
|
|
35
35
|
*/
|
|
36
36
|
export function groupScopeWarnings(input, locale) {
|
|
37
37
|
const warnings = input;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// SnapshotDiagnostics 的聚合层:把 snapshotDiagnosticsData 的产物按来源 experiment 分组,
|
|
2
|
-
// web / text 两面共用(docs/feature/reports/
|
|
2
|
+
// web / text 两面共用(docs/feature/reports/components/site/run-diagnostics.md)。
|
|
3
3
|
// 输入已经由 snapshotDiagnosticsData 按 experiment id 字典序、组内 startedAt 新到旧排好序,
|
|
4
4
|
// 这里只按 experimentId 分桶,不重新排序、不跨来源合并 DiagnosticRecord。
|
|
5
5
|
import { localeText } from "../../model/locale.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ReportInput, ScopeSummaryData } from "../../model/types.ts";
|
|
2
2
|
/**
|
|
3
3
|
* `scopeSummaryData(input)`:范围摘要——快照时间窗、experiment / eval / attempt 数、
|
|
4
|
-
* 两级判定计票、端到端通过率与总成本(docs/feature/reports/
|
|
4
|
+
* 两级判定计票、端到端通过率与总成本(docs/feature/reports/components/summaries/sample-summary.md)。
|
|
5
5
|
* data 恒携带两级计票;通过率来自官方两级指标引擎,不从任一计票重算。
|
|
6
6
|
*/
|
|
7
7
|
export declare function scopeSummaryData(input: ReportInput): Promise<ScopeSummaryData>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// 计算函数(*Data):ReportInput → 一份组件数据。ScopeSummary 的 scopeSummaryData 住在这里
|
|
2
|
-
// (docs/feature/reports/
|
|
2
|
+
// (docs/feature/reports/components/summaries/README.md)。ExperimentComparison 是纯组合组件,不产生
|
|
3
3
|
// 独立的 data,没有对应的计算函数。
|
|
4
4
|
//
|
|
5
5
|
// 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
|
|
@@ -21,7 +21,7 @@ const totalCostMetric = defineMetric({
|
|
|
21
21
|
});
|
|
22
22
|
/**
|
|
23
23
|
* `scopeSummaryData(input)`:范围摘要——快照时间窗、experiment / eval / attempt 数、
|
|
24
|
-
* 两级判定计票、端到端通过率与总成本(docs/feature/reports/
|
|
24
|
+
* 两级判定计票、端到端通过率与总成本(docs/feature/reports/components/summaries/sample-summary.md)。
|
|
25
25
|
* data 恒携带两级计票;通过率来自官方两级指标引擎,不从任一计票重算。
|
|
26
26
|
*/
|
|
27
27
|
export async function scopeSummaryData(input) {
|
|
@@ -14,7 +14,7 @@ export function scopeSummaryText(data, votes, ctx) {
|
|
|
14
14
|
const tally = votes === "attempt" ? data.attemptVerdicts : data.evalVerdicts;
|
|
15
15
|
const head = [
|
|
16
16
|
// 计分制 Scope 隐藏通过率只显示总分;混型 Scope 两者都显示;纯通过制 Scope 只显示通过率
|
|
17
|
-
// (现状不变),见 docs/feature/reports/
|
|
17
|
+
// (现状不变),见 docs/feature/reports/components/summaries/sample-summary.md。
|
|
18
18
|
...(data.scoringComposition !== "points"
|
|
19
19
|
? [`${localeText(locale, "scopeSummary.passRate")} ${cellText(data.endToEndPassRate, locale)}`]
|
|
20
20
|
: []),
|
|
@@ -24,7 +24,7 @@ export type ExperimentComparisonProps = ComparisonChrome & {
|
|
|
24
24
|
* 「同引用 input + 深相等 spec」记忆化保证。主读数按 scoringComposition(input) 切换:
|
|
25
25
|
* "pass" 用 endToEndPassRate、"points" 用 totalScore;"mixed" 按题型把 input 拆成两个
|
|
26
26
|
* 子 Scope,散点与 ExperimentList 每组各一份、各用各的主读数,ScopeSummary 始终是整个
|
|
27
|
-
* input 一份(docs/feature/reports/
|
|
27
|
+
* input 一份(docs/feature/reports/components/summaries/experiment-comparison.md)。
|
|
28
28
|
*/
|
|
29
29
|
export declare const ExperimentComparison: ReportComponent<ExperimentComparisonProps>;
|
|
30
30
|
export {};
|
|
@@ -90,7 +90,7 @@ function filterInputBySnapshot(input, predicate) {
|
|
|
90
90
|
* 「同引用 input + 深相等 spec」记忆化保证。主读数按 scoringComposition(input) 切换:
|
|
91
91
|
* "pass" 用 endToEndPassRate、"points" 用 totalScore;"mixed" 按题型把 input 拆成两个
|
|
92
92
|
* 子 Scope,散点与 ExperimentList 每组各一份、各用各的主读数,ScopeSummary 始终是整个
|
|
93
|
-
* input 一份(docs/feature/reports/
|
|
93
|
+
* input 一份(docs/feature/reports/components/summaries/experiment-comparison.md)。
|
|
94
94
|
*/
|
|
95
95
|
export const ExperimentComparison = defineComponent(async (props, ctx) => {
|
|
96
96
|
const input = props.input ?? ctx.scope;
|
package/dist/report/index.js
CHANGED
|
@@ -27,7 +27,7 @@ export { ScopeSummary, ExperimentComparison } from "./components/summaries/index
|
|
|
27
27
|
export { AttemptList, EvalList, ExperimentList, FailureList } from "./components/entity-lists/index.js";
|
|
28
28
|
export { DeltaTable, MetricBars, MetricLine, MetricMatrix, MetricScatter, MetricTable, Scoreboard, StabilityMatrix, } from "./components/metric-views/index.js";
|
|
29
29
|
export { CopyFixPrompt, Hero, HeroCard, PoweredBy, ScopeWarnings, SnapshotDiagnostics, TraceWaterfall, } from "./components/site-components/index.js";
|
|
30
|
-
// Attempt 详情组件族(docs/feature/reports/
|
|
30
|
+
// Attempt 详情组件族(docs/feature/reports/components/attempt-detail/README.md):11 个叶子 + 2 个
|
|
31
31
|
// 只装配叶子的组合组件(AttemptAssessment / AttemptDetail),都从 niceeval/report 导出。
|
|
32
32
|
export { AttemptAssertions, AttemptAssessment, AttemptConversation, AttemptDetail, AttemptDiagnostics, AttemptDiff, AttemptError, AttemptFixPrompt, AttemptSource, AttemptSummary, AttemptTimeline, AttemptTrace, UsageTable, } from "./components/attempt-detail/index.js";
|
|
33
33
|
// 计算函数(组件解析面的具名形式,与组件成对;spec 形态下由管线代调,data 形态与
|
|
@@ -7,7 +7,7 @@ import { evalPrefixPredicate } from "../../shared/aggregate.ts";
|
|
|
7
7
|
* `ReportInput` 归一化:Scope 分支直接消费它已经按口径物化好的 `attempts`(不再重新
|
|
8
8
|
* flatten `snapshots`——真实 Snapshot 各自持有完整、未按 Scope 口径收窄的 evals/attempts,
|
|
9
9
|
* 只有 `Scope.attempts` 才是这次选择的真正结果);裸 `Snapshot[]` 分支没有挑选过程,取全部
|
|
10
|
-
* (docs/feature/
|
|
10
|
+
* (docs/feature/record/library.md「官方现刻水位」)。
|
|
11
11
|
*/
|
|
12
12
|
export declare function resolveInput(input: ReportInput): {
|
|
13
13
|
snapshots: readonly Snapshot[];
|
|
@@ -30,7 +30,7 @@ export interface Item {
|
|
|
30
30
|
export declare function experimentIdOf(item: Item): string;
|
|
31
31
|
export declare function evalIdOf(item: Item): string;
|
|
32
32
|
/**
|
|
33
|
-
* 历史执行判定(docs/feature/
|
|
33
|
+
* 历史执行判定(docs/feature/sample/library.md「时效:新执行与历史执行」):携带条目,或
|
|
34
34
|
* 真实来源(`item.snapshot`)早于该实验在这份输入里的水位基准(`item.watermark`)——
|
|
35
35
|
* latest() 口径下二者是同一个对象,比较恒假,只剩 carried 生效;current() 口径下贡献
|
|
36
36
|
* 来源可能有多个,水位基准是其中 startedAt 最新的一个。
|
|
@@ -17,7 +17,7 @@ const KEY_SEP = "\u0000";
|
|
|
17
17
|
* `ReportInput` 归一化:Scope 分支直接消费它已经按口径物化好的 `attempts`(不再重新
|
|
18
18
|
* flatten `snapshots`——真实 Snapshot 各自持有完整、未按 Scope 口径收窄的 evals/attempts,
|
|
19
19
|
* 只有 `Scope.attempts` 才是这次选择的真正结果);裸 `Snapshot[]` 分支没有挑选过程,取全部
|
|
20
|
-
* (docs/feature/
|
|
20
|
+
* (docs/feature/record/library.md「官方现刻水位」)。
|
|
21
21
|
*/
|
|
22
22
|
export function resolveInput(input) {
|
|
23
23
|
if (Array.isArray(input)) {
|
|
@@ -34,7 +34,7 @@ export function evalIdOf(item) {
|
|
|
34
34
|
return item.attempt.evalId || item.attempt.result.id;
|
|
35
35
|
}
|
|
36
36
|
/**
|
|
37
|
-
* 历史执行判定(docs/feature/
|
|
37
|
+
* 历史执行判定(docs/feature/sample/library.md「时效:新执行与历史执行」):携带条目,或
|
|
38
38
|
* 真实来源(`item.snapshot`)早于该实验在这份输入里的水位基准(`item.watermark`)——
|
|
39
39
|
* latest() 口径下二者是同一个对象,比较恒假,只剩 carried 生效;current() 口径下贡献
|
|
40
40
|
* 来源可能有多个,水位基准是其中 startedAt 最新的一个。
|
|
@@ -55,7 +55,7 @@ export declare function formatReportDateTimeRange(fromIso: string, toIso: string
|
|
|
55
55
|
* 历史执行的紧凑时距("3d" / "2h" / "5m" / "10s"):自 `startedAt` 起算,渲染时刻由调用方
|
|
56
56
|
* 传入(`nowIso` 缺省当前时刻)——粒度阈值复用 `gapParts`(与曾经的 stale-snapshot message
|
|
57
57
|
* 同一套单源,见 `results/select.ts`),只是这里的呈现是紧凑单字母,不是完整单词
|
|
58
|
-
* (docs/feature/reports/
|
|
58
|
+
* (docs/feature/reports/components/entity-lists/README.md「时效标注」)。
|
|
59
59
|
*/
|
|
60
60
|
export declare function formatHistoricalGap(startedAtIso: string, nowIso?: string): string;
|
|
61
61
|
/** passed / failed / errored / skipped 的判定符。 */
|
|
@@ -67,7 +67,7 @@ export declare function verdictMark(verdict: Verdict): string;
|
|
|
67
67
|
export declare function fitFailureSummary(summary: string, maxChars: number): string;
|
|
68
68
|
/**
|
|
69
69
|
* 一份 `ExperimentList` data 的题型构成:主读数列该显示 Pass rate、Total score,还是两者
|
|
70
|
-
* 并存(docs/feature/reports/
|
|
70
|
+
* 并存(docs/feature/reports/components/entity-lists/experiment-list.md 主读数列)。与
|
|
71
71
|
* `entity-lists/compute.ts` 里 `experimentListData` 默认排序专用的 `listScoringComposition`
|
|
72
72
|
* 同一套判据——跳过 `attempts === 0` 的行(coverage-only 占位,`scoring` 是占位默认值不是
|
|
73
73
|
* 读到的事实,一屏占位行不该把纯计分制列表误判成 mixed)。web 面与 text 面在这里读同一份
|
|
@@ -223,7 +223,7 @@ export function formatReportDateTimeRange(fromIso, toIso, locale) {
|
|
|
223
223
|
* 历史执行的紧凑时距("3d" / "2h" / "5m" / "10s"):自 `startedAt` 起算,渲染时刻由调用方
|
|
224
224
|
* 传入(`nowIso` 缺省当前时刻)——粒度阈值复用 `gapParts`(与曾经的 stale-snapshot message
|
|
225
225
|
* 同一套单源,见 `results/select.ts`),只是这里的呈现是紧凑单字母,不是完整单词
|
|
226
|
-
* (docs/feature/reports/
|
|
226
|
+
* (docs/feature/reports/components/entity-lists/README.md「时效标注」)。
|
|
227
227
|
*/
|
|
228
228
|
export function formatHistoricalGap(startedAtIso, nowIso = new Date().toISOString()) {
|
|
229
229
|
const { n, unit } = gapParts(startedAtIso, nowIso);
|
|
@@ -253,7 +253,7 @@ export function fitFailureSummary(summary, maxChars) {
|
|
|
253
253
|
// ── ExperimentList(web ExperimentList.tsx / text faces.ts)共用的题型构成判据 ──
|
|
254
254
|
/**
|
|
255
255
|
* 一份 `ExperimentList` data 的题型构成:主读数列该显示 Pass rate、Total score,还是两者
|
|
256
|
-
* 并存(docs/feature/reports/
|
|
256
|
+
* 并存(docs/feature/reports/components/entity-lists/experiment-list.md 主读数列)。与
|
|
257
257
|
* `entity-lists/compute.ts` 里 `experimentListData` 默认排序专用的 `listScoringComposition`
|
|
258
258
|
* 同一套判据——跳过 `attempts === 0` 的行(coverage-only 占位,`scoring` 是占位默认值不是
|
|
259
259
|
* 读到的事实,一屏占位行不该把纯计分制列表误判成 mixed)。web 面与 text 面在这里读同一份
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
// assistantTurns(assistant-turns) null 实测;o11y 缺失→null 同左 同左 lower
|
|
17
17
|
// repeatedFailedCommands(repeated-failed-commands) null 实测;o11y 缺失→null 同左 同左 lower
|
|
18
18
|
//
|
|
19
|
-
// bounds(自然边界,驱动图轴呼吸边距的钳制,见 docs/feature/reports/
|
|
20
|
-
//
|
|
19
|
+
// bounds(自然边界,驱动图轴呼吸边距的钳制,见 docs/feature/reports/components/charts/README.md
|
|
20
|
+
// 「值域」):三个通过率指标与 examScore 是 { min: 0, max: 1 };其余七个(totalScore、
|
|
21
21
|
// durationMs、tokens、costUSD、assistantTurns、repeatedFailedCommands)是 { min: 0 }。
|
|
22
22
|
//
|
|
23
23
|
// 两档指标(docs/feature/reports/library/metrics.md「内置指标」):以上除 assistantTurns 与
|
|
@@ -187,7 +187,7 @@ export const tokens = defineMetric({
|
|
|
187
187
|
return null;
|
|
188
188
|
const usage = a.result.usage;
|
|
189
189
|
// input/output 缺失(协议没提供)→ null,不拿 0 冒充「实测就是 0」
|
|
190
|
-
// (docs/feature/
|
|
190
|
+
// (docs/feature/record/architecture.md#usage)。
|
|
191
191
|
if (!usage || usage.inputTokens === undefined || usage.outputTokens === undefined)
|
|
192
192
|
return null;
|
|
193
193
|
// 只加 input + output:缓存读写量大但便宜,计进去会把缓存热的 agent 画成 token 大户;
|
|
@@ -40,7 +40,7 @@ export interface Metric<Name extends string = string> {
|
|
|
40
40
|
/**
|
|
41
41
|
* 指标值的自然边界(如通过率 0–1、成本下界 0)。图轴呼吸边距不越过声明的边界——
|
|
42
42
|
* 贴边数据点如实落在框线上(如通过率 100%),那是指标的自然边界,不是裁剪
|
|
43
|
-
* (docs/feature/reports/
|
|
43
|
+
* (docs/feature/reports/components/charts/README.md「值域」)。
|
|
44
44
|
*/
|
|
45
45
|
bounds?: {
|
|
46
46
|
min?: number;
|
|
@@ -114,7 +114,7 @@ export interface MetricColumn {
|
|
|
114
114
|
unit?: string;
|
|
115
115
|
/** 渲染提示:排序方向、轴向、涨跌配色。 */
|
|
116
116
|
better?: "higher" | "lower";
|
|
117
|
-
/** = metric.bounds,原样投影;图轴值域推定读这里(docs/feature/reports/
|
|
117
|
+
/** = metric.bounds,原样投影;图轴值域推定读这里(docs/feature/reports/components/charts/README.md「值域」)。 */
|
|
118
118
|
bounds?: {
|
|
119
119
|
min?: number;
|
|
120
120
|
max?: number;
|
|
@@ -139,7 +139,7 @@ export interface MetricCell {
|
|
|
139
139
|
refs: AttemptLocator[];
|
|
140
140
|
}
|
|
141
141
|
/**
|
|
142
|
-
* 数据形状的字段命名规则(docs/feature/reports/
|
|
142
|
+
* 数据形状的字段命名规则(docs/feature/reports/components/tables/README.md「共用数据形状」):
|
|
143
143
|
* 维度名字段 = 产生它的选项名 + `Dimension` 后缀,值是解析后的维度 name;
|
|
144
144
|
* 条目数组一律叫 `rows`(Matrix 的稀疏格子叫 `cells`);条目内的 key / series 是维度值,不带后缀。
|
|
145
145
|
*/
|
|
@@ -231,8 +231,7 @@ export interface ScoreboardData {
|
|
|
231
231
|
}>;
|
|
232
232
|
}
|
|
233
233
|
/**
|
|
234
|
-
* `DeltaTable` 的一格:同一条件值 × eval 的折叠(docs/feature/reports/
|
|
235
|
-
* 「DeltaTable」)。`verdict` / `totalScore` 用与榜单同一套题目级判定口径(`totalScore` 取各
|
|
234
|
+
* `DeltaTable` 的一格:同一条件值 × eval 的折叠(docs/feature/reports/components/tables/delta-table.md)。`verdict` / `totalScore` 用与榜单同一套题目级判定口径(`totalScore` 取各
|
|
236
235
|
* attempt 的均值);`totalTokens` / `totalCostUSD` 是该题在该条件下全部 attempt 的**合计**,
|
|
237
236
|
* 不是均值。
|
|
238
237
|
*/
|
|
@@ -343,7 +342,7 @@ export type ScoringComposition = "pass" | "points" | "mixed";
|
|
|
343
342
|
/**
|
|
344
343
|
* 一个范围的摘要:快照时间窗、experiment / eval / attempt 数、两级判定计票、端到端通过率
|
|
345
344
|
* 和总成本。eval 的身份键是 experimentId + evalId;data 恒携带两级计票,渲染面显示哪一级
|
|
346
|
-
* 由呈现 prop `votes` 决定,不改变 data(docs/feature/reports/
|
|
345
|
+
* 由呈现 prop `votes` 决定,不改变 data(docs/feature/reports/components/summaries/sample-summary.md)。
|
|
347
346
|
*/
|
|
348
347
|
export interface ScopeSummaryData {
|
|
349
348
|
/** 贡献当前数据的快照时间范围;空范围为 null,不编造当前时间。 */
|
|
@@ -375,7 +374,7 @@ export interface ScopeSummaryData {
|
|
|
375
374
|
totalCostUSD: MetricCell;
|
|
376
375
|
}
|
|
377
376
|
/**
|
|
378
|
-
* `HeroCard` 的数据(docs/feature/reports/
|
|
377
|
+
* `HeroCard` 的数据(docs/feature/reports/components/site/hero-card.md):站点标题区的
|
|
379
378
|
* 运行 meta——最后运行时间与快照合成来源。标题不在 data 里,它是站点声明与 Scope 的合成物,
|
|
380
379
|
* 经 `HeroCardProps.title` 传入。
|
|
381
380
|
*/
|
|
@@ -386,7 +385,7 @@ export interface HeroData {
|
|
|
386
385
|
snapshots: number;
|
|
387
386
|
}
|
|
388
387
|
/**
|
|
389
|
-
* `SnapshotDiagnostics` 一条来源快照的诊断投影(docs/feature/reports/
|
|
388
|
+
* `SnapshotDiagnostics` 一条来源快照的诊断投影(docs/feature/reports/components/site/run-diagnostics.md):
|
|
390
389
|
* 只携带 experimentId / startedAt / DiagnosticRecord,不带 Snapshot 本体、`evals` 或
|
|
391
390
|
* `AttemptHandle`,避免把文件读取能力拖进浏览器边界。
|
|
392
391
|
*/
|
|
@@ -402,7 +401,7 @@ export interface SnapshotDiagnosticsItem {
|
|
|
402
401
|
export type SnapshotDiagnosticsData = readonly SnapshotDiagnosticsItem[];
|
|
403
402
|
/**
|
|
404
403
|
* `CopyFixPrompt` 的数据:resolve 期算好的修复 prompt 全文与参与的失败数
|
|
405
|
-
* (docs/feature/reports/
|
|
404
|
+
* (docs/feature/reports/components/site/copy-fix-prompt.md)。
|
|
406
405
|
*/
|
|
407
406
|
export interface CopyFixPromptData {
|
|
408
407
|
/** 修复 prompt 全文;失败逐条含 eval id、主失败摘要与 attempt 下钻命令。 */
|
|
@@ -728,7 +727,7 @@ export interface AttemptConversationData {
|
|
|
728
727
|
locator: AttemptLocator;
|
|
729
728
|
rounds: AttemptConversationRound[];
|
|
730
729
|
/**
|
|
731
|
-
* `commands.json` 的投影(见 docs/feature/
|
|
730
|
+
* `commands.json` 的投影(见 docs/feature/record/architecture.md#commandsjson):按关联
|
|
732
731
|
* timing 节点(`timingNodeId` 对应 `PhaseTiming.children` 中 `kind === "command"` 的节点)
|
|
733
732
|
* 的 `startOffsetMs` 排序;关联不到 timing 节点的排在最后,不按数组偶然顺序猜时间。没有失败
|
|
734
733
|
* 命令时字段省略,不摆空数组。
|
|
@@ -744,14 +743,14 @@ export interface AttemptDiagnosticsData {
|
|
|
744
743
|
}
|
|
745
744
|
/**
|
|
746
745
|
* `UsageTable` 的 data:判定、轮数、工具调用数、token 拆分与成本摊成的单行用量摘要;组装口径单源
|
|
747
|
-
* 见 docs/feature/reports/
|
|
746
|
+
* 见 docs/feature/reports/components/attempt-detail/usage-table.md#组装口径单源。identity 字段
|
|
748
747
|
* (`locator`/`experimentId`/`evalId`/`attempt`/`verdict`)恒有——它们不是「usage 有没有」的一部分,
|
|
749
748
|
* 是这一行归属哪个 attempt 的身份。其余字段各自独立地只在事实真实存在时出现:
|
|
750
749
|
*
|
|
751
750
|
* - `turns`/`toolCalls`:events 派生(与 `o11y.json` 行为摘要同源),只在有非空 events 时出现,
|
|
752
751
|
* 哪怕派生值恰好是 0(有 events 但零轮/零工具调用是观测到的事实,不是缺失)。
|
|
753
752
|
* - `usage`:落盘 `Usage` 原样,只在 `result.usage` 存在时出现。桶恒互斥,`inputTokens` 本身
|
|
754
|
-
* 就是未缓存输入(契约见 docs/feature/
|
|
753
|
+
* 就是未缓存输入(契约见 docs/feature/record/architecture.md#usage);"uncached in" 标注只在
|
|
755
754
|
* `cacheReadTokens` 在场时由 face 层给出,不派生第二个字段。
|
|
756
755
|
* - `estimatedCostUSD`:能算出成本(`usage.costUSD` 或 `result.estimatedCostUSD`)时才出现。
|
|
757
756
|
*
|
package/dist/results/locator.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// AttemptLocator:Attempt 的不透明、可复制、可发布的短标识(定稿见 docs/feature/
|
|
1
|
+
// AttemptLocator:Attempt 的不透明、可复制、可发布的短标识(定稿见 docs/feature/record/library.md
|
|
2
2
|
// 「按 locator 寻址一个 attempt」、docs/concepts.md「Attempt 定位符」、docs/feature/reports/view.md 的 `#/attempt/@<locator>` 路由)。
|
|
3
3
|
//
|
|
4
4
|
// locator 由不可变身份元组 {experimentId, 快照 startedAt, evalId, attempt 下标(0-indexed)}
|
package/dist/results/select.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ export interface ResultScope {
|
|
|
20
20
|
fresh?: boolean;
|
|
21
21
|
}
|
|
22
22
|
/**
|
|
23
|
-
* current() 跨快照拼接的可比性前提所比较的字段集(docs/feature/
|
|
23
|
+
* current() 跨快照拼接的可比性前提所比较的字段集(docs/feature/record/library.md
|
|
24
24
|
* 「官方现刻水位」):会改变单题被测行为或判定的字段。runs / earlyExit / maxConcurrency /
|
|
25
25
|
* selectedEvalIds / evalFilterFingerprint / description 是编排与选题字段,不参与比较。
|
|
26
26
|
*/
|
|
@@ -40,7 +40,7 @@ export declare function deepEqualJson(a: unknown, b: unknown): boolean;
|
|
|
40
40
|
/**
|
|
41
41
|
* 一个快照实际选中的 eval id 全集:优先读落盘的 `experiment.selectedEvalIds`(niceeval 自己的
|
|
42
42
|
* 写入面必有此字段);第三方 harness 未实现该字段时退化为该快照实际写出的 `evals`,不把整份
|
|
43
|
-
* 来源排除在外(见 docs/feature/
|
|
43
|
+
* 来源排除在外(见 docs/feature/record/architecture.md「selectedEvalIds」)。
|
|
44
44
|
*/
|
|
45
45
|
export declare function selectedEvalIdsOf(snapshot: Snapshot): readonly string[];
|
|
46
46
|
/**
|
|
@@ -49,7 +49,7 @@ export declare function selectedEvalIdsOf(snapshot: Snapshot): readonly string[]
|
|
|
49
49
|
* 快照」,带 eval 前缀的局部重跑会产出残缺快照;现刻水位承诺「不会因为一次局部重跑变残缺」,
|
|
50
50
|
* 所以在实验的历史快照上逐 eval 向更早的 run 补齐——但补齐只发生在 `Scope.attempts` 这份
|
|
51
51
|
* 物化选择上,真正贡献过至少一道题的来源 Snapshot 原样进 `Scope.snapshots`,不重建、不
|
|
52
|
-
* 合并成报告专用对象(见 docs/feature/
|
|
52
|
+
* 合并成报告专用对象(见 docs/feature/record/library.md「官方现刻水位」)。
|
|
53
53
|
*
|
|
54
54
|
* **可比性前提**:每个 experiment 以最新快照的可比性配置(agent / model / reasoningEffort /
|
|
55
55
|
* flags / budget / timeoutMs / sandbox)为基准,只有配置与基准深相等的历史快照才参与补齐;
|
|
@@ -102,7 +102,7 @@ export declare function filterExperiments(experiments: Experiment[], filter?: st
|
|
|
102
102
|
/**
|
|
103
103
|
* 时效标注(`↩` + 人话时距)的粒度选择:选粒度最大的单位,四舍五入。结构化形态是单源——
|
|
104
104
|
* entity-lists 渲染面的紧凑时距("3d")与曾经的 stale-snapshot message 用同一套阈值,
|
|
105
|
-
* 阈值不写两份(docs/feature/reports/
|
|
105
|
+
* 阈值不写两份(docs/feature/reports/components/entity-lists/README.md「时效标注」)。
|
|
106
106
|
*/
|
|
107
107
|
export declare function gapParts(fromIso: string, toIso: string): {
|
|
108
108
|
n: number;
|
package/dist/results/select.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// 快照 Scope 与 attempt 去重(定稿见 docs/feature/
|
|
1
|
+
// 快照 Scope 与 attempt 去重(定稿见 docs/feature/record/library.md「选择快照」「官方现刻水位」「身份键与去重」)。
|
|
2
2
|
//
|
|
3
3
|
// 选择器长在集合上(results.latest() / results.current()),不是 DSL,只是最常用的两种口径。
|
|
4
4
|
// 选择器必须诚实:残缺、落后、未收尾都被算出来,以结构化 warnings 随 Scope 走 ——
|
|
@@ -23,7 +23,7 @@ export function selectLatest(results, opts) {
|
|
|
23
23
|
// fresh: true 只保留新执行的 attempt——在 latest() 口径下,选中集合永远只有这一份
|
|
24
24
|
// 快照,「所属快照早于该实验在 Scope 中最新快照」这条历史出身天生不成立(只有它自己),
|
|
25
25
|
// 唯一的历史出身是携带条目(attempt.carried)。只过滤显式物化的 attempts 集,不克隆/
|
|
26
|
-
// 改写来源 Snapshot 的 evals(见 docs/feature/
|
|
26
|
+
// 改写来源 Snapshot 的 evals(见 docs/feature/record/library.md「选择快照」)。
|
|
27
27
|
const picked = fresh ? raw.attempts.filter((a) => !a.carried) : raw.attempts;
|
|
28
28
|
attempts.push(...picked);
|
|
29
29
|
// 覆盖事实:分母 = 该实验已知 eval 并集(本地历史 ∪ 各快照携带的 knownEvalIds),
|
|
@@ -84,7 +84,7 @@ export function deepEqualJson(a, b) {
|
|
|
84
84
|
/**
|
|
85
85
|
* 一个快照实际选中的 eval id 全集:优先读落盘的 `experiment.selectedEvalIds`(niceeval 自己的
|
|
86
86
|
* 写入面必有此字段);第三方 harness 未实现该字段时退化为该快照实际写出的 `evals`,不把整份
|
|
87
|
-
* 来源排除在外(见 docs/feature/
|
|
87
|
+
* 来源排除在外(见 docs/feature/record/architecture.md「selectedEvalIds」)。
|
|
88
88
|
*/
|
|
89
89
|
export function selectedEvalIdsOf(snapshot) {
|
|
90
90
|
return snapshot.experiment?.selectedEvalIds ?? snapshot.evals.map((ev) => ev.id);
|
|
@@ -95,7 +95,7 @@ export function selectedEvalIdsOf(snapshot) {
|
|
|
95
95
|
* 快照」,带 eval 前缀的局部重跑会产出残缺快照;现刻水位承诺「不会因为一次局部重跑变残缺」,
|
|
96
96
|
* 所以在实验的历史快照上逐 eval 向更早的 run 补齐——但补齐只发生在 `Scope.attempts` 这份
|
|
97
97
|
* 物化选择上,真正贡献过至少一道题的来源 Snapshot 原样进 `Scope.snapshots`,不重建、不
|
|
98
|
-
* 合并成报告专用对象(见 docs/feature/
|
|
98
|
+
* 合并成报告专用对象(见 docs/feature/record/library.md「官方现刻水位」)。
|
|
99
99
|
*
|
|
100
100
|
* **可比性前提**:每个 experiment 以最新快照的可比性配置(agent / model / reasoningEffort /
|
|
101
101
|
* flags / budget / timeoutMs / sandbox)为基准,只有配置与基准深相等的历史快照才参与补齐;
|
|
@@ -347,7 +347,7 @@ export function filterExperiments(experiments, filter) {
|
|
|
347
347
|
/**
|
|
348
348
|
* 时效标注(`↩` + 人话时距)的粒度选择:选粒度最大的单位,四舍五入。结构化形态是单源——
|
|
349
349
|
* entity-lists 渲染面的紧凑时距("3d")与曾经的 stale-snapshot message 用同一套阈值,
|
|
350
|
-
* 阈值不写两份(docs/feature/reports/
|
|
350
|
+
* 阈值不写两份(docs/feature/reports/components/entity-lists/README.md「时效标注」)。
|
|
351
351
|
*/
|
|
352
352
|
export function gapParts(fromIso, toIso) {
|
|
353
353
|
const ms = Math.max(0, Date.parse(toIso) - Date.parse(fromIso));
|
package/dist/results/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { DiagnosticRecord, EvalResult, ExperimentRunInfo, LocalizedText } f
|
|
|
2
2
|
import type { FailedCommandEvidence, O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
|
|
3
3
|
import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
|
|
4
4
|
import type { AttemptLocator } from "./locator.ts";
|
|
5
|
-
/** attempt 级 artifact 的种类;文件名见 format.ts 的 artifactFileOf,布局见 docs/feature/
|
|
5
|
+
/** attempt 级 artifact 的种类;文件名见 format.ts 的 artifactFileOf,布局见 docs/feature/record/architecture.md。 */
|
|
6
6
|
export declare const ARTIFACT_KINDS: readonly ["commands", "events", "trace", "o11y", "agentSetup", "diff", "sources"];
|
|
7
7
|
export type ArtifactKind = (typeof ARTIFACT_KINDS)[number];
|
|
8
8
|
/** 写这份结果的工具:niceeval 自己,或经 niceeval/results 写入面转换的第三方 harness。 */
|
|
@@ -14,7 +14,7 @@ export interface Producer {
|
|
|
14
14
|
/**
|
|
15
15
|
* `snapshot.json` 的持久化契约:快照元数据 —— 身份、快照级字段与版本元数据,
|
|
16
16
|
* 不含任何逐 attempt 数据。快照开始时写入;收尾时补写 `completedAt`。
|
|
17
|
-
* 字段规则与版本判定见 docs/feature/
|
|
17
|
+
* 字段规则与版本判定见 docs/feature/record/architecture.md「snapshot.json」「版本与升级设计」。
|
|
18
18
|
*/
|
|
19
19
|
export interface SnapshotMeta {
|
|
20
20
|
/** 恒为 "niceeval.results";和 schemaVersion、producer 一起构成持久化契约,永不移动或改名。 */
|
|
@@ -39,7 +39,7 @@ export interface SnapshotMeta {
|
|
|
39
39
|
diagnostics?: DiagnosticRecord[];
|
|
40
40
|
/**
|
|
41
41
|
* experiment 作用域生命周期代码经 `ctx.fact()` 上报的运行事实;与 completedAt 同批在快照
|
|
42
|
-
* 封口补写。字段契约见 result.json 的 facts 小节(docs/feature/
|
|
42
|
+
* 封口补写。字段契约见 result.json 的 facts 小节(docs/feature/record/architecture.md#facts运行事实)。
|
|
43
43
|
*/
|
|
44
44
|
facts?: Record<string, string | number | boolean>;
|
|
45
45
|
/** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
|
|
@@ -85,7 +85,7 @@ export interface AttemptHandle {
|
|
|
85
85
|
/**
|
|
86
86
|
* 携带条目投影:true = fingerprint 未变、上一轮终态结果合入本快照(`result.artifactBase`
|
|
87
87
|
* 有值);false = 本快照那次运行真实执行。`startedAt` 为原执行时刻,不因携带而改写
|
|
88
|
-
* (时效语义见 docs/feature/
|
|
88
|
+
* (时效语义见 docs/feature/sample/library.md「时效:新执行与历史执行」)。
|
|
89
89
|
*/
|
|
90
90
|
carried: boolean;
|
|
91
91
|
/** 非零 Sandbox 命令的 stdout/stderr 证据(`commands.json`);没有失败命令时 null。 */
|
|
@@ -178,7 +178,7 @@ export interface Results {
|
|
|
178
178
|
* 每个实验取最新一次快照,返回 Scope(快照与挑选警告绑在一起走)。
|
|
179
179
|
* `experiments` 是 experiment id 前缀过滤(string | string[]),同 CLI 位置参数语义。
|
|
180
180
|
* `fresh: true` 只保留新执行的 attempt(排除携带条目),被排除的题按覆盖事实进入
|
|
181
|
-
* `coverage.missingEvalIds`(见 docs/feature/
|
|
181
|
+
* `coverage.missingEvalIds`(见 docs/feature/sample/library.md「时效:新执行与历史执行」)。
|
|
182
182
|
*/
|
|
183
183
|
latest(opts?: {
|
|
184
184
|
experiments?: string | string[];
|
|
@@ -189,7 +189,7 @@ export interface Results {
|
|
|
189
189
|
* 跨历史拼出当前判定水位。可比性前提:每个 experiment 以最新快照的可比性配置
|
|
190
190
|
* (agent / model / reasoningEffort / flags / budget / timeoutMs / sandbox)为基准,
|
|
191
191
|
* 配置不一致的旧快照不贡献 attempt,缺口进 `coverage.missingEvalIds`
|
|
192
|
-
* (见 docs/feature/
|
|
192
|
+
* (见 docs/feature/record/library.md「官方现刻水位」)。`fresh: true` 同 `latest()`。
|
|
193
193
|
*/
|
|
194
194
|
current(opts?: {
|
|
195
195
|
experiments?: string | string[];
|
|
@@ -199,7 +199,7 @@ export interface Results {
|
|
|
199
199
|
/**
|
|
200
200
|
* 一个实验的覆盖事实:已知 eval 并集(分母)与当前口径下没有任何 attempt 的题。
|
|
201
201
|
* `missingEvalIds` 永远被算出来,不静默——渲染面把它转成榜单占位行
|
|
202
|
-
* (见 docs/feature/
|
|
202
|
+
* (见 docs/feature/sample/library.md「选择快照」「时效:新执行与历史执行」)。
|
|
203
203
|
*/
|
|
204
204
|
export interface ScopeCoverage {
|
|
205
205
|
experimentId: string;
|
|
@@ -235,7 +235,7 @@ export interface Scope {
|
|
|
235
235
|
/**
|
|
236
236
|
* 挑选警告:每种带 kind、可判断的结构化字段和渲染好的英文 message;能用一条命令直接推进的
|
|
237
237
|
* kind 同时带 `command`(已替换真实 id,复制即跑)。kind 是契约的一部分,全集与触发条件见
|
|
238
|
-
* docs/feature/
|
|
238
|
+
* docs/feature/record/library.md「警告 kind 全集」——三种都是**定位不到任何一行**的完整性
|
|
239
239
|
* 事实:覆盖缺口(行级事实,见 `ScopeCoverage`)与时效(`AttemptHandle.carried` 投影的行级
|
|
240
240
|
* 属性)不在这个联合里。
|
|
241
241
|
*/
|