niceeval 0.11.4-canary.20 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/dist/i18n/en.d.ts +9 -1
- package/dist/i18n/en.js +11 -2
- package/dist/i18n/zh-CN.d.ts +10 -2
- package/dist/i18n/zh-CN.js +11 -2
- package/dist/record/attempt-evidence.d.ts +1 -1
- package/dist/record/open.js +25 -0
- package/dist/report/assets/series-encoding.d.ts +36 -0
- package/dist/report/assets/series-encoding.js +104 -0
- package/dist/report/built-in/result-components.js +12 -2
- package/dist/report/components/attempt-detail/compute.d.ts +8 -1
- package/dist/report/components/attempt-detail/compute.js +19 -1
- package/dist/report/components/attempt-detail/index.js +25 -3
- package/dist/report/components/attempt-detail/validate.js +3 -2
- package/dist/report/components/entity-lists/compute.d.ts +6 -0
- package/dist/report/components/entity-lists/compute.js +55 -8
- package/dist/report/components/entity-lists/content.d.ts +2 -0
- package/dist/report/components/entity-lists/content.js +59 -6
- package/dist/report/components/entity-lists/index.d.ts +21 -1
- package/dist/report/components/entity-lists/index.js +40 -7
- package/dist/report/components/entity-lists/validate.js +21 -0
- package/dist/report/components/fixtures.js +2 -0
- package/dist/report/definition/cell.d.ts +22 -0
- package/dist/report/definition/cell.js +23 -9
- package/dist/report/definition/primitives/chart.js +125 -25
- package/dist/report/definition/primitives/conversation.d.ts +1 -1
- package/dist/report/definition/primitives/conversation.js +5 -3
- package/dist/report/definition/primitives.js +32 -7
- package/dist/report/definition/tree.d.ts +5 -2
- package/dist/report/definition/tree.js +1 -0
- package/dist/report/index.d.ts +5 -5
- package/dist/report/index.js +2 -2
- package/dist/report/model/aggregate.d.ts +14 -1
- package/dist/report/model/aggregate.js +34 -2
- package/dist/report/model/conversions.d.ts +1 -0
- package/dist/report/model/conversions.js +4 -1
- package/dist/report/model/format.d.ts +6 -5
- package/dist/report/model/format.js +30 -9
- package/dist/report/model/locale.d.ts +19 -3
- package/dist/report/model/locale.js +31 -5
- package/dist/report/model/types.d.ts +41 -4
- package/dist/report/presentation.d.ts +53 -6
- package/dist/report/presentation.js +40 -5
- package/dist/report/react/index.d.ts +3 -3
- package/dist/report/react/index.js +1 -1
- package/dist/report/slices/compute.js +52 -4
- package/dist/report/slices/content.d.ts +2 -1
- package/dist/report/slices/content.js +48 -15
- package/dist/report/slices/validate.js +23 -0
- package/dist/report/tasks.d.ts +7 -1
- package/dist/report/tasks.js +5 -3
- package/dist/runner/types.d.ts +30 -10
- package/dist/sample/index.d.ts +0 -9
- package/dist/sample/index.js +0 -18
- package/dist/sandbox/backend.d.ts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.d.ts +2 -1
- package/dist/sandbox/docker.d.ts +28 -7
- package/dist/sandbox/e2b.d.ts +11 -0
- package/dist/sandbox/identity.d.ts +6 -1
- package/dist/sandbox/layer.d.ts +36 -3
- package/dist/sandbox/local.d.ts +9 -0
- package/dist/sandbox/types.d.ts +14 -8
- package/dist/sandbox/vercel.d.ts +9 -0
- package/docs-site/zh/reference/builtin-agents.mdx +2 -0
- package/docs-site/zh/reference/cli.mdx +1 -0
- package/docs-site/zh/reference/define-agent.mdx +13 -9
- package/docs-site/zh/reference/define-config.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +1 -1
- package/docs-site/zh/reference/results-data.mdx +1 -1
- package/docs-site/zh/tutorials/deploy-report-site.mdx +157 -0
- package/docs-site/zh/tutorials/publish-report.mdx +1 -1
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-providers.mdx +19 -4
- package/package.json +1 -1
- package/src/agents/codex-env.test.ts +8 -1
- package/src/agents/codex.ts +8 -0
- package/src/agents/coding-cli-versions.ts +22 -6
- package/src/cli.ts +23 -1
- package/src/i18n/en.ts +14 -3
- package/src/i18n/zh-CN.ts +13 -2
- package/src/record/attempt-evidence.ts +1 -1
- package/src/record/open.ts +21 -0
- package/src/record/results.test.ts +74 -13
- package/src/record/truncate.ts +32 -7
- package/src/record/writer.ts +39 -8
- package/src/report/assets/series-encoding.tsx +177 -0
- package/src/report/assets/styles.css +97 -5
- package/src/report/built-in/result-components.tsx +23 -2
- package/src/report/components/attempt-detail/attempt-components.test.tsx +41 -0
- package/src/report/components/attempt-detail/compute.ts +20 -2
- package/src/report/components/attempt-detail/index.tsx +32 -2
- package/src/report/components/attempt-detail/validate.test.ts +1 -0
- package/src/report/components/attempt-detail/validate.tsx +3 -1
- package/src/report/components/compute.test.ts +196 -0
- package/src/report/components/entity-lists/compute.ts +73 -7
- package/src/report/components/entity-lists/content.test.ts +14 -8
- package/src/report/components/entity-lists/content.ts +63 -6
- package/src/report/components/entity-lists/index.tsx +93 -8
- package/src/report/components/entity-lists/validate.test.ts +9 -0
- package/src/report/components/entity-lists/validate.ts +16 -0
- package/src/report/components/fixtures.ts +2 -0
- package/src/report/definition/cell.ts +46 -11
- package/src/report/definition/conversation.test.tsx +5 -1
- package/src/report/definition/primitives/chart.tsx +197 -32
- package/src/report/definition/primitives/conversation.tsx +7 -4
- package/src/report/definition/primitives.tsx +81 -9
- package/src/report/definition/tree.ts +6 -2
- package/src/report/index.ts +15 -2
- package/src/report/model/aggregate.ts +40 -2
- package/src/report/model/conversions.ts +5 -0
- package/src/report/model/format.ts +32 -9
- package/src/report/model/locale.ts +35 -5
- package/src/report/model/presentation.test.ts +23 -1
- package/src/report/model/types.ts +40 -4
- package/src/report/presentation.test.tsx +179 -19
- package/src/report/presentation.ts +114 -11
- package/src/report/react/index.tsx +13 -2
- package/src/report/runtime/dual-render.test.tsx +38 -2
- package/src/report/slices/compute.ts +59 -4
- package/src/report/slices/content.ts +55 -16
- package/src/report/slices/delta-table.test.ts +138 -3
- package/src/report/slices/validate.test.ts +29 -0
- package/src/report/slices/validate.ts +16 -0
- package/src/report/tasks.ts +12 -2
- package/src/runner/accept.test.ts +328 -10
- package/src/runner/accept.ts +239 -72
- package/src/runner/attempt.test.ts +25 -13
- package/src/runner/attempt.ts +9 -9
- package/src/runner/config-identity.test.ts +25 -0
- package/src/runner/config-identity.ts +6 -7
- package/src/runner/feedback/coordinator.test.ts +51 -0
- package/src/runner/feedback/coordinator.ts +23 -0
- package/src/runner/feedback/failure.test.ts +15 -0
- package/src/runner/feedback/failure.ts +7 -0
- package/src/runner/feedback/human.test.ts +548 -27
- package/src/runner/feedback/human.ts +321 -47
- package/src/runner/feedback/index.ts +10 -0
- package/src/runner/feedback/input-guard.test.ts +251 -0
- package/src/runner/feedback/input-guard.ts +171 -0
- package/src/runner/feedback/reducer.ts +1 -0
- package/src/runner/fingerprint.test.ts +24 -0
- package/src/runner/gate-lease.test.ts +38 -14
- package/src/runner/ledger.test.ts +10 -7
- package/src/runner/ledger.ts +9 -2
- package/src/runner/lock.test.ts +40 -16
- package/src/runner/manifest.ts +8 -9
- package/src/runner/reporters/artifacts.ts +25 -3
- package/src/runner/run.test.ts +61 -0
- package/src/runner/run.ts +34 -9
- package/src/runner/sandbox-pool.test.ts +216 -12
- package/src/runner/sandbox-pool.ts +20 -2
- package/src/runner/types.ts +30 -10
- package/src/sample/index.ts +0 -16
- package/src/sample/sample.test.ts +78 -0
- package/src/sandbox/backend.ts +1 -1
- package/src/sandbox/commands.ts +4 -7
- package/src/sandbox/compose.test.ts +26 -1
- package/src/sandbox/compose.ts +7 -3
- package/src/sandbox/docker.test.ts +66 -1
- package/src/sandbox/docker.ts +89 -65
- package/src/sandbox/dockerfile-build.ts +4 -1
- package/src/sandbox/e2b-agent-template.test.ts +18 -1
- package/src/sandbox/e2b-agent-template.ts +22 -6
- package/src/sandbox/e2b.test.ts +64 -0
- package/src/sandbox/e2b.ts +27 -5
- package/src/sandbox/identity.ts +7 -2
- package/src/sandbox/layer.test.ts +23 -4
- package/src/sandbox/layer.ts +127 -43
- package/src/sandbox/link.test.ts +5 -2
- package/src/sandbox/local.test.ts +8 -8
- package/src/sandbox/local.ts +28 -6
- package/src/sandbox/operations.test.ts +4 -4
- package/src/sandbox/operations.ts +5 -5
- package/src/sandbox/runtime.ts +18 -3
- package/src/sandbox/types.ts +14 -8
- package/src/sandbox/vercel.test.ts +58 -3
- package/src/sandbox/vercel.ts +38 -6
- package/src/show/index.ts +11 -30
- package/src/show/render.test.ts +23 -1
- package/src/show/render.ts +6 -3
|
@@ -61,6 +61,38 @@
|
|
|
61
61
|
.niceeval-report .niceeval-locator.niceeval-verdict-skipped { color: var(--soft); }
|
|
62
62
|
.niceeval-report .niceeval-locator-mark { font-weight: 650; }
|
|
63
63
|
|
|
64
|
+
/* ---- 时效:降饱和样式,与判定色同一条路径(experiment-table.md「时效不写字」) ----
|
|
65
|
+
历史执行(locator 格 staleSinceMs 在场)与覆盖缺口的过期结论参考共用同一条降饱和纪律:
|
|
66
|
+
判定色与 currentColor 混入中性 --soft,不额外发明第二套色板。 */
|
|
67
|
+
.niceeval-report .niceeval-locator.niceeval-stale,
|
|
68
|
+
.niceeval-report .niceeval-locator.niceeval-stale .niceeval-locator-mark {
|
|
69
|
+
color: color-mix(in oklch, currentColor, var(--soft) 55%);
|
|
70
|
+
}
|
|
71
|
+
.niceeval-report .niceeval-stale-distance {
|
|
72
|
+
margin-left: 2px;
|
|
73
|
+
font-variant-numeric: tabular-nums;
|
|
74
|
+
color: var(--soft);
|
|
75
|
+
}
|
|
76
|
+
.niceeval-report .niceeval-missing-reason { color: var(--muted); }
|
|
77
|
+
|
|
78
|
+
/* ---- 构成格:一条分段条,按计数占比分宽,hover 出段名与计数(table.md「构成格」) ----
|
|
79
|
+
段的业务含义住在投影层,这里只按声明序给一条位置驱动的渐隐色阶,不认识任何段名。 */
|
|
80
|
+
.niceeval-report .niceeval-composition {
|
|
81
|
+
display: flex;
|
|
82
|
+
align-items: stretch;
|
|
83
|
+
height: 8px;
|
|
84
|
+
min-width: 72px;
|
|
85
|
+
border-radius: 4px;
|
|
86
|
+
overflow: hidden;
|
|
87
|
+
background: var(--line);
|
|
88
|
+
}
|
|
89
|
+
.niceeval-report .niceeval-composition-segment {
|
|
90
|
+
display: block;
|
|
91
|
+
height: 100%;
|
|
92
|
+
min-width: 2px;
|
|
93
|
+
background: color-mix(in oklch, var(--accent), var(--soft) calc(var(--composition-step, 0) * 22%));
|
|
94
|
+
}
|
|
95
|
+
|
|
64
96
|
/* ---- 维度键上色:同一个 key 在任何块里同色(文字) ---- */
|
|
65
97
|
.niceeval-report .niceeval-c0 { color: var(--c0); }
|
|
66
98
|
.niceeval-report .niceeval-c1 { color: var(--c1); }
|
|
@@ -78,6 +110,41 @@
|
|
|
78
110
|
.niceeval-report .niceeval-series-c5 { --series: var(--c5); }
|
|
79
111
|
.niceeval-report .niceeval-series-none { --series: var(--muted); }
|
|
80
112
|
|
|
113
|
+
/*
|
|
114
|
+
* HTML 横向柱的 variant 填充图案(CSS 等效 SVG <pattern>)。
|
|
115
|
+
* variant 1 = 实心(默认 .niceeval-chart-bar-fill);2 对角斜线 / 3 水平条纹 / 4 点阵。
|
|
116
|
+
* 颜色始终读 --series,换主题令牌自动换色,不写死 hex。
|
|
117
|
+
*/
|
|
118
|
+
.niceeval-report .niceeval-series-fill-v2 {
|
|
119
|
+
background-color: transparent;
|
|
120
|
+
background-image: repeating-linear-gradient(
|
|
121
|
+
-45deg,
|
|
122
|
+
color-mix(in srgb, var(--series) 78%, var(--panel)),
|
|
123
|
+
color-mix(in srgb, var(--series) 78%, var(--panel)) 2px,
|
|
124
|
+
color-mix(in srgb, var(--series) 28%, var(--panel)) 2px,
|
|
125
|
+
color-mix(in srgb, var(--series) 28%, var(--panel)) 5px
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
.niceeval-report .niceeval-series-fill-v3 {
|
|
129
|
+
background-color: transparent;
|
|
130
|
+
background-image: repeating-linear-gradient(
|
|
131
|
+
0deg,
|
|
132
|
+
color-mix(in srgb, var(--series) 78%, var(--panel)),
|
|
133
|
+
color-mix(in srgb, var(--series) 78%, var(--panel)) 2px,
|
|
134
|
+
color-mix(in srgb, var(--series) 28%, var(--panel)) 2px,
|
|
135
|
+
color-mix(in srgb, var(--series) 28%, var(--panel)) 5px
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
.niceeval-report .niceeval-series-fill-v4 {
|
|
139
|
+
background-color: color-mix(in srgb, var(--series) 28%, var(--panel));
|
|
140
|
+
background-image: radial-gradient(
|
|
141
|
+
circle at 25% 25%,
|
|
142
|
+
color-mix(in srgb, var(--series) 88%, var(--panel)) 1.2px,
|
|
143
|
+
transparent 1.3px
|
|
144
|
+
);
|
|
145
|
+
background-size: 6px 6px;
|
|
146
|
+
}
|
|
147
|
+
|
|
81
148
|
/* ---- 表格通用:圆角 panel、细分隔线、tabular-nums 数字、uppercase 小标签 ---- */
|
|
82
149
|
table.niceeval-report,
|
|
83
150
|
.niceeval-report table {
|
|
@@ -160,6 +227,23 @@ table.niceeval-report,
|
|
|
160
227
|
}
|
|
161
228
|
.niceeval-table-wrap > .niceeval-filter:focus { border-color: var(--line-strong); }
|
|
162
229
|
.niceeval-table-wrap > .niceeval-filter::placeholder { color: var(--soft); }
|
|
230
|
+
|
|
231
|
+
/* ---- 「只看新执行」开关(experiment-table.md「只看新执行」) ----
|
|
232
|
+
增强层:初始 HTML 是关闭态的完整表(niceeval-fresh-all 可见,niceeval-fresh-only 隐藏),
|
|
233
|
+
开关只切换这两块的可见性,不重新请求或重新计算任何数据。 */
|
|
234
|
+
.niceeval-report .niceeval-fresh-toggle {
|
|
235
|
+
display: inline-flex;
|
|
236
|
+
align-items: center;
|
|
237
|
+
gap: 6px;
|
|
238
|
+
margin: 0 0 12px;
|
|
239
|
+
color: var(--muted);
|
|
240
|
+
font-size: 12.5px;
|
|
241
|
+
cursor: pointer;
|
|
242
|
+
}
|
|
243
|
+
.niceeval-report .niceeval-fresh-toggle input[type="checkbox"] { margin: 0; }
|
|
244
|
+
.niceeval-report .niceeval-fresh-only { display: none; }
|
|
245
|
+
.niceeval-report.niceeval-experiment-table:has(.niceeval-fresh-toggle input:checked) .niceeval-fresh-all { display: none; }
|
|
246
|
+
.niceeval-report.niceeval-experiment-table:has(.niceeval-fresh-toggle input:checked) .niceeval-fresh-only { display: block; }
|
|
163
247
|
.niceeval-report .niceeval-cell-text { display: block; min-width: 0; }
|
|
164
248
|
.niceeval-report .niceeval-cell-detail {
|
|
165
249
|
display: block;
|
|
@@ -963,6 +1047,7 @@ table.niceeval-report,
|
|
|
963
1047
|
}
|
|
964
1048
|
.niceeval-command-evidence--failed { border-left-color: var(--bad); }
|
|
965
1049
|
.niceeval-command-evidence--observed { border-left-color: var(--muted); }
|
|
1050
|
+
.niceeval-command-evidence--succeeded { border-left-color: var(--good); }
|
|
966
1051
|
.niceeval-command-evidence-card .niceeval-conversation-turn-head {
|
|
967
1052
|
display: flex;
|
|
968
1053
|
flex-wrap: wrap;
|
|
@@ -1133,12 +1218,19 @@ table.niceeval-report,
|
|
|
1133
1218
|
align-items: center;
|
|
1134
1219
|
gap: 5px;
|
|
1135
1220
|
}
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1221
|
+
/* 图例色块由 renderer 显式输出(含 variant 图案 / 线型 / marker),不再用 ::before 单色方块。 */
|
|
1222
|
+
.niceeval-chart .niceeval-chart-legend-swatch {
|
|
1223
|
+
display: inline-block;
|
|
1224
|
+
width: 10px;
|
|
1225
|
+
height: 10px;
|
|
1226
|
+
flex: 0 0 auto;
|
|
1140
1227
|
border-radius: var(--radius);
|
|
1141
|
-
background: var(--series);
|
|
1228
|
+
background: color-mix(in srgb, var(--series) 74%, var(--panel));
|
|
1229
|
+
}
|
|
1230
|
+
.niceeval-chart .niceeval-chart-legend-swatch-svg {
|
|
1231
|
+
display: block;
|
|
1232
|
+
flex: 0 0 auto;
|
|
1233
|
+
overflow: visible;
|
|
1142
1234
|
}
|
|
1143
1235
|
.niceeval-chart .niceeval-chart-missing {
|
|
1144
1236
|
max-width: 880px;
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
import { HeroCard } from "../components/site-components/index.tsx";
|
|
23
23
|
import { AttemptSummary } from "../components/attempt-detail/index.tsx";
|
|
24
24
|
import { experimentListContent } from "../components/entity-lists/content.ts";
|
|
25
|
+
import { ExperimentTableView } from "../components/entity-lists/index.tsx";
|
|
25
26
|
import {
|
|
26
27
|
attemptAssertionsContent,
|
|
27
28
|
attemptConversationContent,
|
|
@@ -39,7 +40,7 @@ import {
|
|
|
39
40
|
formatReportDateTimeRange,
|
|
40
41
|
} from "../model/format.ts";
|
|
41
42
|
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../model/locale.ts";
|
|
42
|
-
import type { Dataset, UsageTableData } from "../model/types.ts";
|
|
43
|
+
import type { AttemptFactsData, Dataset, UsageTableData } from "../model/types.ts";
|
|
43
44
|
import type { MetricValue } from "../model/calculation.ts";
|
|
44
45
|
import type {
|
|
45
46
|
AttemptDetailsResult,
|
|
@@ -135,6 +136,7 @@ export const StandardOverviewResultView = defineComponent<{
|
|
|
135
136
|
result: StandardOverviewResult;
|
|
136
137
|
}>(async ({ result }, ctx) => {
|
|
137
138
|
const table = experimentListContent(result.experiments);
|
|
139
|
+
const freshTable = result.freshExperiments ? experimentListContent(result.freshExperiments) : null;
|
|
138
140
|
const hasPassRate = table.columns.some((column) => column.key === "passRate");
|
|
139
141
|
const hasTotalScore = table.columns.some((column) => column.key === "totalScore");
|
|
140
142
|
const defaultSort = hasPassRate === hasTotalScore
|
|
@@ -178,7 +180,11 @@ export const StandardOverviewResultView = defineComponent<{
|
|
|
178
180
|
),
|
|
179
181
|
)}
|
|
180
182
|
</Col>
|
|
181
|
-
|
|
183
|
+
{freshTable === null ? (
|
|
184
|
+
<TableContentView data={table} sort={defaultSort} searchable />
|
|
185
|
+
) : (
|
|
186
|
+
<ExperimentTableView fullContent={table} freshContent={freshTable} sort={defaultSort} searchable />
|
|
187
|
+
)}
|
|
182
188
|
</Col>
|
|
183
189
|
);
|
|
184
190
|
});
|
|
@@ -340,6 +346,20 @@ const TaskUsageKpi = defineComponent<{ label: string; value: string }>({
|
|
|
340
346
|
});
|
|
341
347
|
TaskUsageKpi.displayName = "TaskUsageKpi";
|
|
342
348
|
|
|
349
|
+
/** attempt 级 `ctx.fact()` 运行事实的完整键值表(docs/feature/record/architecture.md#facts运行事实);
|
|
350
|
+
* 没有 facts 时零输出,不摆空表。 */
|
|
351
|
+
const TaskFactsResultView = defineComponent<{ data: AttemptFactsData | null }>(({ data }) => {
|
|
352
|
+
if (data === null || data.facts.length === 0) return null;
|
|
353
|
+
return (
|
|
354
|
+
<Grid className="niceeval-facts-table">
|
|
355
|
+
{data.facts.map(({ key, value }) => (
|
|
356
|
+
<TaskUsageKpi key={key} label={key} value={String(value)} />
|
|
357
|
+
))}
|
|
358
|
+
</Grid>
|
|
359
|
+
);
|
|
360
|
+
});
|
|
361
|
+
TaskFactsResultView.displayName = "TaskFactsResultView";
|
|
362
|
+
|
|
343
363
|
export const AttemptDetailsResultView = defineComponent<{
|
|
344
364
|
result: AttemptDetailsResult;
|
|
345
365
|
}>(({ result }) => {
|
|
@@ -369,6 +389,7 @@ export const AttemptDetailsResultView = defineComponent<{
|
|
|
369
389
|
<CopyBlock content={attemptFixPromptContent(result.fixPrompt)} />
|
|
370
390
|
<Waterfall nodes={timeline ?? []} title={{ en: "Execution timeline", "zh-CN": "执行时间轴" }} />
|
|
371
391
|
<TaskUsageResultView data={result.usage} />
|
|
392
|
+
<TaskFactsResultView data={result.facts} />
|
|
372
393
|
{conversation === null ? (
|
|
373
394
|
<Callouts items={executionEvidenceUnavailableCallouts} />
|
|
374
395
|
) : embedded.conversation !== null ? (
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
attemptDiagnosticsData,
|
|
37
37
|
attemptDiffData,
|
|
38
38
|
attemptErrorData,
|
|
39
|
+
attemptFactsData,
|
|
39
40
|
attemptFixPromptData,
|
|
40
41
|
attemptSummaryData,
|
|
41
42
|
attemptTimelineData,
|
|
@@ -485,6 +486,7 @@ describe("AttemptCommandEvidence:命令与 Conversation 分离、错误提示只
|
|
|
485
486
|
stderr: "",
|
|
486
487
|
};
|
|
487
488
|
const failed = { ...observed, timingNodeId: "failed-command", checked: true };
|
|
489
|
+
const succeeded = { ...observed, timingNodeId: "succeeded-command", exitCode: 0, checked: false };
|
|
488
490
|
|
|
489
491
|
it("无 events 时 Conversation 仍为空,独立命令证据保留并排序", () => {
|
|
490
492
|
const evidence = evidenceOf({ commands: [failed, observed] });
|
|
@@ -492,6 +494,11 @@ describe("AttemptCommandEvidence:命令与 Conversation 分离、错误提示只
|
|
|
492
494
|
expect(attemptCommandEvidenceData(evidence)?.commands.map((command) => command.classification)).toEqual(["failed", "observed"]);
|
|
493
495
|
});
|
|
494
496
|
|
|
497
|
+
it("exitCode 0 分类为 succeeded,即使 checked 为 true 也不推导成 failed", () => {
|
|
498
|
+
const evidence = evidenceOf({ commands: [succeeded, { ...succeeded, timingNodeId: "checked-zero", checked: true }] });
|
|
499
|
+
expect(attemptCommandEvidenceData(evidence)?.commands.map((command) => command.classification)).toEqual(["succeeded", "succeeded"]);
|
|
500
|
+
});
|
|
501
|
+
|
|
495
502
|
it("observed 非零不触发截断错误摘要提示,failed 才触发", () => {
|
|
496
503
|
const observedError = attemptErrorData(evidenceOf({
|
|
497
504
|
commands: [observed],
|
|
@@ -507,6 +514,23 @@ describe("AttemptCommandEvidence:命令与 Conversation 分离、错误提示只
|
|
|
507
514
|
});
|
|
508
515
|
});
|
|
509
516
|
|
|
517
|
+
describe("attemptFactsData:facts 完整键值表投影", () => {
|
|
518
|
+
it("按落盘 key 插入顺序投影为数组;没有 facts 或空对象时返回 null,不摆空表", () => {
|
|
519
|
+
expect(attemptFactsData(evidenceOf())).toBeNull();
|
|
520
|
+
expect(attemptFactsData(evidenceOf({ result: resultOf({ facts: {} }) }))).toBeNull();
|
|
521
|
+
const withFacts = evidenceOf({
|
|
522
|
+
result: resultOf({ facts: { "memory.notesLoaded": 73, "nowledge.endpoint": "https://tunnel.example", ready: true } }),
|
|
523
|
+
});
|
|
524
|
+
expect(attemptFactsData(withFacts)).toEqual({
|
|
525
|
+
facts: [
|
|
526
|
+
{ key: "memory.notesLoaded", value: 73 },
|
|
527
|
+
{ key: "nowledge.endpoint", value: "https://tunnel.example" },
|
|
528
|
+
{ key: "ready", value: true },
|
|
529
|
+
],
|
|
530
|
+
});
|
|
531
|
+
});
|
|
532
|
+
});
|
|
533
|
+
|
|
510
534
|
describe("standardAttemptRender:执行证据投影", () => {
|
|
511
535
|
it("source 与 events 同时存在时仍保留 Conversation", async () => {
|
|
512
536
|
const events: StreamEvent[] = [
|
|
@@ -553,6 +577,23 @@ describe("standardAttemptRender:执行证据投影", () => {
|
|
|
553
577
|
const tree = await resolveOnAttemptPage(<AttemptDetails />, evidence);
|
|
554
578
|
expectEmbeddedConversationHtml(renderAttemptHtml(tree));
|
|
555
579
|
});
|
|
580
|
+
|
|
581
|
+
it("有 facts 时标准结果视图与公开 AttemptDetails 都渲染完整键值表;没有 facts 时零输出", async () => {
|
|
582
|
+
const withFacts = evidenceOf({ result: resultOf({ facts: { "memory.notesLoaded": 73 } }) });
|
|
583
|
+
const standardTree = await resolveOnAttemptPage(await standardAttemptRender(withFacts), withFacts);
|
|
584
|
+
const standardText = renderNodeToText(standardTree, createTextContext({ width: 120 }));
|
|
585
|
+
expect(standardText).toContain("memory.notesLoaded");
|
|
586
|
+
expect(standardText).toContain("73");
|
|
587
|
+
|
|
588
|
+
const publicTree = await resolveOnAttemptPage(<AttemptDetails />, withFacts);
|
|
589
|
+
const publicText = renderNodeToText(publicTree, createTextContext({ width: 120 }));
|
|
590
|
+
expect(publicText).toContain("memory.notesLoaded");
|
|
591
|
+
expect(publicText).toContain("73");
|
|
592
|
+
|
|
593
|
+
const withoutFacts = evidenceOf();
|
|
594
|
+
const emptyTree = await resolveOnAttemptPage(await standardAttemptRender(withoutFacts), withoutFacts);
|
|
595
|
+
expect(renderNodeToText(emptyTree, createTextContext({ width: 120 }))).not.toContain("notesLoaded");
|
|
596
|
+
});
|
|
556
597
|
});
|
|
557
598
|
// 「Attempt 证据数据源」——timeline / trace 投影的时间树语义
|
|
558
599
|
// (docs/engineering/testing/unit/reports.md)。
|
|
@@ -16,6 +16,7 @@ import type {
|
|
|
16
16
|
AttemptDiagnosticsData,
|
|
17
17
|
AttemptDiffData,
|
|
18
18
|
AttemptErrorData,
|
|
19
|
+
AttemptFactsData,
|
|
19
20
|
AttemptFixPromptData,
|
|
20
21
|
AttemptSummaryData,
|
|
21
22
|
AttemptTimelineData,
|
|
@@ -80,8 +81,9 @@ function looksLikeTruncatedCommandTail(message: string, commands: readonly Comma
|
|
|
80
81
|
}
|
|
81
82
|
|
|
82
83
|
/** 展示层唯一的命令分类规则;Record 只保存 checked 与 exitCode。 */
|
|
83
|
-
function commandClassification(command: Pick<CommandExitEvidence, "checked" | "exitCode">): "observed" | "failed" {
|
|
84
|
-
|
|
84
|
+
function commandClassification(command: Pick<CommandExitEvidence, "checked" | "exitCode">): "succeeded" | "observed" | "failed" {
|
|
85
|
+
if (command.exitCode === 0) return "succeeded";
|
|
86
|
+
return command.checked ? "failed" : "observed";
|
|
85
87
|
}
|
|
86
88
|
|
|
87
89
|
export function attemptErrorData(evidence: AttemptEvidence): AttemptErrorData | null {
|
|
@@ -432,6 +434,22 @@ export function usageTableData(evidence: AttemptEvidence): UsageTableData | null
|
|
|
432
434
|
};
|
|
433
435
|
}
|
|
434
436
|
|
|
437
|
+
// ───────────────────────── AttemptFacts ─────────────────────────
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* attempt 级 `ctx.fact()` 运行事实的完整键值表;`AttemptRecord.facts` 缺失或为空对象时返回
|
|
441
|
+
* null,不渲染空表(与其余叶子同一条「没有证据时零输出」规则)。
|
|
442
|
+
* `facts` 落盘就是 `Record<string, string | number | boolean>`,JS 对象的字符串键天然保留
|
|
443
|
+
* 写入顺序,这里按该顺序投影成数组,不重新排序。
|
|
444
|
+
*/
|
|
445
|
+
export function attemptFactsData(evidence: AttemptEvidence): AttemptFactsData | null {
|
|
446
|
+
const facts = evidence.result.facts;
|
|
447
|
+
if (!facts) return null;
|
|
448
|
+
const entries = Object.entries(facts);
|
|
449
|
+
if (entries.length === 0) return null;
|
|
450
|
+
return { facts: entries.map(([key, value]) => ({ key, value })) };
|
|
451
|
+
}
|
|
452
|
+
|
|
435
453
|
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
436
454
|
|
|
437
455
|
export function attemptTraceData(evidence: AttemptEvidence): AttemptTraceData | null {
|
|
@@ -15,13 +15,14 @@ import {
|
|
|
15
15
|
Waterfall,
|
|
16
16
|
} from "../../definition/primitives.tsx";
|
|
17
17
|
import type { CommandEvidenceContent } from "../../definition/primitives/conversation.tsx";
|
|
18
|
-
import type { AttemptSummaryData, UsageTableData } from "../../model/types.ts";
|
|
18
|
+
import type { AttemptFactsData, AttemptSummaryData, UsageTableData } from "../../model/types.ts";
|
|
19
19
|
import { formatDurationMs, formatInstant, formatPoints, formatUSD } from "../../model/format.ts";
|
|
20
20
|
import { localeText } from "../../model/locale.ts";
|
|
21
21
|
import { cx, type ValueProps } from "../shared.ts";
|
|
22
22
|
import type { PageContext } from "../../definition/tree.ts";
|
|
23
23
|
import {
|
|
24
24
|
toAttemptAssertions,
|
|
25
|
+
toAttemptFacts,
|
|
25
26
|
toAttemptFixPrompt,
|
|
26
27
|
toAttemptNotices,
|
|
27
28
|
toAttemptSource,
|
|
@@ -154,6 +155,33 @@ const AttemptUsage = defineComponent<UsageProps>({
|
|
|
154
155
|
});
|
|
155
156
|
AttemptUsage.displayName = "AttemptUsage";
|
|
156
157
|
|
|
158
|
+
type FactsProps = ValueProps<AttemptFactsData | null, { className?: string }>;
|
|
159
|
+
|
|
160
|
+
/** attempt 级 `ctx.fact()` 运行事实的完整键值表(docs/feature/record/architecture.md#facts运行事实);
|
|
161
|
+
* 没有 facts 时零输出,不摆空表。 */
|
|
162
|
+
const AttemptFacts = defineComponent<FactsProps>({
|
|
163
|
+
dimensions: () => ({}),
|
|
164
|
+
web(props) {
|
|
165
|
+
const d = props.data;
|
|
166
|
+
if (d === null || d === undefined || d.facts.length === 0) return null;
|
|
167
|
+
return (
|
|
168
|
+
<Grid className={cx("niceeval-facts-table", props.className)}>
|
|
169
|
+
{d.facts.map(({ key, value }) => (
|
|
170
|
+
<Kpi key={key} label={key} value={String(value)} />
|
|
171
|
+
))}
|
|
172
|
+
</Grid>
|
|
173
|
+
);
|
|
174
|
+
},
|
|
175
|
+
text(props) {
|
|
176
|
+
const d = props.data;
|
|
177
|
+
if (d === null || d === undefined || d.facts.length === 0) return "";
|
|
178
|
+
const lines = ["facts:"];
|
|
179
|
+
for (const { key, value } of d.facts) lines.push(` ${key} ${value}`);
|
|
180
|
+
return lines.join("\n");
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
AttemptFacts.displayName = "AttemptFacts";
|
|
184
|
+
|
|
157
185
|
function evidenceOf(props: { attempt?: AttemptEvidence }, ctx: { page: PageContext }): AttemptEvidence {
|
|
158
186
|
if (props.attempt !== undefined) return props.attempt;
|
|
159
187
|
const input = ctx.page.input;
|
|
@@ -208,7 +236,7 @@ AttemptAssessment.displayName = "AttemptAssessment";
|
|
|
208
236
|
export const AttemptDetails = defineComponent<AttemptDetailsProps>(async (props, ctx) => {
|
|
209
237
|
const evidence = evidenceOf(props, ctx);
|
|
210
238
|
const hasSource = evidence.capabilities.source;
|
|
211
|
-
const [notices, source, assertions, summary, fixPrompt, timeline, usage, commandEvidence, conversation, diff] = await Promise.all([
|
|
239
|
+
const [notices, source, assertions, summary, fixPrompt, timeline, usage, facts, commandEvidence, conversation, diff] = await Promise.all([
|
|
212
240
|
toAttemptNotices(evidence),
|
|
213
241
|
hasSource ? toAttemptSource(evidence) : Promise.resolve(null),
|
|
214
242
|
hasSource ? Promise.resolve(null) : toAttemptAssertions(evidence),
|
|
@@ -216,6 +244,7 @@ export const AttemptDetails = defineComponent<AttemptDetailsProps>(async (props,
|
|
|
216
244
|
toAttemptFixPrompt(evidence),
|
|
217
245
|
toTimelineNodes(evidence),
|
|
218
246
|
toAttemptUsage(evidence),
|
|
247
|
+
toAttemptFacts(evidence),
|
|
219
248
|
toCommandEvidence(evidence),
|
|
220
249
|
toConversationTurns(evidence),
|
|
221
250
|
toDiffFiles(evidence),
|
|
@@ -240,6 +269,7 @@ export const AttemptDetails = defineComponent<AttemptDetailsProps>(async (props,
|
|
|
240
269
|
title={{ en: "Execution timeline", "zh-CN": "执行时间轴" }}
|
|
241
270
|
/>
|
|
242
271
|
<AttemptUsage data={usage} />
|
|
272
|
+
<AttemptFacts data={facts} />
|
|
243
273
|
{conversation === null ? (
|
|
244
274
|
<Callouts items={executionEvidenceUnavailableCallouts} />
|
|
245
275
|
) : embedded.conversation !== null ? (
|
|
@@ -174,6 +174,7 @@ describe("validateConversationData — AttemptConversationReply 判别联合", (
|
|
|
174
174
|
],
|
|
175
175
|
};
|
|
176
176
|
expect(validateCommandEvidenceData(withCommands)).toBeNull();
|
|
177
|
+
expect(validateCommandEvidenceData({ ...withCommands, commands: [{ ...withCommands.commands[0], exitCode: 0, classification: "succeeded" }] })).toBeNull();
|
|
177
178
|
expect(validateCommandEvidenceData({ ...withCommands, commands: [{ ...withCommands.commands[0], classification: "unknown" }] })).toMatch(/"commands\[0\]\.classification"/);
|
|
178
179
|
expect(validateCommandEvidenceData({ ...withCommands, commands: [{ ...withCommands.commands[0], exitCode: "1" }] })).toMatch(/"commands\[0\]\.exitCode"/);
|
|
179
180
|
});
|
|
@@ -242,7 +242,9 @@ function commandEvidenceProblem(value: unknown, path: string): string | null {
|
|
|
242
242
|
if (typeof value.phase !== "string") return `"${path}.phase" must be a string`;
|
|
243
243
|
if (typeof value.display !== "string") return `"${path}.display" must be a string`;
|
|
244
244
|
if (typeof value.exitCode !== "number") return `"${path}.exitCode" must be a number`;
|
|
245
|
-
if (value.classification !== "
|
|
245
|
+
if (value.classification !== "succeeded" && value.classification !== "observed" && value.classification !== "failed") {
|
|
246
|
+
return `"${path}.classification" must be succeeded | observed | failed`;
|
|
247
|
+
}
|
|
246
248
|
if (value.durationMs !== undefined && typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
|
|
247
249
|
if (typeof value.stdout !== "string") return `"${path}.stdout" must be a string`;
|
|
248
250
|
if (typeof value.stderr !== "string") return `"${path}.stderr" must be a string`;
|
|
@@ -16,6 +16,10 @@ import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict }
|
|
|
16
16
|
import { completeEvidenceCoverage } from "../../assertions/coverage.ts";
|
|
17
17
|
import type { AttemptHandle, Sample, SampleIssue, Run } from "../../record/index.ts";
|
|
18
18
|
import { attemptHandleOf, scopeOf } from "./scope.harness.ts";
|
|
19
|
+
import { makeSample } from "../../sample/index.ts";
|
|
20
|
+
import { encodeAttemptLocator } from "../../record/locator.ts";
|
|
21
|
+
import { COVERAGE_ROW_PREFIX, experimentListContent } from "./entity-lists/content.ts";
|
|
22
|
+
import { formatCellText, type Cell } from "../definition/cell.ts";
|
|
19
23
|
import type { Record } from "../../record/types.ts";
|
|
20
24
|
import {
|
|
21
25
|
assistantTurns,
|
|
@@ -712,6 +716,198 @@ describe("实体列表 data", () => {
|
|
|
712
716
|
});
|
|
713
717
|
});
|
|
714
718
|
|
|
719
|
+
// ─────── 覆盖缺口的两档占位与过期结论参考(experiment-table.md「覆盖缺口的两档占位行」) ───────
|
|
720
|
+
|
|
721
|
+
describe("覆盖缺口的两档占位与过期结论参考", () => {
|
|
722
|
+
/**
|
|
723
|
+
* 区分力场景:evalId "b" 只在两份旧 configHash 快照下跑过(与当前基准不可比,过期结论候选
|
|
724
|
+
* 多于一条),"c" 从未在任何快照出现过(未跑到)。`current` 是当前基准,只贡献 "a"。
|
|
725
|
+
*/
|
|
726
|
+
function staleScope(opts: { fresh?: boolean } = {}): { scope: Sample; older: Run; newerStale: Run } {
|
|
727
|
+
const current = snap({ experimentId: "exp/stale", results: [res("a", "passed")] });
|
|
728
|
+
current.configHash = "cfg-2";
|
|
729
|
+
const older = snap({
|
|
730
|
+
experimentId: "exp/stale",
|
|
731
|
+
results: [res("b", "failed")],
|
|
732
|
+
runStartedAt: "2026-01-01T00:00:00.000Z",
|
|
733
|
+
});
|
|
734
|
+
older.configHash = "cfg-1";
|
|
735
|
+
const newerStale = snap({
|
|
736
|
+
experimentId: "exp/stale",
|
|
737
|
+
results: [res("b", "passed")],
|
|
738
|
+
runStartedAt: "2026-02-01T00:00:00.000Z",
|
|
739
|
+
});
|
|
740
|
+
newerStale.configHash = "cfg-1"; // 同样不可比,但比 older 晚出现——参考要取这一条
|
|
741
|
+
const coverage = [
|
|
742
|
+
{ experimentId: "exp/stale", run: current, knownEvalIds: ["a", "b", "c"], missingEvalIds: ["b", "c"] },
|
|
743
|
+
];
|
|
744
|
+
const scope = makeSample(
|
|
745
|
+
"current",
|
|
746
|
+
[current],
|
|
747
|
+
[...current.attempts],
|
|
748
|
+
[],
|
|
749
|
+
coverage,
|
|
750
|
+
opts.fresh ?? false,
|
|
751
|
+
[...current.attempts, ...older.attempts, ...newerStale.attempts],
|
|
752
|
+
);
|
|
753
|
+
return { scope, older, newerStale };
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function recordCellOf(row: { cells: Readonly<globalThis.Record<string, Cell>> }): Cell {
|
|
757
|
+
return row.cells.record!;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
it("两档占位行的结果格都是 missing 且都带补跑命令;只有过期结论的那档带 reference,取候选中最新的一条", async () => {
|
|
761
|
+
const { scope, newerStale } = staleScope();
|
|
762
|
+
const items = await experimentListData(scope);
|
|
763
|
+
const content = experimentListContent(items);
|
|
764
|
+
const sub = content.rows[0]!.subRows!;
|
|
765
|
+
const staleRow = sub.find((row) => row.key.endsWith("b:missing"))!;
|
|
766
|
+
const notRunRow = sub.find((row) => row.key.endsWith("c:missing"))!;
|
|
767
|
+
expect(staleRow.variant).toBe("placeholder");
|
|
768
|
+
expect(notRunRow.variant).toBe("placeholder");
|
|
769
|
+
|
|
770
|
+
const staleCell = recordCellOf(staleRow);
|
|
771
|
+
const notRunCell = recordCellOf(notRunRow);
|
|
772
|
+
if (staleCell.kind !== "missing" || notRunCell.kind !== "missing") throw new Error("expected missing cells");
|
|
773
|
+
expect(staleCell.code).toBe("noCurrentResult");
|
|
774
|
+
expect(notRunCell.code).toBe("noCurrentResult");
|
|
775
|
+
expect(staleCell.detail).toBe("niceeval exp exp/stale");
|
|
776
|
+
expect(notRunCell.detail).toBe("niceeval exp exp/stale");
|
|
777
|
+
expect(notRunCell.reference).toBeUndefined();
|
|
778
|
+
expect(staleCell.reference).toBeDefined();
|
|
779
|
+
// 两档必须给出不同的格——把两档折成同一种占位的实现在这里失败。
|
|
780
|
+
expect(staleCell).not.toEqual(notRunCell);
|
|
781
|
+
// 候选多于一条(older 与 newerStale 都不可比)时取最新那条:newerStale 的判定是 "passed"。
|
|
782
|
+
expect(staleCell.reference!.verdict).toBe("passed");
|
|
783
|
+
expect(staleCell.reference!.locator).toBe(
|
|
784
|
+
encodeAttemptLocator({ runId: newerStale.runId, evalId: "b", attempt: 0 }),
|
|
785
|
+
);
|
|
786
|
+
|
|
787
|
+
// text 面:带参考时用参考替代原因文案,不把「当前配置下无结果」再说一遍。
|
|
788
|
+
expect(formatCellText(staleCell, "en")).toMatch(/^✓ /);
|
|
789
|
+
expect(formatCellText(notRunCell, "en")).toBe("no result for current config · niceeval exp exp/stale");
|
|
790
|
+
expect(formatCellText(notRunCell, "zh-CN")).toBe("当前配置下无结果 · niceeval exp exp/stale");
|
|
791
|
+
});
|
|
792
|
+
|
|
793
|
+
it("reference 不进任何计数:带参考前后,该 experiment 的判定计票、通过率与覆盖分母逐字不变", async () => {
|
|
794
|
+
const { scope } = staleScope();
|
|
795
|
+
const [item] = await experimentListData(scope);
|
|
796
|
+
// 分母只数 "a" 这一道有 attempt 的题,"b" 的参考不进 evals/attempts/passRate 分母。
|
|
797
|
+
expect(item!.evals).toBe(1);
|
|
798
|
+
expect(item!.attempts).toBe(1);
|
|
799
|
+
expect(item!.endToEndPassRate).toMatchObject({ value: 1, samples: 1, total: 1 });
|
|
800
|
+
expect(item!.evalVerdicts).toEqual({ passed: 1, failed: 0, errored: 0, skipped: 0 });
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
it("sample.fresh 为 true 时两档占位行都不带 reference", async () => {
|
|
804
|
+
const nonFresh = experimentListContent(await experimentListData(staleScope({ fresh: false }).scope));
|
|
805
|
+
const fresh = experimentListContent(await experimentListData(staleScope({ fresh: true }).scope));
|
|
806
|
+
const staleCellNonFresh = recordCellOf(nonFresh.rows[0]!.subRows!.find((row) => row.key.endsWith("b:missing"))!);
|
|
807
|
+
const staleCellFresh = recordCellOf(fresh.rows[0]!.subRows!.find((row) => row.key.endsWith("b:missing"))!);
|
|
808
|
+
if (staleCellNonFresh.kind !== "missing" || staleCellFresh.kind !== "missing") throw new Error("expected missing cells");
|
|
809
|
+
expect(staleCellNonFresh.reference).toBeDefined();
|
|
810
|
+
expect(staleCellFresh.reference).toBeUndefined();
|
|
811
|
+
});
|
|
812
|
+
});
|
|
813
|
+
|
|
814
|
+
// ─────── 覆盖构成的四段与两面(experiment-table.md「覆盖构成」) ───────
|
|
815
|
+
|
|
816
|
+
describe("覆盖构成的四段与两面", () => {
|
|
817
|
+
/**
|
|
818
|
+
* 区分力场景:"retry" 重试两次,第一次携带(历史执行)、第二次新执行——按 attempt 计数的
|
|
819
|
+
* 错误实现会让合计超过题数;"solo" 只有历史执行;"stale" 只有过期结论;"gone" 未跑到。
|
|
820
|
+
*/
|
|
821
|
+
function compositionScope(): Sample {
|
|
822
|
+
const current = snap({
|
|
823
|
+
experimentId: "exp/composition",
|
|
824
|
+
results: [
|
|
825
|
+
res("retry", "passed", { artifactBase: "exp/composition/old/retry/a0" }), // 携带,attempt 0
|
|
826
|
+
res("retry", "passed", { attempt: 1 }), // 新执行,attempt 1
|
|
827
|
+
res("solo", "passed", { artifactBase: "exp/composition/old/solo/a0" }),
|
|
828
|
+
],
|
|
829
|
+
});
|
|
830
|
+
current.configHash = "cfg-current";
|
|
831
|
+
const staleRun = snap({ experimentId: "exp/composition", results: [res("stale", "failed")] });
|
|
832
|
+
staleRun.configHash = "cfg-old";
|
|
833
|
+
const coverage = [
|
|
834
|
+
{
|
|
835
|
+
experimentId: "exp/composition",
|
|
836
|
+
run: current,
|
|
837
|
+
knownEvalIds: ["retry", "solo", "stale", "gone"],
|
|
838
|
+
missingEvalIds: ["stale", "gone"],
|
|
839
|
+
},
|
|
840
|
+
];
|
|
841
|
+
return makeSample("current", [current], [...current.attempts], [], coverage, false, [
|
|
842
|
+
...current.attempts,
|
|
843
|
+
...staleRun.attempts,
|
|
844
|
+
]);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/** experiment 行的覆盖构成副行(subRows 末尾,key 以 `coverage:` 起头)的 record 格。 */
|
|
848
|
+
function coverageCellOf(content: ReturnType<typeof experimentListContent>, experimentId: string): Cell {
|
|
849
|
+
const row = content.rows[0]!.subRows!.find((r) => r.key === `${COVERAGE_ROW_PREFIX}${experimentId}`)!;
|
|
850
|
+
return row.cells.record!;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
it("四段互斥且合计等于已知题数;一题同时有新执行与携带 attempt 时只落新执行段", async () => {
|
|
854
|
+
const [item] = await experimentListData(compositionScope());
|
|
855
|
+
const content = experimentListContent([item!]);
|
|
856
|
+
const coverageCell = coverageCellOf(content, "exp/composition");
|
|
857
|
+
if (coverageCell.kind !== "composition") throw new Error("expected composition cell");
|
|
858
|
+
const byLabel = new Map(
|
|
859
|
+
coverageCell.segments.map((segment) => [
|
|
860
|
+
typeof segment.label === "string" ? segment.label : segment.label.en,
|
|
861
|
+
segment.count,
|
|
862
|
+
]),
|
|
863
|
+
);
|
|
864
|
+
expect(byLabel.get("fresh")).toBe(1); // retry 只落新执行段,不因为它也有携带 attempt 而重复计
|
|
865
|
+
expect(byLabel.get("historical")).toBe(1); // solo
|
|
866
|
+
expect(byLabel.get("stale")).toBe(1); // stale
|
|
867
|
+
expect(byLabel.get("not run")).toBe(1); // gone
|
|
868
|
+
const total = [...coverageCell.segments].reduce((sum, s) => sum + s.count, 0);
|
|
869
|
+
expect(total).toBe(4); // 合计 = knownEvalIds 数,不是 attempts 数
|
|
870
|
+
});
|
|
871
|
+
|
|
872
|
+
it("计数为零的段不出现在两面输出里,与判定计票同一条规则", async () => {
|
|
873
|
+
const s = snap({ experimentId: "exp/all-fresh", results: [res("q", "passed")] });
|
|
874
|
+
const [item] = await experimentListData([s]);
|
|
875
|
+
const content = experimentListContent([item!]);
|
|
876
|
+
const coverageCell = coverageCellOf(content, "exp/all-fresh");
|
|
877
|
+
if (coverageCell.kind !== "composition") throw new Error("expected composition cell");
|
|
878
|
+
expect(coverageCell.segments.filter((s) => s.count > 0)).toHaveLength(1);
|
|
879
|
+
expect(formatCellText(coverageCell, "en")).toBe("1 fresh");
|
|
880
|
+
expect(formatCellText(coverageCell, "zh-CN")).toBe("1 新执行");
|
|
881
|
+
});
|
|
882
|
+
|
|
883
|
+
it("段名走 LocalizedText:zh-CN 与 en 取同一份 segments,只有文案不同,计数与段序逐字相同", async () => {
|
|
884
|
+
const [item] = await experimentListData(compositionScope());
|
|
885
|
+
const content = experimentListContent([item!]);
|
|
886
|
+
const coverageCell = coverageCellOf(content, "exp/composition");
|
|
887
|
+
if (coverageCell.kind !== "composition") throw new Error("expected composition cell");
|
|
888
|
+
const en = formatCellText(coverageCell, "en");
|
|
889
|
+
const zh = formatCellText(coverageCell, "zh-CN");
|
|
890
|
+
expect(en).toBe("1 fresh · 1 historical · 1 stale · 1 not run");
|
|
891
|
+
expect(zh).toBe("1 新执行 · 1 历史执行 · 1 过期结论 · 1 未跑到");
|
|
892
|
+
expect(coverageCell.segments.map((s) => s.count)).toEqual([1, 1, 1, 1]);
|
|
893
|
+
});
|
|
894
|
+
|
|
895
|
+
it("构成格不携带业务语义:它是 experiment 副行独有的格,换一套无关段名照常渲染,渲染面不认识段含义", async () => {
|
|
896
|
+
const [item] = await experimentListData(compositionScope());
|
|
897
|
+
const content = experimentListContent([item!]);
|
|
898
|
+
// 副行紧跟在 Eval / 组行之后,key 前缀把它与题目行区分开——它不是又一道 "retry" 之类的题。
|
|
899
|
+
const rowKeys = content.rows[0]!.subRows!.map((r) => r.key);
|
|
900
|
+
expect(rowKeys.at(-1)).toBe(`${COVERAGE_ROW_PREFIX}exp/composition`);
|
|
901
|
+
expect(rowKeys).toContain("retry");
|
|
902
|
+
|
|
903
|
+
const arbitrarySegments: Cell = {
|
|
904
|
+
kind: "composition",
|
|
905
|
+
segments: [{ label: "x", count: 2 }, { label: "y", count: 0 }],
|
|
906
|
+
};
|
|
907
|
+
expect(formatCellText(arbitrarySegments)).toBe("2 x");
|
|
908
|
+
});
|
|
909
|
+
});
|
|
910
|
+
|
|
715
911
|
// ─────── 计分制字段:evaluationKind 定义期投影 / totalScore 的两级聚合方向 / 预排 ───────
|
|
716
912
|
|
|
717
913
|
it('ExperimentListItem.evaluationKind 是定义期事实投影,不从 attempt 判定推断:failed 的计分制 eval 仍读 "points",passed 的通过制 eval 仍读 "pass"', async () => {
|