niceeval 0.11.4-canary.20 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/dist/i18n/en.d.ts +9 -1
- package/dist/i18n/en.js +11 -2
- package/dist/i18n/zh-CN.d.ts +10 -2
- package/dist/i18n/zh-CN.js +11 -2
- package/dist/record/attempt-evidence.d.ts +1 -1
- package/dist/record/open.js +25 -0
- package/dist/report/assets/series-encoding.d.ts +36 -0
- package/dist/report/assets/series-encoding.js +104 -0
- package/dist/report/built-in/result-components.js +12 -2
- package/dist/report/components/attempt-detail/compute.d.ts +8 -1
- package/dist/report/components/attempt-detail/compute.js +19 -1
- package/dist/report/components/attempt-detail/index.js +25 -3
- package/dist/report/components/attempt-detail/validate.js +3 -2
- package/dist/report/components/entity-lists/compute.d.ts +6 -0
- package/dist/report/components/entity-lists/compute.js +55 -8
- package/dist/report/components/entity-lists/content.d.ts +2 -0
- package/dist/report/components/entity-lists/content.js +59 -6
- package/dist/report/components/entity-lists/index.d.ts +21 -1
- package/dist/report/components/entity-lists/index.js +40 -7
- package/dist/report/components/entity-lists/validate.js +21 -0
- package/dist/report/components/fixtures.js +2 -0
- package/dist/report/definition/cell.d.ts +22 -0
- package/dist/report/definition/cell.js +23 -9
- package/dist/report/definition/primitives/chart.js +125 -25
- package/dist/report/definition/primitives/conversation.d.ts +1 -1
- package/dist/report/definition/primitives/conversation.js +5 -3
- package/dist/report/definition/primitives.js +32 -7
- package/dist/report/definition/tree.d.ts +5 -2
- package/dist/report/definition/tree.js +1 -0
- package/dist/report/index.d.ts +5 -5
- package/dist/report/index.js +2 -2
- package/dist/report/model/aggregate.d.ts +14 -1
- package/dist/report/model/aggregate.js +34 -2
- package/dist/report/model/conversions.d.ts +1 -0
- package/dist/report/model/conversions.js +4 -1
- package/dist/report/model/format.d.ts +6 -5
- package/dist/report/model/format.js +30 -9
- package/dist/report/model/locale.d.ts +19 -3
- package/dist/report/model/locale.js +31 -5
- package/dist/report/model/types.d.ts +41 -4
- package/dist/report/presentation.d.ts +53 -6
- package/dist/report/presentation.js +40 -5
- package/dist/report/react/index.d.ts +3 -3
- package/dist/report/react/index.js +1 -1
- package/dist/report/slices/compute.js +52 -4
- package/dist/report/slices/content.d.ts +2 -1
- package/dist/report/slices/content.js +48 -15
- package/dist/report/slices/validate.js +23 -0
- package/dist/report/tasks.d.ts +7 -1
- package/dist/report/tasks.js +5 -3
- package/dist/runner/types.d.ts +30 -10
- package/dist/sample/index.d.ts +0 -9
- package/dist/sample/index.js +0 -18
- package/dist/sandbox/backend.d.ts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.d.ts +2 -1
- package/dist/sandbox/docker.d.ts +28 -7
- package/dist/sandbox/e2b.d.ts +11 -0
- package/dist/sandbox/identity.d.ts +6 -1
- package/dist/sandbox/layer.d.ts +36 -3
- package/dist/sandbox/local.d.ts +9 -0
- package/dist/sandbox/types.d.ts +14 -8
- package/dist/sandbox/vercel.d.ts +9 -0
- package/docs-site/zh/reference/builtin-agents.mdx +2 -0
- package/docs-site/zh/reference/cli.mdx +1 -0
- package/docs-site/zh/reference/define-agent.mdx +13 -9
- package/docs-site/zh/reference/define-config.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +1 -1
- package/docs-site/zh/reference/results-data.mdx +1 -1
- package/docs-site/zh/tutorials/deploy-report-site.mdx +157 -0
- package/docs-site/zh/tutorials/publish-report.mdx +1 -1
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-providers.mdx +19 -4
- package/package.json +1 -1
- package/src/agents/codex-env.test.ts +8 -1
- package/src/agents/codex.ts +8 -0
- package/src/agents/coding-cli-versions.ts +22 -6
- package/src/cli.ts +23 -1
- package/src/i18n/en.ts +14 -3
- package/src/i18n/zh-CN.ts +13 -2
- package/src/record/attempt-evidence.ts +1 -1
- package/src/record/open.ts +21 -0
- package/src/record/results.test.ts +74 -13
- package/src/record/truncate.ts +32 -7
- package/src/record/writer.ts +39 -8
- package/src/report/assets/series-encoding.tsx +177 -0
- package/src/report/assets/styles.css +97 -5
- package/src/report/built-in/result-components.tsx +23 -2
- package/src/report/components/attempt-detail/attempt-components.test.tsx +41 -0
- package/src/report/components/attempt-detail/compute.ts +20 -2
- package/src/report/components/attempt-detail/index.tsx +32 -2
- package/src/report/components/attempt-detail/validate.test.ts +1 -0
- package/src/report/components/attempt-detail/validate.tsx +3 -1
- package/src/report/components/compute.test.ts +196 -0
- package/src/report/components/entity-lists/compute.ts +73 -7
- package/src/report/components/entity-lists/content.test.ts +14 -8
- package/src/report/components/entity-lists/content.ts +63 -6
- package/src/report/components/entity-lists/index.tsx +93 -8
- package/src/report/components/entity-lists/validate.test.ts +9 -0
- package/src/report/components/entity-lists/validate.ts +16 -0
- package/src/report/components/fixtures.ts +2 -0
- package/src/report/definition/cell.ts +46 -11
- package/src/report/definition/conversation.test.tsx +5 -1
- package/src/report/definition/primitives/chart.tsx +197 -32
- package/src/report/definition/primitives/conversation.tsx +7 -4
- package/src/report/definition/primitives.tsx +81 -9
- package/src/report/definition/tree.ts +6 -2
- package/src/report/index.ts +15 -2
- package/src/report/model/aggregate.ts +40 -2
- package/src/report/model/conversions.ts +5 -0
- package/src/report/model/format.ts +32 -9
- package/src/report/model/locale.ts +35 -5
- package/src/report/model/presentation.test.ts +23 -1
- package/src/report/model/types.ts +40 -4
- package/src/report/presentation.test.tsx +179 -19
- package/src/report/presentation.ts +114 -11
- package/src/report/react/index.tsx +13 -2
- package/src/report/runtime/dual-render.test.tsx +38 -2
- package/src/report/slices/compute.ts +59 -4
- package/src/report/slices/content.ts +55 -16
- package/src/report/slices/delta-table.test.ts +138 -3
- package/src/report/slices/validate.test.ts +29 -0
- package/src/report/slices/validate.ts +16 -0
- package/src/report/tasks.ts +12 -2
- package/src/runner/accept.test.ts +328 -10
- package/src/runner/accept.ts +239 -72
- package/src/runner/attempt.test.ts +25 -13
- package/src/runner/attempt.ts +9 -9
- package/src/runner/config-identity.test.ts +25 -0
- package/src/runner/config-identity.ts +6 -7
- package/src/runner/feedback/coordinator.test.ts +51 -0
- package/src/runner/feedback/coordinator.ts +23 -0
- package/src/runner/feedback/failure.test.ts +15 -0
- package/src/runner/feedback/failure.ts +7 -0
- package/src/runner/feedback/human.test.ts +548 -27
- package/src/runner/feedback/human.ts +321 -47
- package/src/runner/feedback/index.ts +10 -0
- package/src/runner/feedback/input-guard.test.ts +251 -0
- package/src/runner/feedback/input-guard.ts +171 -0
- package/src/runner/feedback/reducer.ts +1 -0
- package/src/runner/fingerprint.test.ts +24 -0
- package/src/runner/gate-lease.test.ts +38 -14
- package/src/runner/ledger.test.ts +10 -7
- package/src/runner/ledger.ts +9 -2
- package/src/runner/lock.test.ts +40 -16
- package/src/runner/manifest.ts +8 -9
- package/src/runner/reporters/artifacts.ts +25 -3
- package/src/runner/run.test.ts +61 -0
- package/src/runner/run.ts +34 -9
- package/src/runner/sandbox-pool.test.ts +216 -12
- package/src/runner/sandbox-pool.ts +20 -2
- package/src/runner/types.ts +30 -10
- package/src/sample/index.ts +0 -16
- package/src/sample/sample.test.ts +78 -0
- package/src/sandbox/backend.ts +1 -1
- package/src/sandbox/commands.ts +4 -7
- package/src/sandbox/compose.test.ts +26 -1
- package/src/sandbox/compose.ts +7 -3
- package/src/sandbox/docker.test.ts +66 -1
- package/src/sandbox/docker.ts +89 -65
- package/src/sandbox/dockerfile-build.ts +4 -1
- package/src/sandbox/e2b-agent-template.test.ts +18 -1
- package/src/sandbox/e2b-agent-template.ts +22 -6
- package/src/sandbox/e2b.test.ts +64 -0
- package/src/sandbox/e2b.ts +27 -5
- package/src/sandbox/identity.ts +7 -2
- package/src/sandbox/layer.test.ts +23 -4
- package/src/sandbox/layer.ts +127 -43
- package/src/sandbox/link.test.ts +5 -2
- package/src/sandbox/local.test.ts +8 -8
- package/src/sandbox/local.ts +28 -6
- package/src/sandbox/operations.test.ts +4 -4
- package/src/sandbox/operations.ts +5 -5
- package/src/sandbox/runtime.ts +18 -3
- package/src/sandbox/types.ts +14 -8
- package/src/sandbox/vercel.test.ts +58 -3
- package/src/sandbox/vercel.ts +38 -6
- package/src/show/index.ts +11 -30
- package/src/show/render.test.ts +23 -1
- package/src/show/render.ts +6 -3
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
|
|
9
9
|
import { comparabilityConfigOf, deepEqualJson } from "../../../sample/index.js";
|
|
10
10
|
import { foldEvalVerdict } from "../../../shared/verdict.js";
|
|
11
|
-
import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems, historicalOf, locatorOf, resolveInput, } from "../../model/aggregate.js";
|
|
11
|
+
import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems, historicalOf, locatorOf, msSince, resolveInput, staleReferenceOf, } from "../../model/aggregate.js";
|
|
12
12
|
import { attemptCostUSD, costUSD, durationMs, examScore, passRate, tokens, totalScore } from "../../model/metrics.js";
|
|
13
13
|
import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../../../assertions/display.js";
|
|
14
14
|
import { firstLine } from "../../../util.js";
|
|
@@ -47,6 +47,11 @@ export function failureSummaryOf(result) {
|
|
|
47
47
|
async function attemptListItemOf(item) {
|
|
48
48
|
const result = item.attempt.result;
|
|
49
49
|
const { summary, more } = failureSummaryOf(result);
|
|
50
|
+
const historical = historicalOf(item);
|
|
51
|
+
// 缺 startedAt(legacy / 第三方落盘)时退化到所属快照的 startedAt——时效标注宁可粗一档
|
|
52
|
+
// 时距,不留空字段(与 dedupeAttempts「缺才不去重」同一条「不伪造」纪律,这里伪造的只是
|
|
53
|
+
// 展示粒度,不影响身份判定)。
|
|
54
|
+
const startedAt = result.startedAt ?? item.run.startedAt;
|
|
50
55
|
return {
|
|
51
56
|
experimentId: experimentIdOf(item),
|
|
52
57
|
evalId: evalIdOf(item),
|
|
@@ -60,14 +65,21 @@ async function attemptListItemOf(item) {
|
|
|
60
65
|
totalScore: await computeCell(totalScore, [item]),
|
|
61
66
|
durationMs: result.durationMs,
|
|
62
67
|
costUSD: attemptCostUSD(result),
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
startedAt: result.startedAt ?? item.run.startedAt,
|
|
67
|
-
historical: historicalOf(item),
|
|
68
|
+
startedAt,
|
|
69
|
+
historical,
|
|
70
|
+
...(historical ? { staleSinceMs: msSince(startedAt) } : {}),
|
|
68
71
|
locator: locatorOf(item),
|
|
69
72
|
};
|
|
70
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* 「只看新执行」开关在场的判据(docs/feature/reports/components/summaries/experiment-table.md
|
|
76
|
+
* 「只看新执行」):Sample 里既没有历史执行也没有过期结论时不画开关——一个永远不改变行集的
|
|
77
|
+
* 控件只会让人怀疑自己看漏了什么。`ExperimentTable` 与内建默认报告共用同一条判据。
|
|
78
|
+
*/
|
|
79
|
+
export function hasHistoricalOrStale(items) {
|
|
80
|
+
return items.some((item) => Object.keys(item.staleReferences).length > 0 ||
|
|
81
|
+
item.evalRows.some((row) => row.attempts.some((attempt) => attempt.historical)));
|
|
82
|
+
}
|
|
71
83
|
/** 已选出的 AttemptHandle[] → 列表行;顺序保持传入顺序(不再次按 Sample 去重)。 */
|
|
72
84
|
export async function attemptRowsOf(attempts) {
|
|
73
85
|
return Promise.all(attempts.map((attempt) => attemptListItemOf({ attempt, run: attempt.run, watermark: attempt.run })));
|
|
@@ -158,6 +170,38 @@ function byMetricDescThenId(valueOf) {
|
|
|
158
170
|
return vb - va || a.experimentId.localeCompare(b.experimentId);
|
|
159
171
|
};
|
|
160
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* 覆盖缺口两档占位的「过期结论」参考(docs/feature/reports/components/summaries/experiment-table.md
|
|
175
|
+
* 「覆盖缺口的两档占位行」):`missingEvalIds` 里、`historyAttempts` 中存在与 `anchorConfigHash`
|
|
176
|
+
* 不可比判定的题,取其中最近一条。`fresh` 为 true 时整份不给参考——读者已声明只看新执行,
|
|
177
|
+
* 占位行就不再把被排除的历史结论请回来。
|
|
178
|
+
*/
|
|
179
|
+
function staleReferencesFor(experimentId, missingEvalIds, historyAttempts, anchorConfigHash, fresh) {
|
|
180
|
+
if (fresh || missingEvalIds.length === 0)
|
|
181
|
+
return {};
|
|
182
|
+
const missing = new Set(missingEvalIds);
|
|
183
|
+
const candidatesByEval = new Map();
|
|
184
|
+
for (const attempt of historyAttempts) {
|
|
185
|
+
if (attempt.experimentId !== experimentId)
|
|
186
|
+
continue;
|
|
187
|
+
if (!missing.has(attempt.evalId))
|
|
188
|
+
continue;
|
|
189
|
+
if (attempt.run.configHash === anchorConfigHash)
|
|
190
|
+
continue; // 可比,不是「过期结论」候选
|
|
191
|
+
const list = candidatesByEval.get(attempt.evalId);
|
|
192
|
+
if (list)
|
|
193
|
+
list.push(attempt);
|
|
194
|
+
else
|
|
195
|
+
candidatesByEval.set(attempt.evalId, [attempt]);
|
|
196
|
+
}
|
|
197
|
+
const out = {};
|
|
198
|
+
for (const [evalId, candidates] of candidatesByEval) {
|
|
199
|
+
const reference = staleReferenceOf(candidates);
|
|
200
|
+
if (reference)
|
|
201
|
+
out[evalId] = reference;
|
|
202
|
+
}
|
|
203
|
+
return out;
|
|
204
|
+
}
|
|
161
205
|
/**
|
|
162
206
|
* `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始排序按这份列表
|
|
163
207
|
* 自身的题型构成选择主读数——纯通过制沿用端到端通过率降序,纯计分制改按总分降序(缺数据
|
|
@@ -168,7 +212,7 @@ function byMetricDescThenId(valueOf) {
|
|
|
168
212
|
* 看跨配置演化用 run 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
169
213
|
*/
|
|
170
214
|
export async function experimentListData(input) {
|
|
171
|
-
const { runs, attempts, coverage } = resolveInput(input);
|
|
215
|
+
const { runs, attempts, coverage, historyAttempts, fresh } = resolveInput(input);
|
|
172
216
|
const coverageByExperiment = new Map(coverage.map((c) => [c.experimentId, c]));
|
|
173
217
|
// 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
|
|
174
218
|
const configByExperiment = new Map();
|
|
@@ -216,6 +260,7 @@ export async function experimentListData(input) {
|
|
|
216
260
|
}
|
|
217
261
|
const experiment = newest.run.experiment ?? newest.attempt.result.experiment;
|
|
218
262
|
const model = newest.attempt.result.model ?? newest.run.model;
|
|
263
|
+
const missingEvalIds = coverageByExperiment.get(experimentId)?.missingEvalIds ?? [];
|
|
219
264
|
out.push({
|
|
220
265
|
experimentId,
|
|
221
266
|
agent: newest.run.agent || newest.attempt.result.agent,
|
|
@@ -231,7 +276,8 @@ export async function experimentListData(input) {
|
|
|
231
276
|
evals: stats.evals,
|
|
232
277
|
attempts: stats.attempts,
|
|
233
278
|
historicalAttempts: group.filter(historicalOf).length,
|
|
234
|
-
missingEvalIds
|
|
279
|
+
missingEvalIds,
|
|
280
|
+
staleReferences: staleReferencesFor(experimentId, missingEvalIds, historyAttempts, coverageByExperiment.get(experimentId)?.run.configHash, fresh),
|
|
235
281
|
lastRunAt: stats.lastRunAt,
|
|
236
282
|
evalRows,
|
|
237
283
|
});
|
|
@@ -263,6 +309,7 @@ export async function experimentListData(input) {
|
|
|
263
309
|
attempts: 0,
|
|
264
310
|
historicalAttempts: 0,
|
|
265
311
|
missingEvalIds: coverageEntry.missingEvalIds,
|
|
312
|
+
staleReferences: staleReferencesFor(coverageEntry.experimentId, coverageEntry.missingEvalIds, historyAttempts, anchor.configHash, fresh),
|
|
266
313
|
lastRunAt: anchor.startedAt,
|
|
267
314
|
evalRows: [],
|
|
268
315
|
});
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { TableContent } from "../../definition/cell.ts";
|
|
2
2
|
import type { AttemptListItem, EvalListItem, ExperimentListItem } from "../../model/types.ts";
|
|
3
|
+
/** 覆盖构成副行的 key 前缀;测试与消费方靠它把这一行从 Eval / 组行里筛出去。 */
|
|
4
|
+
export declare const COVERAGE_ROW_PREFIX = "coverage:";
|
|
3
5
|
export declare function experimentListContent(items: readonly ExperimentListItem[]): TableContent;
|
|
4
6
|
export declare function evalListContent(items: readonly EvalListItem[]): TableContent;
|
|
5
7
|
export declare function attemptListContent(items: readonly AttemptListItem[]): TableContent;
|
|
@@ -5,6 +5,10 @@ import { localizedMessage } from "../../model/locale.js";
|
|
|
5
5
|
import { experimentListEvaluationKindComposition } from "../../model/format.js";
|
|
6
6
|
/** 组内零样本读数格的结构化原因码;renderer 经 missingText 映射文案。 */
|
|
7
7
|
const GROUP_NO_SAMPLES = "noSamples";
|
|
8
|
+
/** 覆盖缺口两档占位行共用的原因码——两档说的是同一个事实,`code` 相同。 */
|
|
9
|
+
const NO_CURRENT_RESULT = "noCurrentResult";
|
|
10
|
+
/** 覆盖构成四段的声明序:新执行 → 历史执行 → 过期结论 → 未跑到(段序即声明序)。 */
|
|
11
|
+
const COVERAGE_SEGMENT_KEYS = ["coverage.fresh", "coverage.historical", "coverage.stale", "coverage.notRun"];
|
|
8
12
|
/** 三张实体表共用的列表头。文案单源在 locale 字典,这里只烤成随列走的 LocalizedText。 */
|
|
9
13
|
const HEADER = {
|
|
10
14
|
entity: localizedMessage("experimentList.experiment"),
|
|
@@ -86,7 +90,7 @@ function attemptCells(item) {
|
|
|
86
90
|
entity: {
|
|
87
91
|
kind: "locator",
|
|
88
92
|
locator: item.locator,
|
|
89
|
-
staleSinceMs: item.
|
|
93
|
+
staleSinceMs: item.staleSinceMs,
|
|
90
94
|
verdict,
|
|
91
95
|
},
|
|
92
96
|
verdict: { kind: "verdict", verdict },
|
|
@@ -216,10 +220,20 @@ function evalRow(row, view, label) {
|
|
|
216
220
|
subRows: row.attempts.map((a) => attemptRow(a, view.columns)),
|
|
217
221
|
};
|
|
218
222
|
}
|
|
219
|
-
|
|
223
|
+
/**
|
|
224
|
+
* 覆盖缺口的两档占位行(docs/feature/reports/components/summaries/experiment-table.md
|
|
225
|
+
* 「覆盖缺口的两档占位行」):`code` 与补跑命令两档相同——都是「当前配置下没有结果」这同一个
|
|
226
|
+
* 事实;差别只在有没有 `reference`,带参考时它替代原因文案,不把同一句话说两遍。
|
|
227
|
+
*/
|
|
228
|
+
function placeholderRow(experimentId, evalId, label, columns, reference) {
|
|
220
229
|
const bag = {
|
|
221
230
|
entity: textCell(label),
|
|
222
|
-
|
|
231
|
+
record: {
|
|
232
|
+
kind: "missing",
|
|
233
|
+
code: NO_CURRENT_RESULT,
|
|
234
|
+
detail: `niceeval exp ${experimentId}`,
|
|
235
|
+
...(reference ? { reference } : {}),
|
|
236
|
+
},
|
|
223
237
|
};
|
|
224
238
|
return {
|
|
225
239
|
key: `${experimentId}:${evalId}:missing`,
|
|
@@ -252,7 +266,7 @@ function evalRowsEvaluationKindComposition(evalRows) {
|
|
|
252
266
|
function leafTableRow(member, label, item, view) {
|
|
253
267
|
return member.kind === "eval"
|
|
254
268
|
? evalRow(member.row, view, label)
|
|
255
|
-
: placeholderRow(item.experimentId, member.evalId, label, view.columns);
|
|
269
|
+
: placeholderRow(item.experimentId, member.evalId, label, view.columns, item.staleReferences[member.evalId]);
|
|
256
270
|
}
|
|
257
271
|
function groupTableRow(
|
|
258
272
|
/** 组行显示的这一段(不带祖先前缀)。 */
|
|
@@ -348,13 +362,52 @@ function nestLevel(members, dirPrefix, labelPrefix, item, view) {
|
|
|
348
362
|
.map((member) => leafTableRow(member, relativeLabel(memberEvalId(member), labelPrefix), item, view));
|
|
349
363
|
return [...rows, ...flat];
|
|
350
364
|
}
|
|
351
|
-
/**
|
|
365
|
+
/**
|
|
366
|
+
* 覆盖构成的四段计数(docs/feature/reports/components/summaries/experiment-table.md「覆盖构成」):
|
|
367
|
+
* 新执行 / 历史执行 = 有 attempt 的题按其中是否有非历史(新执行)划分;过期结论 = 缺口题里
|
|
368
|
+
* `staleReferences` 命中的那些;未跑到 = 缺口题里剩下的那些。四段互斥且合计等于已知题数。
|
|
369
|
+
*/
|
|
370
|
+
function coverageSegments(item) {
|
|
371
|
+
let fresh = 0;
|
|
372
|
+
for (const row of item.evalRows) {
|
|
373
|
+
if (row.attempts.some((a) => !a.historical))
|
|
374
|
+
fresh += 1;
|
|
375
|
+
}
|
|
376
|
+
const historical = item.evalRows.length - fresh;
|
|
377
|
+
const stale = Object.keys(item.staleReferences).length;
|
|
378
|
+
const notRun = item.missingEvalIds.length - stale;
|
|
379
|
+
return COVERAGE_SEGMENT_KEYS.map((key, i) => ({
|
|
380
|
+
label: localizedMessage(key),
|
|
381
|
+
count: [fresh, historical, stale, notRun][i],
|
|
382
|
+
}));
|
|
383
|
+
}
|
|
384
|
+
/** 覆盖构成副行的 key 前缀;测试与消费方靠它把这一行从 Eval / 组行里筛出去。 */
|
|
385
|
+
export const COVERAGE_ROW_PREFIX = "coverage:";
|
|
386
|
+
/**
|
|
387
|
+
* 覆盖构成副行(docs/feature/reports/components/summaries/experiment-table.md「覆盖构成」):
|
|
388
|
+
* experiment 行 subRows 的最后一条,把已知题按结论出身分成四段互斥的构成格,交给中立的
|
|
389
|
+
* `composition` 格——渲染在同一个 `record` 列位置(与 Eval / Attempt 行的判定构成、占位行的
|
|
390
|
+
* missing 格同一个槽位,三种形态各自对应不同的行语义,不是同一行的三种读法)。
|
|
391
|
+
* 它不是 Eval / 组行,没有身份(entity 是 notApplicable),不参与嵌套排序或收起判定。
|
|
392
|
+
*/
|
|
393
|
+
function coverageRow(item, view) {
|
|
394
|
+
const bag = {
|
|
395
|
+
entity: { kind: "notApplicable" },
|
|
396
|
+
record: { kind: "composition", segments: coverageSegments(item) },
|
|
397
|
+
};
|
|
398
|
+
return {
|
|
399
|
+
key: `${COVERAGE_ROW_PREFIX}${item.experimentId}`,
|
|
400
|
+
cells: projectCells(bag, view.columns),
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
/** experiment 的 evalRows + missingEvalIds → 递归嵌套的 subRows,末尾追加覆盖构成副行。 */
|
|
352
404
|
function experimentSubRows(item, view) {
|
|
353
405
|
const members = [
|
|
354
406
|
...item.evalRows.map((row) => ({ kind: "eval", row })),
|
|
355
407
|
...item.missingEvalIds.map((evalId) => ({ kind: "missing", evalId })),
|
|
356
408
|
];
|
|
357
|
-
|
|
409
|
+
const nested = nestLevel(members, "", "", item, view);
|
|
410
|
+
return members.length > 0 ? [...nested, coverageRow(item, view)] : nested;
|
|
358
411
|
}
|
|
359
412
|
function experimentRow(item, view) {
|
|
360
413
|
const bag = {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import type { TableContent } from "../../definition/cell.ts";
|
|
1
2
|
import type { ReportInput } from "../../model/types.ts";
|
|
2
3
|
import type { AttemptHandle, Sample } from "../../../record/types.ts";
|
|
3
|
-
import type
|
|
4
|
+
import { type ReportLocale } from "../../model/locale.ts";
|
|
4
5
|
import { attemptRowsOf } from "./compute.ts";
|
|
5
6
|
export { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./validate.ts";
|
|
6
7
|
export interface FailureListProps {
|
|
@@ -24,5 +25,24 @@ export interface ExperimentTableProps {
|
|
|
24
25
|
locale?: ReportLocale;
|
|
25
26
|
className?: string;
|
|
26
27
|
}
|
|
28
|
+
interface ExperimentTableViewProps {
|
|
29
|
+
fullContent: TableContent;
|
|
30
|
+
/** 只在 Sample 里确有历史执行或过期结论时才非 null——开关是否出现由这个字段的存在与否决定。 */
|
|
31
|
+
freshContent: TableContent | null;
|
|
32
|
+
sort?: string;
|
|
33
|
+
searchable?: boolean;
|
|
34
|
+
locale?: ReportLocale;
|
|
35
|
+
className?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* `ExperimentTable` 的渲染叶子:web 面在有 `freshContent` 时叠一层「只看新执行」开关——
|
|
39
|
+
* 两份 Content 都已经由 `ExperimentTable` 按 fresh 与非 fresh 两态重新投影好,`Table` 原语
|
|
40
|
+
* 本身不知道这个开关存在(本体里没有任何时效分支,语义全部住在投影层)。切换纯靠原生
|
|
41
|
+
* checkbox 状态经 CSS `:has()` 驱动的可见性完成(styles.css `.niceeval-fresh-toggle`),
|
|
42
|
+
* 不重新请求或重新计算任何数据,也不需要 JavaScript——初始态(未勾选)展示 `fullContent`
|
|
43
|
+
* 这份完整表。text 面没有这个开关的等价物,`--fresh` CLI 选项就是它的入口,这里只渲染
|
|
44
|
+
* `fullContent`。
|
|
45
|
+
*/
|
|
46
|
+
export declare const ExperimentTableView: import("../../definition/tree.ts").ReportComponent<ExperimentTableViewProps>;
|
|
27
47
|
export declare const ExperimentTable: import("../../definition/tree.ts").ReportComponent<ExperimentTableProps>;
|
|
28
48
|
export { attemptRowsOf };
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { jsx as _jsx } from "react/jsx-runtime";
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
2
|
// 实体列表组合件:FailureList / AttemptList;列表本体是 Table 原语。
|
|
3
3
|
import { defineComponent } from "../../definition/tree.js";
|
|
4
4
|
import { TableContentView } from "../../definition/primitives.js";
|
|
5
5
|
import { collectItems, locatorOf, resolveInput } from "../../model/aggregate.js";
|
|
6
|
-
import {
|
|
6
|
+
import { localeText } from "../../model/locale.js";
|
|
7
|
+
import { attemptListData, attemptRowsOf, hasHistoricalOrStale } from "./compute.js";
|
|
7
8
|
import { attemptListContent, experimentListContent } from "./content.js";
|
|
8
9
|
import { toAttemptRows, toExperimentRows } from "../../model/conversions.js";
|
|
9
10
|
export { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./validate.js";
|
|
@@ -35,13 +36,45 @@ export const AttemptList = defineComponent(async (props) => {
|
|
|
35
36
|
return _jsx(TableContentView, { data: attemptListContent(rows), locale: props.locale, className: props.className });
|
|
36
37
|
});
|
|
37
38
|
AttemptList.displayName = "AttemptList";
|
|
39
|
+
/**
|
|
40
|
+
* `ExperimentTable` 的渲染叶子:web 面在有 `freshContent` 时叠一层「只看新执行」开关——
|
|
41
|
+
* 两份 Content 都已经由 `ExperimentTable` 按 fresh 与非 fresh 两态重新投影好,`Table` 原语
|
|
42
|
+
* 本身不知道这个开关存在(本体里没有任何时效分支,语义全部住在投影层)。切换纯靠原生
|
|
43
|
+
* checkbox 状态经 CSS `:has()` 驱动的可见性完成(styles.css `.niceeval-fresh-toggle`),
|
|
44
|
+
* 不重新请求或重新计算任何数据,也不需要 JavaScript——初始态(未勾选)展示 `fullContent`
|
|
45
|
+
* 这份完整表。text 面没有这个开关的等价物,`--fresh` CLI 选项就是它的入口,这里只渲染
|
|
46
|
+
* `fullContent`。
|
|
47
|
+
*/
|
|
48
|
+
export const ExperimentTableView = defineComponent({
|
|
49
|
+
dimensions: () => ({}),
|
|
50
|
+
web(props, ctx) {
|
|
51
|
+
const table = (_jsx(TableContentView, { data: props.fullContent, sort: props.sort, searchable: props.searchable, locale: props.locale, className: props.className }));
|
|
52
|
+
if (!props.freshContent)
|
|
53
|
+
return table;
|
|
54
|
+
const locale = props.locale ?? ctx.locale;
|
|
55
|
+
return (_jsxs("div", { className: "niceeval-report niceeval-experiment-table", children: [_jsxs("label", { className: "niceeval-fresh-toggle", children: [_jsx("input", { type: "checkbox" }), localeText(locale, "experimentTable.freshOnlyToggle")] }), _jsx("div", { className: "niceeval-fresh-all", children: table }), _jsx("div", { className: "niceeval-fresh-only", children: _jsx(TableContentView, { data: props.freshContent, sort: props.sort, searchable: props.searchable, locale: props.locale, className: props.className }) })] }));
|
|
56
|
+
},
|
|
57
|
+
text(props, ctx) {
|
|
58
|
+
return ctx.render(_jsx(TableContentView, { data: props.fullContent, sort: props.sort, searchable: props.searchable, locale: props.locale, className: props.className }));
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
ExperimentTableView.displayName = "ExperimentTableView";
|
|
38
62
|
export const ExperimentTable = defineComponent(async (props, ctx) => {
|
|
39
|
-
const
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
const
|
|
63
|
+
const sample = props.input ?? ctx.scope;
|
|
64
|
+
const full = await toExperimentRows(sample);
|
|
65
|
+
const fullContent = experimentListContent(full);
|
|
66
|
+
const hasPassRate = fullContent.columns.some((column) => column.key === "passRate");
|
|
67
|
+
const hasTotalScore = fullContent.columns.some((column) => column.key === "totalScore");
|
|
43
68
|
const defaultSort = hasPassRate === hasTotalScore ? undefined : hasPassRate ? "passRate" : "totalScore";
|
|
44
|
-
|
|
69
|
+
const freshContent = hasHistoricalOrStale(full)
|
|
70
|
+
? experimentListContent(await toExperimentRows(sample.freshOnly()))
|
|
71
|
+
: null;
|
|
72
|
+
// 没有历史执行也没有过期结论时直接产出 `<TableContentView>`,与开关引入前的树形状逐字相同——
|
|
73
|
+
// 只有真的需要开关那一态,才多绕一层 `ExperimentTableView`。
|
|
74
|
+
if (freshContent === null) {
|
|
75
|
+
return (_jsx(TableContentView, { data: fullContent, sort: props.sort ?? defaultSort, searchable: props.searchable ?? true, locale: props.locale, className: props.className }));
|
|
76
|
+
}
|
|
77
|
+
return (_jsx(ExperimentTableView, { fullContent: fullContent, freshContent: freshContent, sort: props.sort ?? defaultSort, searchable: props.searchable ?? true, locale: props.locale, className: props.className }));
|
|
45
78
|
});
|
|
46
79
|
ExperimentTable.displayName = "ExperimentTable";
|
|
47
80
|
// re-export for callers that still import attemptRowsOf locally
|
|
@@ -40,8 +40,22 @@ function attemptListItemProblem(value, path) {
|
|
|
40
40
|
return `"${path}.startedAt" must be a string`;
|
|
41
41
|
if (typeof value.historical !== "boolean")
|
|
42
42
|
return `"${path}.historical" must be a boolean`;
|
|
43
|
+
if (!(value.staleSinceMs === undefined || typeof value.staleSinceMs === "number")) {
|
|
44
|
+
return `"${path}.staleSinceMs" must be a number or omitted`;
|
|
45
|
+
}
|
|
46
|
+
if (typeof value.locator !== "string")
|
|
47
|
+
return `"${path}.locator" must be a string`;
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
function staleConclusionReferenceProblem(value, path) {
|
|
51
|
+
if (!isObject(value))
|
|
52
|
+
return `"${path}" must be an object`;
|
|
43
53
|
if (typeof value.locator !== "string")
|
|
44
54
|
return `"${path}.locator" must be a string`;
|
|
55
|
+
if (typeof value.verdict !== "string")
|
|
56
|
+
return `"${path}.verdict" must be a string`;
|
|
57
|
+
if (typeof value.staleSinceMs !== "number")
|
|
58
|
+
return `"${path}.staleSinceMs" must be a number`;
|
|
45
59
|
return null;
|
|
46
60
|
}
|
|
47
61
|
export const validateExperimentListData = (data) => arrayProblem(data, "data", (item, path) => {
|
|
@@ -81,6 +95,13 @@ export const validateExperimentListData = (data) => arrayProblem(data, "data", (
|
|
|
81
95
|
const missingProblem = arrayProblem(item.missingEvalIds, `${path}.missingEvalIds`, (id, idPath) => typeof id === "string" ? null : `"${idPath}" must be a string`);
|
|
82
96
|
if (missingProblem !== null)
|
|
83
97
|
return missingProblem;
|
|
98
|
+
if (!isObject(item.staleReferences))
|
|
99
|
+
return `"${path}.staleReferences" must be an object`;
|
|
100
|
+
for (const [evalId, reference] of Object.entries(item.staleReferences)) {
|
|
101
|
+
const referenceProblem = staleConclusionReferenceProblem(reference, `${path}.staleReferences.${evalId}`);
|
|
102
|
+
if (referenceProblem !== null)
|
|
103
|
+
return referenceProblem;
|
|
104
|
+
}
|
|
84
105
|
if (typeof item.lastRunAt !== "string")
|
|
85
106
|
return `"${path}.lastRunAt" must be a string`;
|
|
86
107
|
return arrayProblem(item.evalRows, `${path}.evalRows`, (row, rowPath) => {
|
|
@@ -281,6 +281,7 @@ export const experimentListItems = [
|
|
|
281
281
|
attempts: 3,
|
|
282
282
|
historicalAttempts: 0,
|
|
283
283
|
missingEvalIds: [],
|
|
284
|
+
staleReferences: {},
|
|
284
285
|
lastRunAt: "2026-07-01T10:00:00Z",
|
|
285
286
|
evalRows: [
|
|
286
287
|
{
|
|
@@ -321,6 +322,7 @@ export const experimentListItems = [
|
|
|
321
322
|
attempts: 1,
|
|
322
323
|
historicalAttempts: 0,
|
|
323
324
|
missingEvalIds: [],
|
|
325
|
+
staleReferences: {},
|
|
324
326
|
lastRunAt: "2026-07-01T11:30:00Z",
|
|
325
327
|
evalRows: [
|
|
326
328
|
{
|
|
@@ -25,6 +25,10 @@ export type Cell = {
|
|
|
25
25
|
readonly counts?: VerdictCounts;
|
|
26
26
|
/** 计票覆盖的 attempt 引用(有证据可下钻的计票格才携带,如稳定性矩阵)。 */
|
|
27
27
|
readonly refs?: readonly AttemptLocator[];
|
|
28
|
+
/** 单判定形态历史执行的距今毫秒数;新执行时省略,不伪造 0(与 locator 格同一条纪律)。 */
|
|
29
|
+
readonly staleSinceMs?: number;
|
|
30
|
+
/** 单判定形态省略判定词、只留判定符(如对照矩阵逐格只放得下一个符号的场景)。 */
|
|
31
|
+
readonly bare?: boolean;
|
|
28
32
|
} | {
|
|
29
33
|
readonly kind: "score";
|
|
30
34
|
readonly earned: number;
|
|
@@ -48,6 +52,24 @@ export type Cell = {
|
|
|
48
52
|
} | {
|
|
49
53
|
readonly kind: "missing";
|
|
50
54
|
readonly code: string;
|
|
55
|
+
/** 补上这一格的命令,可直接复制(如 `niceeval exp <experimentId>`)。 */
|
|
56
|
+
readonly detail?: string;
|
|
57
|
+
/**
|
|
58
|
+
* 记录里存在、但与当前基准不可比的最近一条判定;不进任何计数,两面都降饱和显示,
|
|
59
|
+
* 只提供 locator 下钻(docs/feature/reports/library/presentation.md「missing 格的完整形状」)。
|
|
60
|
+
*/
|
|
61
|
+
readonly reference?: {
|
|
62
|
+
readonly locator: AttemptLocator;
|
|
63
|
+
readonly verdict: Verdict | "skipped";
|
|
64
|
+
readonly staleSinceMs: number;
|
|
65
|
+
};
|
|
66
|
+
} | {
|
|
67
|
+
readonly kind: "composition";
|
|
68
|
+
/** 段名由投影按 locale 给出;Table 不认识段的业务含义,只按声明序渲染,计数为零的段不显示。 */
|
|
69
|
+
readonly segments: readonly {
|
|
70
|
+
readonly label: LocalizedText;
|
|
71
|
+
readonly count: number;
|
|
72
|
+
}[];
|
|
51
73
|
};
|
|
52
74
|
export interface ColumnSpec {
|
|
53
75
|
readonly key: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// 官方表格数据协议的单元格类型(docs/feature/reports/components/primitives/table.md「单元格类型」)。
|
|
2
2
|
// Table / Grid / Stat 共用这份判别联合;自定义 Source 要交给这些原语必须适配成这个形状。
|
|
3
|
-
import { formatMetricValue, missingText, verdictMark } from "../model/format.js";
|
|
3
|
+
import { formatMetricValue, formatTimeDistance, missingText, verdictMark } from "../model/format.js";
|
|
4
4
|
import { DEFAULT_REPORT_LOCALE, localeText, resolveLocalizedText, } from "../model/locale.js";
|
|
5
5
|
/** 把 Cell 折成 text 面可见字符串(缺数据 / 不适用统一不补成 0)。 */
|
|
6
6
|
export function formatCellText(cell, locale) {
|
|
@@ -9,12 +9,18 @@ export function formatCellText(cell, locale) {
|
|
|
9
9
|
switch (cell.kind) {
|
|
10
10
|
case "notApplicable":
|
|
11
11
|
return "—";
|
|
12
|
-
case "missing":
|
|
13
|
-
|
|
12
|
+
case "missing": {
|
|
13
|
+
const loc = locale ?? DEFAULT_REPORT_LOCALE;
|
|
14
|
+
// 带参考时用参考替代原因文案——参考本身已经说明这一格为什么空着,不把同一句话说两遍。
|
|
15
|
+
const reason = cell.reference
|
|
16
|
+
? `${verdictMark(cell.reference.verdict === "skipped" ? "skipped" : cell.reference.verdict)} ${cell.reference.locator} ${formatTimeDistance(cell.reference.staleSinceMs, loc)}`
|
|
17
|
+
: missingText(cell.code, loc);
|
|
18
|
+
return cell.detail ? `${reason} · ${cell.detail}` : reason;
|
|
19
|
+
}
|
|
14
20
|
case "text":
|
|
15
21
|
return cell.detail ? `${cell.text}\n ${cell.detail}` : cell.text;
|
|
16
22
|
case "locator": {
|
|
17
|
-
const stale = cell.staleSinceMs !== undefined ? `
|
|
23
|
+
const stale = cell.staleSinceMs !== undefined ? ` ${formatTimeDistance(cell.staleSinceMs, locale ?? DEFAULT_REPORT_LOCALE)}` : "";
|
|
18
24
|
const mark = cell.verdict !== undefined ? `${verdictMark(cell.verdict === "skipped" ? "skipped" : cell.verdict)} ` : "";
|
|
19
25
|
return `${mark}${cell.locator}${stale}`;
|
|
20
26
|
}
|
|
@@ -34,8 +40,12 @@ export function formatCellText(cell, locale) {
|
|
|
34
40
|
}
|
|
35
41
|
if (cell.verdict !== undefined) {
|
|
36
42
|
const v = cell.verdict === "skipped" ? "skipped" : cell.verdict;
|
|
43
|
+
const stale = cell.staleSinceMs !== undefined ? ` ${formatTimeDistance(cell.staleSinceMs, loc)}` : "";
|
|
37
44
|
// 判定符与判定词同场,与 locator 格、web 面同一条纪律:单色打印下照样读得出。
|
|
38
|
-
|
|
45
|
+
// bare 省略判定词,只留判定符(+ 可选时距),供逐格空间紧张的场景(如对照矩阵)使用。
|
|
46
|
+
if (cell.bare)
|
|
47
|
+
return `${verdictMark(v)}${stale}`;
|
|
48
|
+
return `${verdictMark(v)} ${localeText(loc, `verdict.${v}`)}${stale}`;
|
|
39
49
|
}
|
|
40
50
|
return "—";
|
|
41
51
|
}
|
|
@@ -43,6 +53,14 @@ export function formatCellText(cell, locale) {
|
|
|
43
53
|
const m = cell.metric;
|
|
44
54
|
return formatMetricValue(m.value, m.unit, m.format, locale ?? DEFAULT_REPORT_LOCALE);
|
|
45
55
|
}
|
|
56
|
+
case "composition": {
|
|
57
|
+
const loc = locale ?? DEFAULT_REPORT_LOCALE;
|
|
58
|
+
// text 面退化成与判定计票同构的计数串,与「计数为零的段不出现」同一条规则。
|
|
59
|
+
const parts = cell.segments
|
|
60
|
+
.filter((segment) => segment.count > 0)
|
|
61
|
+
.map((segment) => `${segment.count} ${resolveLocalizedText(segment.label, loc)}`);
|
|
62
|
+
return parts.join(" · ") || "—";
|
|
63
|
+
}
|
|
46
64
|
default: {
|
|
47
65
|
const _exhaustive = cell;
|
|
48
66
|
return _exhaustive;
|
|
@@ -76,7 +94,3 @@ export function flattenTableContentForText(content, locale) {
|
|
|
76
94
|
walk(row, 0);
|
|
77
95
|
return { columns, rows };
|
|
78
96
|
}
|
|
79
|
-
function formatStale(ms) {
|
|
80
|
-
const days = Math.max(1, Math.round(ms / 86_400_000));
|
|
81
|
-
return `${days}d`;
|
|
82
|
-
}
|