niceeval 0.11.4-canary.20 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/dist/i18n/en.d.ts +9 -1
- package/dist/i18n/en.js +11 -2
- package/dist/i18n/zh-CN.d.ts +10 -2
- package/dist/i18n/zh-CN.js +11 -2
- package/dist/record/attempt-evidence.d.ts +1 -1
- package/dist/record/open.js +25 -0
- package/dist/report/assets/series-encoding.d.ts +36 -0
- package/dist/report/assets/series-encoding.js +104 -0
- package/dist/report/built-in/result-components.js +12 -2
- package/dist/report/components/attempt-detail/compute.d.ts +8 -1
- package/dist/report/components/attempt-detail/compute.js +19 -1
- package/dist/report/components/attempt-detail/index.js +25 -3
- package/dist/report/components/attempt-detail/validate.js +3 -2
- package/dist/report/components/entity-lists/compute.d.ts +6 -0
- package/dist/report/components/entity-lists/compute.js +55 -8
- package/dist/report/components/entity-lists/content.d.ts +2 -0
- package/dist/report/components/entity-lists/content.js +59 -6
- package/dist/report/components/entity-lists/index.d.ts +21 -1
- package/dist/report/components/entity-lists/index.js +40 -7
- package/dist/report/components/entity-lists/validate.js +21 -0
- package/dist/report/components/fixtures.js +2 -0
- package/dist/report/definition/cell.d.ts +22 -0
- package/dist/report/definition/cell.js +23 -9
- package/dist/report/definition/primitives/chart.js +125 -25
- package/dist/report/definition/primitives/conversation.d.ts +1 -1
- package/dist/report/definition/primitives/conversation.js +5 -3
- package/dist/report/definition/primitives.js +32 -7
- package/dist/report/definition/tree.d.ts +5 -2
- package/dist/report/definition/tree.js +1 -0
- package/dist/report/index.d.ts +5 -5
- package/dist/report/index.js +2 -2
- package/dist/report/model/aggregate.d.ts +14 -1
- package/dist/report/model/aggregate.js +34 -2
- package/dist/report/model/conversions.d.ts +1 -0
- package/dist/report/model/conversions.js +4 -1
- package/dist/report/model/format.d.ts +6 -5
- package/dist/report/model/format.js +30 -9
- package/dist/report/model/locale.d.ts +19 -3
- package/dist/report/model/locale.js +31 -5
- package/dist/report/model/types.d.ts +41 -4
- package/dist/report/presentation.d.ts +53 -6
- package/dist/report/presentation.js +40 -5
- package/dist/report/react/index.d.ts +3 -3
- package/dist/report/react/index.js +1 -1
- package/dist/report/slices/compute.js +52 -4
- package/dist/report/slices/content.d.ts +2 -1
- package/dist/report/slices/content.js +48 -15
- package/dist/report/slices/validate.js +23 -0
- package/dist/report/tasks.d.ts +7 -1
- package/dist/report/tasks.js +5 -3
- package/dist/runner/types.d.ts +30 -10
- package/dist/sample/index.d.ts +0 -9
- package/dist/sample/index.js +0 -18
- package/dist/sandbox/backend.d.ts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.d.ts +2 -1
- package/dist/sandbox/docker.d.ts +28 -7
- package/dist/sandbox/e2b.d.ts +11 -0
- package/dist/sandbox/identity.d.ts +6 -1
- package/dist/sandbox/layer.d.ts +36 -3
- package/dist/sandbox/local.d.ts +9 -0
- package/dist/sandbox/types.d.ts +14 -8
- package/dist/sandbox/vercel.d.ts +9 -0
- package/docs-site/zh/reference/builtin-agents.mdx +2 -0
- package/docs-site/zh/reference/cli.mdx +1 -0
- package/docs-site/zh/reference/define-agent.mdx +13 -9
- package/docs-site/zh/reference/define-config.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +1 -1
- package/docs-site/zh/reference/results-data.mdx +1 -1
- package/docs-site/zh/tutorials/deploy-report-site.mdx +157 -0
- package/docs-site/zh/tutorials/publish-report.mdx +1 -1
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-providers.mdx +19 -4
- package/package.json +1 -1
- package/src/agents/codex-env.test.ts +8 -1
- package/src/agents/codex.ts +8 -0
- package/src/agents/coding-cli-versions.ts +22 -6
- package/src/cli.ts +23 -1
- package/src/i18n/en.ts +14 -3
- package/src/i18n/zh-CN.ts +13 -2
- package/src/record/attempt-evidence.ts +1 -1
- package/src/record/open.ts +21 -0
- package/src/record/results.test.ts +74 -13
- package/src/record/truncate.ts +32 -7
- package/src/record/writer.ts +39 -8
- package/src/report/assets/series-encoding.tsx +177 -0
- package/src/report/assets/styles.css +97 -5
- package/src/report/built-in/result-components.tsx +23 -2
- package/src/report/components/attempt-detail/attempt-components.test.tsx +41 -0
- package/src/report/components/attempt-detail/compute.ts +20 -2
- package/src/report/components/attempt-detail/index.tsx +32 -2
- package/src/report/components/attempt-detail/validate.test.ts +1 -0
- package/src/report/components/attempt-detail/validate.tsx +3 -1
- package/src/report/components/compute.test.ts +196 -0
- package/src/report/components/entity-lists/compute.ts +73 -7
- package/src/report/components/entity-lists/content.test.ts +14 -8
- package/src/report/components/entity-lists/content.ts +63 -6
- package/src/report/components/entity-lists/index.tsx +93 -8
- package/src/report/components/entity-lists/validate.test.ts +9 -0
- package/src/report/components/entity-lists/validate.ts +16 -0
- package/src/report/components/fixtures.ts +2 -0
- package/src/report/definition/cell.ts +46 -11
- package/src/report/definition/conversation.test.tsx +5 -1
- package/src/report/definition/primitives/chart.tsx +197 -32
- package/src/report/definition/primitives/conversation.tsx +7 -4
- package/src/report/definition/primitives.tsx +81 -9
- package/src/report/definition/tree.ts +6 -2
- package/src/report/index.ts +15 -2
- package/src/report/model/aggregate.ts +40 -2
- package/src/report/model/conversions.ts +5 -0
- package/src/report/model/format.ts +32 -9
- package/src/report/model/locale.ts +35 -5
- package/src/report/model/presentation.test.ts +23 -1
- package/src/report/model/types.ts +40 -4
- package/src/report/presentation.test.tsx +179 -19
- package/src/report/presentation.ts +114 -11
- package/src/report/react/index.tsx +13 -2
- package/src/report/runtime/dual-render.test.tsx +38 -2
- package/src/report/slices/compute.ts +59 -4
- package/src/report/slices/content.ts +55 -16
- package/src/report/slices/delta-table.test.ts +138 -3
- package/src/report/slices/validate.test.ts +29 -0
- package/src/report/slices/validate.ts +16 -0
- package/src/report/tasks.ts +12 -2
- package/src/runner/accept.test.ts +328 -10
- package/src/runner/accept.ts +239 -72
- package/src/runner/attempt.test.ts +25 -13
- package/src/runner/attempt.ts +9 -9
- package/src/runner/config-identity.test.ts +25 -0
- package/src/runner/config-identity.ts +6 -7
- package/src/runner/feedback/coordinator.test.ts +51 -0
- package/src/runner/feedback/coordinator.ts +23 -0
- package/src/runner/feedback/failure.test.ts +15 -0
- package/src/runner/feedback/failure.ts +7 -0
- package/src/runner/feedback/human.test.ts +548 -27
- package/src/runner/feedback/human.ts +321 -47
- package/src/runner/feedback/index.ts +10 -0
- package/src/runner/feedback/input-guard.test.ts +251 -0
- package/src/runner/feedback/input-guard.ts +171 -0
- package/src/runner/feedback/reducer.ts +1 -0
- package/src/runner/fingerprint.test.ts +24 -0
- package/src/runner/gate-lease.test.ts +38 -14
- package/src/runner/ledger.test.ts +10 -7
- package/src/runner/ledger.ts +9 -2
- package/src/runner/lock.test.ts +40 -16
- package/src/runner/manifest.ts +8 -9
- package/src/runner/reporters/artifacts.ts +25 -3
- package/src/runner/run.test.ts +61 -0
- package/src/runner/run.ts +34 -9
- package/src/runner/sandbox-pool.test.ts +216 -12
- package/src/runner/sandbox-pool.ts +20 -2
- package/src/runner/types.ts +30 -10
- package/src/sample/index.ts +0 -16
- package/src/sample/sample.test.ts +78 -0
- package/src/sandbox/backend.ts +1 -1
- package/src/sandbox/commands.ts +4 -7
- package/src/sandbox/compose.test.ts +26 -1
- package/src/sandbox/compose.ts +7 -3
- package/src/sandbox/docker.test.ts +66 -1
- package/src/sandbox/docker.ts +89 -65
- package/src/sandbox/dockerfile-build.ts +4 -1
- package/src/sandbox/e2b-agent-template.test.ts +18 -1
- package/src/sandbox/e2b-agent-template.ts +22 -6
- package/src/sandbox/e2b.test.ts +64 -0
- package/src/sandbox/e2b.ts +27 -5
- package/src/sandbox/identity.ts +7 -2
- package/src/sandbox/layer.test.ts +23 -4
- package/src/sandbox/layer.ts +127 -43
- package/src/sandbox/link.test.ts +5 -2
- package/src/sandbox/local.test.ts +8 -8
- package/src/sandbox/local.ts +28 -6
- package/src/sandbox/operations.test.ts +4 -4
- package/src/sandbox/operations.ts +5 -5
- package/src/sandbox/runtime.ts +18 -3
- package/src/sandbox/types.ts +14 -8
- package/src/sandbox/vercel.test.ts +58 -3
- package/src/sandbox/vercel.ts +38 -6
- package/src/show/index.ts +11 -30
- package/src/show/render.test.ts +23 -1
- package/src/show/render.ts +6 -3
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// unit 驱动的内置格式化(docs/feature/reports/library/presentation.md):
|
|
2
2
|
// "%" → 87% "ms" → 1.2s "$" → $0.31 其余 → 1.2k 缩写(带 unit 后缀)
|
|
3
3
|
// MetricValue.format 可覆盖;公开入口是 formatMetricValue / formatAxisTick。
|
|
4
|
-
import { gapParts } from "../../sample/index.js";
|
|
5
4
|
import { DEFAULT_REPORT_LOCALE, DISPLAY_LOCALES, localeText, } from "./locale.js";
|
|
6
5
|
/**
|
|
7
6
|
* 一组 id 的显示名:每个 id 缩成在这组里唯一的最短路径后缀,重名逐步加长到能区分为止
|
|
@@ -145,6 +144,7 @@ const MISSING_CODE_KEYS = {
|
|
|
145
144
|
noSamples: "cell.missing",
|
|
146
145
|
notRun: "cell.notRun",
|
|
147
146
|
unscorable: "cell.unscorable",
|
|
147
|
+
noCurrentResult: "cell.noCurrentResult",
|
|
148
148
|
};
|
|
149
149
|
/**
|
|
150
150
|
* `missing` 格的本地化原因。code 是结构化代码,不是显示文本
|
|
@@ -260,16 +260,37 @@ export function formatReportDateTimeRange(fromIso, toIso, locale) {
|
|
|
260
260
|
to: formatReportDate(toDate, locale, toOptions),
|
|
261
261
|
};
|
|
262
262
|
}
|
|
263
|
-
// ── 实体列表(ExperimentList / EvalList / AttemptList)
|
|
263
|
+
// ── 实体列表(ExperimentList / EvalList / AttemptList)共用的相对时距 ──
|
|
264
|
+
/** 时距词表按区间取的单位;`{n}` 插值,en 是紧凑后缀,zh-CN 带空格接量词。 */
|
|
265
|
+
const TIME_DISTANCE_KEYS = {
|
|
266
|
+
minute: "timeDistance.minute",
|
|
267
|
+
hour: "timeDistance.hour",
|
|
268
|
+
day: "timeDistance.day",
|
|
269
|
+
month: "timeDistance.month",
|
|
270
|
+
};
|
|
271
|
+
const MS_PER_MINUTE = 60_000;
|
|
272
|
+
const MS_PER_HOUR = 3_600_000;
|
|
273
|
+
const MS_PER_DAY = 86_400_000;
|
|
274
|
+
const MS_PER_MONTH = MS_PER_DAY * 30;
|
|
264
275
|
/**
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
276
|
+
* 一段时长(毫秒) → 当前 locale 的紧凑相对时距("12d" / "12 天")。按区间选粒度最大的单位
|
|
277
|
+
* 取整,结果恒不小于一个单位(docs/feature/reports/library/presentation.md「相对时距是数据,
|
|
278
|
+
* 不是文案」):不足 1 小时→分钟,不足 1 天→小时,不足 30 天→天,30 天及以上→月。
|
|
279
|
+
* `locator` 格与占位行参考的 `staleSinceMs` 都经这一个入口,不各写各的措辞;不加箭头、回环
|
|
280
|
+
* 之类的装饰记号——记号不携带时长,读者还要先学会它。
|
|
269
281
|
*/
|
|
270
|
-
export function
|
|
271
|
-
const
|
|
272
|
-
|
|
282
|
+
export function formatTimeDistance(ms, locale = DEFAULT_REPORT_LOCALE) {
|
|
283
|
+
const abs = Math.max(0, ms);
|
|
284
|
+
if (abs < MS_PER_HOUR) {
|
|
285
|
+
return localeText(locale, TIME_DISTANCE_KEYS.minute, { n: Math.max(1, Math.round(abs / MS_PER_MINUTE)) });
|
|
286
|
+
}
|
|
287
|
+
if (abs < MS_PER_DAY) {
|
|
288
|
+
return localeText(locale, TIME_DISTANCE_KEYS.hour, { n: Math.max(1, Math.round(abs / MS_PER_HOUR)) });
|
|
289
|
+
}
|
|
290
|
+
if (abs < MS_PER_MONTH) {
|
|
291
|
+
return localeText(locale, TIME_DISTANCE_KEYS.day, { n: Math.max(1, Math.round(abs / MS_PER_DAY)) });
|
|
292
|
+
}
|
|
293
|
+
return localeText(locale, TIME_DISTANCE_KEYS.month, { n: Math.max(1, Math.round(abs / MS_PER_MONTH)) });
|
|
273
294
|
}
|
|
274
295
|
// ── 实体列表(ExperimentList / EvalList / AttemptList)共用的判定符 ──
|
|
275
296
|
/** passed / failed / errored / skipped 的判定符。 */
|
|
@@ -20,12 +20,19 @@ declare const en: {
|
|
|
20
20
|
readonly "verdict.skipped": "skipped";
|
|
21
21
|
/** AttemptSummary 的重试序号(0 起的 identity.attempt 显示前 +1)。 */
|
|
22
22
|
readonly "attemptSummary.attempt": "attempt {n}";
|
|
23
|
+
/** formatTimeDistance 的四区间读法(presentation.md「相对时距是数据,不是文案」)。 */
|
|
24
|
+
readonly "timeDistance.minute": "{n}m";
|
|
25
|
+
readonly "timeDistance.hour": "{n}h";
|
|
26
|
+
readonly "timeDistance.day": "{n}d";
|
|
27
|
+
readonly "timeDistance.month": "{n}mo";
|
|
23
28
|
/** 全 null / 无样本的统一文案,绝不画 0。 */
|
|
24
29
|
readonly "cell.missing": "no data";
|
|
25
30
|
/** missing 格 code=`notRun`:固定题集里这道题没有 attempt。 */
|
|
26
31
|
readonly "cell.notRun": "not run";
|
|
27
32
|
/** missing 格 code=`unscorable`:有 attempt,但读数测不出。 */
|
|
28
33
|
readonly "cell.unscorable": "unscorable";
|
|
34
|
+
/** missing 格 code=`noCurrentResult`:当前配置下这道题没有结果(覆盖缺口占位行)。 */
|
|
35
|
+
readonly "cell.noCurrentResult": "no result for current config";
|
|
29
36
|
readonly "cell.measuredTitle": "{samples}/{total} attempts measured";
|
|
30
37
|
readonly "cell.noneMeasurableTitle": "0/{total} attempts measurable";
|
|
31
38
|
readonly "cell.coverageTitle": "coverage {samples}/{total}: this metric is null for the remaining attempts";
|
|
@@ -136,10 +143,19 @@ declare const en: {
|
|
|
136
143
|
readonly "issues.details.one": "{n} warning";
|
|
137
144
|
readonly "issues.details.other": "{n} issues";
|
|
138
145
|
readonly "issues.badge.unfinishedSnapshot": "unfinished";
|
|
139
|
-
/** 覆盖缺口(scope.coverage)
|
|
146
|
+
/** 覆盖缺口(scope.coverage)。 */
|
|
140
147
|
readonly "overview.evalsCountPartial": "{covered}/{total} evals";
|
|
141
|
-
|
|
142
|
-
readonly "
|
|
148
|
+
/** 覆盖构成四段的段名(docs/feature/reports/components/summaries/experiment-table.md「覆盖构成」)。 */
|
|
149
|
+
readonly "coverage.fresh": "fresh";
|
|
150
|
+
readonly "coverage.historical": "historical";
|
|
151
|
+
readonly "coverage.stale": "stale";
|
|
152
|
+
readonly "coverage.notRun": "not run";
|
|
153
|
+
/** locator 格历史执行时距的 hover 说明(三条通道见「时效不写字」)。 */
|
|
154
|
+
readonly "experimentList.historicalTooltip": "Historical execution — carried over or pulled in from an earlier run; still counts toward every total.";
|
|
155
|
+
/** 覆盖缺口占位行参考的 hover 说明。 */
|
|
156
|
+
readonly "experimentList.staleReferenceTooltip": "Reference only — not comparable with the current configuration, does not count toward any total.";
|
|
157
|
+
/** 「只看新执行」开关的 web 面文案(experiment-table.md「只看新执行」)。 */
|
|
158
|
+
readonly "experimentTable.freshOnlyToggle": "Fresh executions only";
|
|
143
159
|
/** Hero / HeroCard 的运行 meta(hero.noRuns 是 latestStartedAt 为 null 时的内置文案)。 */
|
|
144
160
|
readonly "hero.lastRun": "Last run {time}";
|
|
145
161
|
readonly "hero.noRuns": "No runs yet";
|
|
@@ -46,12 +46,19 @@ const en = {
|
|
|
46
46
|
"verdict.skipped": "skipped",
|
|
47
47
|
/** AttemptSummary 的重试序号(0 起的 identity.attempt 显示前 +1)。 */
|
|
48
48
|
"attemptSummary.attempt": "attempt {n}",
|
|
49
|
+
/** formatTimeDistance 的四区间读法(presentation.md「相对时距是数据,不是文案」)。 */
|
|
50
|
+
"timeDistance.minute": "{n}m",
|
|
51
|
+
"timeDistance.hour": "{n}h",
|
|
52
|
+
"timeDistance.day": "{n}d",
|
|
53
|
+
"timeDistance.month": "{n}mo",
|
|
49
54
|
/** 全 null / 无样本的统一文案,绝不画 0。 */
|
|
50
55
|
"cell.missing": "no data",
|
|
51
56
|
/** missing 格 code=`notRun`:固定题集里这道题没有 attempt。 */
|
|
52
57
|
"cell.notRun": "not run",
|
|
53
58
|
/** missing 格 code=`unscorable`:有 attempt,但读数测不出。 */
|
|
54
59
|
"cell.unscorable": "unscorable",
|
|
60
|
+
/** missing 格 code=`noCurrentResult`:当前配置下这道题没有结果(覆盖缺口占位行)。 */
|
|
61
|
+
"cell.noCurrentResult": "no result for current config",
|
|
55
62
|
"cell.measuredTitle": "{samples}/{total} attempts measured",
|
|
56
63
|
"cell.noneMeasurableTitle": "0/{total} attempts measurable",
|
|
57
64
|
"cell.coverageTitle": "coverage {samples}/{total}: this metric is null for the remaining attempts",
|
|
@@ -162,10 +169,19 @@ const en = {
|
|
|
162
169
|
"issues.details.one": "{n} warning",
|
|
163
170
|
"issues.details.other": "{n} issues",
|
|
164
171
|
"issues.badge.unfinishedSnapshot": "unfinished",
|
|
165
|
-
/** 覆盖缺口(scope.coverage)
|
|
172
|
+
/** 覆盖缺口(scope.coverage)。 */
|
|
166
173
|
"overview.evalsCountPartial": "{covered}/{total} evals",
|
|
167
|
-
|
|
168
|
-
"
|
|
174
|
+
/** 覆盖构成四段的段名(docs/feature/reports/components/summaries/experiment-table.md「覆盖构成」)。 */
|
|
175
|
+
"coverage.fresh": "fresh",
|
|
176
|
+
"coverage.historical": "historical",
|
|
177
|
+
"coverage.stale": "stale",
|
|
178
|
+
"coverage.notRun": "not run",
|
|
179
|
+
/** locator 格历史执行时距的 hover 说明(三条通道见「时效不写字」)。 */
|
|
180
|
+
"experimentList.historicalTooltip": "Historical execution — carried over or pulled in from an earlier run; still counts toward every total.",
|
|
181
|
+
/** 覆盖缺口占位行参考的 hover 说明。 */
|
|
182
|
+
"experimentList.staleReferenceTooltip": "Reference only — not comparable with the current configuration, does not count toward any total.",
|
|
183
|
+
/** 「只看新执行」开关的 web 面文案(experiment-table.md「只看新执行」)。 */
|
|
184
|
+
"experimentTable.freshOnlyToggle": "Fresh executions only",
|
|
169
185
|
/** Hero / HeroCard 的运行 meta(hero.noRuns 是 latestStartedAt 为 null 时的内置文案)。 */
|
|
170
186
|
"hero.lastRun": "Last run {time}",
|
|
171
187
|
"hero.noRuns": "No runs yet",
|
|
@@ -218,9 +234,14 @@ const zhCN = {
|
|
|
218
234
|
"verdict.errored": "错误",
|
|
219
235
|
"verdict.skipped": "跳过",
|
|
220
236
|
"attemptSummary.attempt": "第 {n} 次",
|
|
237
|
+
"timeDistance.minute": "{n} 分钟",
|
|
238
|
+
"timeDistance.hour": "{n} 小时",
|
|
239
|
+
"timeDistance.day": "{n} 天",
|
|
240
|
+
"timeDistance.month": "{n} 个月",
|
|
221
241
|
"cell.missing": "无数据",
|
|
222
242
|
"cell.notRun": "未跑到",
|
|
223
243
|
"cell.unscorable": "测不出",
|
|
244
|
+
"cell.noCurrentResult": "当前配置下无结果",
|
|
224
245
|
"cell.measuredTitle": "{samples}/{total} 次 attempt 测得",
|
|
225
246
|
"cell.noneMeasurableTitle": "0/{total} 次 attempt 可测",
|
|
226
247
|
"cell.coverageTitle": "覆盖率 {samples}/{total}:其余 attempt 测不了这个指标",
|
|
@@ -321,8 +342,13 @@ const zhCN = {
|
|
|
321
342
|
"issues.details.other": "{n} 条原始警告",
|
|
322
343
|
"issues.badge.unfinishedSnapshot": "未收尾",
|
|
323
344
|
"overview.evalsCountPartial": "{covered}/{total} 个 Eval",
|
|
324
|
-
"
|
|
325
|
-
"
|
|
345
|
+
"coverage.fresh": "新执行",
|
|
346
|
+
"coverage.historical": "历史执行",
|
|
347
|
+
"coverage.stale": "过期结论",
|
|
348
|
+
"coverage.notRun": "未跑到",
|
|
349
|
+
"experimentList.historicalTooltip": "历史执行——携带自上一轮或跨 Run 拼入的结果;仍计入所有汇总。",
|
|
350
|
+
"experimentList.staleReferenceTooltip": "仅作参考——与当前配置不可比,不进任何计数。",
|
|
351
|
+
"experimentTable.freshOnlyToggle": "只看新执行",
|
|
326
352
|
"hero.lastRun": "最后运行 {time}",
|
|
327
353
|
"hero.noRuns": "暂无运行",
|
|
328
354
|
"hero.composedRuns": "由 {n} 次运行合成",
|
|
@@ -240,8 +240,10 @@ export interface DeltaCell {
|
|
|
240
240
|
attempts: readonly AttemptLocator[];
|
|
241
241
|
totalTokens?: number;
|
|
242
242
|
totalCostUSD?: number;
|
|
243
|
-
/** true
|
|
243
|
+
/** true 时该格来自跨快照携带的历史执行(时效标注见 experiment-table.md「时效不写字」)。 */
|
|
244
244
|
historical: boolean;
|
|
245
|
+
/** `historical` 为 true 时,距今的毫秒数(渲染时刻起算);新执行时省略,不伪造 0。 */
|
|
246
|
+
staleSinceMs?: number;
|
|
245
247
|
}
|
|
246
248
|
export interface DeltaData {
|
|
247
249
|
byDimension: string;
|
|
@@ -262,6 +264,12 @@ export interface DeltaData {
|
|
|
262
264
|
tokens?: number;
|
|
263
265
|
costUSD?: number;
|
|
264
266
|
}>;
|
|
267
|
+
/**
|
|
268
|
+
* 键是条件值,只在该条件缺席这道题(`cells` 无键)且记录里存在不可比历史判定时才有条目
|
|
269
|
+
* (docs/feature/reports/show/compare.md「— ✓ 12d」);它不进 `totals`、`delta` 与配对覆盖的
|
|
270
|
+
* 任何一个数,只提供 locator 下钻。
|
|
271
|
+
*/
|
|
272
|
+
references?: globalThis.Record<string, StaleConclusionReference>;
|
|
265
273
|
}>;
|
|
266
274
|
/** 各条件自身覆盖面的描述,分母是该条件有结果的 eval 数;不用于跨条件直接归因。 */
|
|
267
275
|
totals: globalThis.Record<string, {
|
|
@@ -463,8 +471,20 @@ export interface AttemptListItem {
|
|
|
463
471
|
startedAt: string;
|
|
464
472
|
/** 历史执行:携带条目,或来自该实验在 Sample 中最新快照之外的快照;false = 最新一次运行实测。 */
|
|
465
473
|
historical: boolean;
|
|
474
|
+
/** `historical` 为 true 时,距今的毫秒数(渲染时刻起算);新执行时省略,不伪造 0。 */
|
|
475
|
+
staleSinceMs?: number;
|
|
466
476
|
locator: AttemptLocator;
|
|
467
477
|
}
|
|
478
|
+
/**
|
|
479
|
+
* 覆盖缺口占位行的参考:记录里存在、但与当前基准 configHash 不可比的最近一条判定
|
|
480
|
+
* (docs/feature/reports/components/summaries/experiment-table.md「覆盖缺口的两档占位行」)。
|
|
481
|
+
* 它不进任何计数,只提供下钻入口。
|
|
482
|
+
*/
|
|
483
|
+
export interface StaleConclusionReference {
|
|
484
|
+
readonly locator: AttemptLocator;
|
|
485
|
+
readonly verdict: Verdict | "skipped";
|
|
486
|
+
readonly staleSinceMs: number;
|
|
487
|
+
}
|
|
468
488
|
/**
|
|
469
489
|
* `EvalList` 一项 = 一个 `experimentId + evalId`(同一个 Eval 跑在两个 experiment 上是
|
|
470
490
|
* 两条不同结果,不合并)。失败原因只存在于各 AttemptListItem,不在 Eval 父项重复一份。
|
|
@@ -523,10 +543,16 @@ export interface ExperimentListItem {
|
|
|
523
543
|
evals: number;
|
|
524
544
|
/** 这个 experiment 覆盖的 attempt 总数(原始计数,含多轮重试)。 */
|
|
525
545
|
attempts: number;
|
|
526
|
-
/** 历史执行的 attempt 数(分母是 attempts)
|
|
546
|
+
/** 历史执行的 attempt 数(分母是 attempts)。 */
|
|
527
547
|
historicalAttempts: number;
|
|
528
548
|
/** 已知 eval 并集里、当前口径下没有任何 attempt 的题(来自 `scope.coverage`);渲染为占位行。 */
|
|
529
549
|
missingEvalIds: string[];
|
|
550
|
+
/**
|
|
551
|
+
* `missingEvalIds` 里、记录中存在不可比历史判定的那些题的参考(evalId → 参考);
|
|
552
|
+
* 其余缺口题没有条目,占位行只给「当前配置下无结果」。`sample.fresh` 为 true 时整份为空——
|
|
553
|
+
* 读者已声明只看新执行,占位行就不再把被排除的历史结论请回来。
|
|
554
|
+
*/
|
|
555
|
+
staleReferences: Readonly<globalThis.Record<string, StaleConclusionReference>>;
|
|
530
556
|
/** 所含快照中最近的 startedAt。 */
|
|
531
557
|
lastRunAt: string;
|
|
532
558
|
evalRows: ExperimentListEvalRow[];
|
|
@@ -685,8 +711,8 @@ export interface AttemptConversationData {
|
|
|
685
711
|
/** 独立的生命周期命令证据项;不属于 Conversation 的轮次或消息。 */
|
|
686
712
|
export interface AttemptCommandEvidence extends CommandExitEvidence {
|
|
687
713
|
key: string;
|
|
688
|
-
/** 消费层由 checked +
|
|
689
|
-
classification: "observed" | "failed";
|
|
714
|
+
/** 消费层由 checked + exitCode 推导,不落盘:exitCode 为 0 是 "succeeded";非零时 checked 才是 "failed",否则是 "observed"。 */
|
|
715
|
+
classification: "succeeded" | "observed" | "failed";
|
|
690
716
|
/** 从关联 timing node 派生,缺少 timing 时省略。 */
|
|
691
717
|
durationMs?: number;
|
|
692
718
|
}
|
|
@@ -729,6 +755,17 @@ export interface UsageTableData {
|
|
|
729
755
|
usage?: Usage;
|
|
730
756
|
estimatedCostUSD?: number;
|
|
731
757
|
}
|
|
758
|
+
/**
|
|
759
|
+
* `AttemptFacts` 的 data:attempt 作用域 `ctx.fact()` 上报的运行事实完整键值表
|
|
760
|
+
* (见 docs/feature/record/architecture.md#facts运行事实),按落盘的 key 插入顺序排列。
|
|
761
|
+
* `AttemptRecord.facts` 缺失或为空对象时整个 data 为 null,不渲染空表。
|
|
762
|
+
*/
|
|
763
|
+
export interface AttemptFactsData {
|
|
764
|
+
facts: {
|
|
765
|
+
key: string;
|
|
766
|
+
value: string | number | boolean;
|
|
767
|
+
}[];
|
|
768
|
+
}
|
|
732
769
|
/** `AttemptTrace` 的 data:不与 runner 节点合并的原始 OTel span 列表;没有 trace 时 null。 */
|
|
733
770
|
export interface AttemptTraceData {
|
|
734
771
|
/** text 面拼 `niceeval show <locator> --timing` 下钻命令用;web 面不需要。 */
|
|
@@ -18,21 +18,68 @@ export type DimensionDeclarations = Readonly<Record<string, DimensionDeclaration
|
|
|
18
18
|
export type DimensionPins = Readonly<Record<string, Readonly<Record<string, number>>>>;
|
|
19
19
|
export declare const VISUAL_SLOT_COUNT = 24;
|
|
20
20
|
export type SeriesVariant = 1 | 2 | 3 | 4;
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* 槽 1..24 → (色板下标 1..6, 形状变体 1..4)。
|
|
23
|
+
* 与 components/README「视觉编码容量」槽序表一致:1–6 第一变体、7–12 第二变体,依此类推。
|
|
24
|
+
* `colorIndex = (slot-1) % 6 + 1`,`variant = floor((slot-1)/6) + 1`。
|
|
25
|
+
*/
|
|
22
26
|
export declare function seriesChannelsOf(slot: number): {
|
|
23
27
|
colorIndex: number;
|
|
24
28
|
variant: SeriesVariant;
|
|
25
29
|
};
|
|
26
|
-
export interface
|
|
30
|
+
export interface PresentationIdentity {
|
|
31
|
+
/** 完整维度值,作为排序、筛选、React key 与证据身份。 */
|
|
27
32
|
readonly value: string;
|
|
33
|
+
/** 当前页完整 label keyset 内生成的显示名。 */
|
|
28
34
|
readonly label: string;
|
|
29
|
-
readonly seriesSlot?: number;
|
|
30
|
-
readonly colorIndex?: number;
|
|
31
|
-
readonly variant?: SeriesVariant;
|
|
32
35
|
}
|
|
36
|
+
export interface LabelPresentation extends PresentationIdentity {
|
|
37
|
+
readonly kind: "label";
|
|
38
|
+
}
|
|
39
|
+
export interface ColorPresentation extends PresentationIdentity {
|
|
40
|
+
readonly kind: "color";
|
|
41
|
+
/** `var(--niceeval-color-series-N)`。 */
|
|
42
|
+
readonly color: string;
|
|
43
|
+
}
|
|
44
|
+
export interface LineSeriesPresentation extends PresentationIdentity {
|
|
45
|
+
readonly kind: "series";
|
|
46
|
+
readonly mark: "line";
|
|
47
|
+
readonly stroke: string;
|
|
48
|
+
readonly strokeDasharray: string;
|
|
49
|
+
readonly marker: {
|
|
50
|
+
readonly path: string;
|
|
51
|
+
readonly viewBox: string;
|
|
52
|
+
readonly fill: string;
|
|
53
|
+
readonly stroke: string;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
export interface ScatterSeriesPresentation extends PresentationIdentity {
|
|
57
|
+
readonly kind: "series";
|
|
58
|
+
readonly mark: "scatter";
|
|
59
|
+
readonly marker: LineSeriesPresentation["marker"];
|
|
60
|
+
}
|
|
61
|
+
export interface FillSeriesPresentation extends PresentationIdentity {
|
|
62
|
+
readonly kind: "series";
|
|
63
|
+
readonly mark: "bar" | "area";
|
|
64
|
+
/** 颜色,或可直接使用的 `url(#pattern-id)`。 */
|
|
65
|
+
readonly fill: string;
|
|
66
|
+
readonly stroke: string;
|
|
67
|
+
readonly strokeDasharray: string;
|
|
68
|
+
}
|
|
69
|
+
export type DimensionPresentation = LabelPresentation | ColorPresentation | LineSeriesPresentation | ScatterSeriesPresentation | FillSeriesPresentation;
|
|
70
|
+
export type PresentationFor<E extends DimensionEncoding> = E extends {
|
|
71
|
+
kind: "label";
|
|
72
|
+
} ? LabelPresentation : E extends {
|
|
73
|
+
kind: "color";
|
|
74
|
+
} ? ColorPresentation : LineSeriesPresentation | ScatterSeriesPresentation | FillSeriesPresentation;
|
|
75
|
+
/**
|
|
76
|
+
* 句柄上的呈现面。`at(i)` 返回可直接交给 SVG/CSS 的值;text 面恒为 label 支。
|
|
77
|
+
* `labels` 是本维度整页 label keyset 的最短唯一后缀表,便于探针与调试。
|
|
78
|
+
*/
|
|
33
79
|
export interface PresentedDimension {
|
|
80
|
+
readonly length: number;
|
|
34
81
|
readonly labels: ReadonlyMap<string, string>;
|
|
35
|
-
at(index: number):
|
|
82
|
+
at(index: number): DimensionPresentation;
|
|
36
83
|
}
|
|
37
84
|
export declare class UndeclaredDimensionValueError extends Error {
|
|
38
85
|
readonly handle: string;
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { shortestUniqueLabels as labels } from "./model/format.js";
|
|
2
|
+
import { seriesColorVar, seriesFill, seriesMarker, seriesStrokeDasharray, } from "./assets/series-encoding.js";
|
|
2
3
|
export const VISUAL_SLOT_COUNT = 24;
|
|
3
|
-
/**
|
|
4
|
+
/**
|
|
5
|
+
* 槽 1..24 → (色板下标 1..6, 形状变体 1..4)。
|
|
6
|
+
* 与 components/README「视觉编码容量」槽序表一致:1–6 第一变体、7–12 第二变体,依此类推。
|
|
7
|
+
* `colorIndex = (slot-1) % 6 + 1`,`variant = floor((slot-1)/6) + 1`。
|
|
8
|
+
*/
|
|
4
9
|
export function seriesChannelsOf(slot) {
|
|
5
10
|
if (!Number.isInteger(slot) || slot < 1 || slot > VISUAL_SLOT_COUNT) {
|
|
6
11
|
throw new Error(`seriesSlot must be an integer in [1, ${VISUAL_SLOT_COUNT}], got ${slot}.`);
|
|
@@ -8,7 +13,7 @@ export function seriesChannelsOf(slot) {
|
|
|
8
13
|
const zero = slot - 1;
|
|
9
14
|
return {
|
|
10
15
|
colorIndex: (zero % 6) + 1,
|
|
11
|
-
variant: (
|
|
16
|
+
variant: (Math.floor(zero / 6) + 1),
|
|
12
17
|
};
|
|
13
18
|
}
|
|
14
19
|
export class UndeclaredDimensionValueError extends Error {
|
|
@@ -33,6 +38,35 @@ export function presentDimension(declaration) {
|
|
|
33
38
|
return presented;
|
|
34
39
|
}
|
|
35
40
|
export const shortestUniqueLabels = labels;
|
|
41
|
+
function presentChannels(value, label, seriesSlot, encoding) {
|
|
42
|
+
const { colorIndex, variant } = seriesChannelsOf(seriesSlot);
|
|
43
|
+
if (encoding.kind === "color") {
|
|
44
|
+
return { kind: "color", value, label, color: seriesColorVar(colorIndex) };
|
|
45
|
+
}
|
|
46
|
+
if (encoding.kind !== "series") {
|
|
47
|
+
return { kind: "label", value, label };
|
|
48
|
+
}
|
|
49
|
+
const stroke = seriesColorVar(colorIndex);
|
|
50
|
+
const strokeDasharray = seriesStrokeDasharray(variant);
|
|
51
|
+
const marker = seriesMarker(colorIndex, variant);
|
|
52
|
+
switch (encoding.mark) {
|
|
53
|
+
case "line":
|
|
54
|
+
return { kind: "series", mark: "line", value, label, stroke, strokeDasharray, marker };
|
|
55
|
+
case "scatter":
|
|
56
|
+
return { kind: "series", mark: "scatter", value, label, marker };
|
|
57
|
+
case "bar":
|
|
58
|
+
case "area":
|
|
59
|
+
return {
|
|
60
|
+
kind: "series",
|
|
61
|
+
mark: encoding.mark,
|
|
62
|
+
value,
|
|
63
|
+
label,
|
|
64
|
+
fill: seriesFill(colorIndex, variant),
|
|
65
|
+
stroke,
|
|
66
|
+
strokeDasharray,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
}
|
|
36
70
|
/**
|
|
37
71
|
* 页级呈现分配:收集整页 dimensions() 声明,产出 label / visual 两套 keyset 上的映射。
|
|
38
72
|
* pins 原样占位;未钉键以稳定散列为起点、撞槽按显示键字典序线性探测。
|
|
@@ -79,6 +113,7 @@ export function allocatePageDimensions(handles, pins = {}, options = { face: "we
|
|
|
79
113
|
const values = declaration.values;
|
|
80
114
|
const encoding = declaration.encoding;
|
|
81
115
|
byHandle.set(handle, {
|
|
116
|
+
length: values.length,
|
|
82
117
|
labels: dimLabels,
|
|
83
118
|
at(index) {
|
|
84
119
|
const value = values[index];
|
|
@@ -86,15 +121,15 @@ export function allocatePageDimensions(handles, pins = {}, options = { face: "we
|
|
|
86
121
|
throw new UndeclaredDimensionValueError(`dimension handle "${handle}" has no value at index ${index} (declared length ${values.length}).`, handle, index);
|
|
87
122
|
}
|
|
88
123
|
const label = dimLabels.get(value) ?? value;
|
|
124
|
+
// text 面恒返回 label 支:不上 ANSI 色,拿不到颜色、线型或 pattern。
|
|
89
125
|
if (options.face === "text" || encoding.kind === "label") {
|
|
90
|
-
return { value, label };
|
|
126
|
+
return { kind: "label", value, label };
|
|
91
127
|
}
|
|
92
128
|
const seriesSlot = slots?.get(value);
|
|
93
129
|
if (seriesSlot === undefined) {
|
|
94
130
|
throw new UndeclaredDimensionValueError(`dimension handle "${handle}" value ${JSON.stringify(value)} has no seriesSlot on this page.`, handle, index);
|
|
95
131
|
}
|
|
96
|
-
|
|
97
|
-
return { value, label, seriesSlot, colorIndex: channels.colorIndex, variant: channels.variant };
|
|
132
|
+
return presentChannels(value, label, seriesSlot, encoding);
|
|
98
133
|
},
|
|
99
134
|
});
|
|
100
135
|
}
|
|
@@ -2,12 +2,12 @@ export { Callouts, Chart, Col, Conversation, CopyBlock, DiffView, Grid, Row, Sec
|
|
|
2
2
|
export { HeroCard } from "../components/site-components/HeroCard.tsx";
|
|
3
3
|
export type { HeroLink, HeroLogo } from "../components/site-components/hero-types.ts";
|
|
4
4
|
export { PoweredBy } from "../components/site-components/PoweredBy.tsx";
|
|
5
|
-
export type { AttemptAssertionsData, AttemptConversationData, AttemptConversationReply, AttemptConversationRound, AttemptDiagnosticsData, AttemptDiffData, AttemptErrorData, AttemptFixPromptData, AttemptListItem, AttemptLocator, AttemptSummaryData, AttemptTimelineData, AttemptTraceData, CopyFixPromptData, Dataset, DatasetField, DatasetRow, EvalListItem, ExperimentListEvalRow, ExperimentListItem, HeroData, MetricColumn, MetricValue, SampleSummaryContent, SampleIssue, SnapshotDiagnosticsData, SnapshotDiagnosticsItem, TraceSpanSummary, TraceWaterfallRow, UsageTableData, VerdictTally, } from "../model/types.ts";
|
|
5
|
+
export type { AttemptAssertionsData, AttemptConversationData, AttemptConversationReply, AttemptConversationRound, AttemptDiagnosticsData, AttemptDiffData, AttemptErrorData, AttemptFixPromptData, AttemptListItem, AttemptLocator, AttemptSummaryData, AttemptTimelineData, AttemptTraceData, CopyFixPromptData, Dataset, DatasetField, DatasetRow, EvalListItem, ExperimentListEvalRow, ExperimentListItem, HeroData, MetricColumn, MetricValue, SampleSummaryContent, SampleIssue, SnapshotDiagnosticsData, SnapshotDiagnosticsItem, StaleConclusionReference, TraceSpanSummary, TraceWaterfallRow, UsageTableData, VerdictTally, } from "../model/types.ts";
|
|
6
6
|
export type { Cell, VerdictCounts } from "../definition/cell.ts";
|
|
7
7
|
export { formatCellText } from "../definition/cell.ts";
|
|
8
8
|
export type { AttemptEvidence, AttemptEvidenceCapabilities } from "../../record/attempt-evidence.ts";
|
|
9
|
-
export { formatAxisTick, formatInstant, formatMetricValue, missingText } from "../model/format.ts";
|
|
9
|
+
export { formatAxisTick, formatInstant, formatMetricValue, formatTimeDistance, missingText } from "../model/format.ts";
|
|
10
10
|
export { presentDimension, shortestUniqueLabels } from "../presentation.ts";
|
|
11
|
-
export type { DimensionDeclaration, DimensionEncoding, PresentedDimension } from "../presentation.ts";
|
|
11
|
+
export type { ColorPresentation, DimensionDeclaration, DimensionEncoding, DimensionPresentation, FillSeriesPresentation, LabelPresentation, LineSeriesPresentation, PresentedDimension, ScatterSeriesPresentation, } from "../presentation.ts";
|
|
12
12
|
export { DEFAULT_REPORT_LOCALE, localizedTextEquals, resolveLocalizedText, resolveMetricLabel, } from "../model/locale.ts";
|
|
13
13
|
export type { LocalizedText, ReportLocale } from "../model/locale.ts";
|
|
@@ -12,7 +12,7 @@ export { HeroCard } from "../components/site-components/HeroCard.js";
|
|
|
12
12
|
export { PoweredBy } from "../components/site-components/PoweredBy.js";
|
|
13
13
|
export { formatCellText } from "../definition/cell.js";
|
|
14
14
|
// 格式化与呈现工具箱(docs/feature/reports/library/presentation.md):自有 React 页面与报告面同实现。
|
|
15
|
-
export { formatAxisTick, formatInstant, formatMetricValue, missingText } from "../model/format.js";
|
|
15
|
+
export { formatAxisTick, formatInstant, formatMetricValue, formatTimeDistance, missingText } from "../model/format.js";
|
|
16
16
|
export { presentDimension, shortestUniqueLabels } from "../presentation.js";
|
|
17
17
|
// locale(官方组件 chrome 文案的语言;LocalizedText 的按 locale 解析也用它)
|
|
18
18
|
export { DEFAULT_REPORT_LOCALE, localizedTextEquals, resolveLocalizedText, resolveMetricLabel, } from "../model/locale.js";
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
|
|
13
13
|
import { comparabilityConfigOf, deepEqualJson } from "../../sample/index.js";
|
|
14
14
|
import { foldEvalVerdict } from "../../shared/verdict.js";
|
|
15
|
-
import { assertUniqueMetricNames, axisValueOf, collectItems, computeCell, dimensionKey, dimensionName, evalGroupOf, evalIdOf, evaluateMetric, experimentIdOf, filterItems, fullEvalKey, groupItems, historicalOf, locatorOf, refDisplayKey, resolveInput, seriesKey, seriesName, toColumn, } from "../model/aggregate.js";
|
|
15
|
+
import { assertUniqueMetricNames, axisValueOf, collectItems, computeCell, dimensionKey, dimensionName, evalGroupOf, evalIdOf, evaluateMetric, experimentIdOf, filterItems, fullEvalKey, groupItems, historicalOf, locatorOf, msSince, refDisplayKey, resolveInput, seriesKey, seriesName, staleReferenceOf, toColumn, } from "../model/aggregate.js";
|
|
16
16
|
import { costUSD as costUSDMetric, examScore, tokens as tokensMetric, totalScore as totalScoreMetric } from "../model/metrics.js";
|
|
17
17
|
import { formatMetricValue, formatPlainNumber } from "../model/format.js";
|
|
18
18
|
import { metricFieldOf } from "../model/dataset.js";
|
|
@@ -420,6 +420,10 @@ async function buildDeltaCell(items) {
|
|
|
420
420
|
const verdict = foldEvalVerdict(items.map((item) => ({ verdict: item.attempt.result.verdict })));
|
|
421
421
|
const refs = new Set();
|
|
422
422
|
let historical = false;
|
|
423
|
+
// 距今最久的历史执行决定这一格的时距——多个 attempt 折进同一格时,时距说的是「这批结论里
|
|
424
|
+
// 最旧的证据有多旧」,不是最新那条(与 experiment-table 的 attempt 行时距同一份数据来源,
|
|
425
|
+
// 只是这里先跨 attempt 折叠成一格)。
|
|
426
|
+
let staleStartedAt;
|
|
423
427
|
let scoreSum = 0;
|
|
424
428
|
let scoreCount = 0;
|
|
425
429
|
let tokensSum = 0;
|
|
@@ -428,8 +432,12 @@ async function buildDeltaCell(items) {
|
|
|
428
432
|
let costCount = 0;
|
|
429
433
|
for (const item of items) {
|
|
430
434
|
refs.add(locatorOf(item));
|
|
431
|
-
if (historicalOf(item))
|
|
435
|
+
if (historicalOf(item)) {
|
|
432
436
|
historical = true;
|
|
437
|
+
const startedAt = item.attempt.result.startedAt ?? item.run.startedAt;
|
|
438
|
+
if (staleStartedAt === undefined || startedAt < staleStartedAt)
|
|
439
|
+
staleStartedAt = startedAt;
|
|
440
|
+
}
|
|
433
441
|
if (evaluationKind === "points") {
|
|
434
442
|
const value = await evaluateMetric(totalScoreMetric, item.attempt);
|
|
435
443
|
if (value !== null) {
|
|
@@ -458,10 +466,21 @@ async function buildDeltaCell(items) {
|
|
|
458
466
|
...(tokensCount > 0 ? { totalTokens: tokensSum } : {}),
|
|
459
467
|
...(costCount > 0 ? { totalCostUSD: costSum } : {}),
|
|
460
468
|
historical,
|
|
469
|
+
...(staleStartedAt !== undefined ? { staleSinceMs: msSince(staleStartedAt) } : {}),
|
|
461
470
|
};
|
|
462
471
|
}
|
|
472
|
+
/** 每个 experiment 当前基准的 configHash:该 experiment 在 `runs` 里最新快照的 configHash。 */
|
|
473
|
+
function latestConfigHashByExperiment(runs) {
|
|
474
|
+
const latest = new Map();
|
|
475
|
+
for (const run of runs) {
|
|
476
|
+
const existing = latest.get(run.experimentId);
|
|
477
|
+
if (existing === undefined || run.startedAt > existing.startedAt)
|
|
478
|
+
latest.set(run.experimentId, run);
|
|
479
|
+
}
|
|
480
|
+
return new Map([...latest].map(([id, run]) => [id, run.configHash]));
|
|
481
|
+
}
|
|
463
482
|
export async function deltaTableData(input, options) {
|
|
464
|
-
const { runs, attempts } = resolveInput(input);
|
|
483
|
+
const { runs, attempts, historyAttempts, fresh } = resolveInput(input);
|
|
465
484
|
const items = filterItems(collectItems(runs, attempts), options.evals);
|
|
466
485
|
let conditions;
|
|
467
486
|
let experiments;
|
|
@@ -526,6 +545,13 @@ export async function deltaTableData(input, options) {
|
|
|
526
545
|
for (const evalId of byEval.keys())
|
|
527
546
|
evalIdsSet.add(evalId);
|
|
528
547
|
const knownEvalIds = [...evalIdsSet].sort();
|
|
548
|
+
// 每个条件背后的 experiment id 集合(by: "experiment" 时条件字符串本身通常就是唯一一个 id);
|
|
549
|
+
// 过期结论参考按各自 experiment 的当前基准 configHash 判定。
|
|
550
|
+
const experimentIdsByCondition = new Map();
|
|
551
|
+
for (const [condition, conditionItems] of itemsByCondition) {
|
|
552
|
+
experimentIdsByCondition.set(condition, new Set(conditionItems.map((item) => experimentIdOf(item))));
|
|
553
|
+
}
|
|
554
|
+
const anchorConfigHashByExperiment = latestConfigHashByExperiment(runs);
|
|
529
555
|
const baseline = conditions[0];
|
|
530
556
|
const rows = [];
|
|
531
557
|
for (const evalId of knownEvalIds) {
|
|
@@ -558,7 +584,29 @@ export async function deltaTableData(input, options) {
|
|
|
558
584
|
}
|
|
559
585
|
}
|
|
560
586
|
}
|
|
561
|
-
|
|
587
|
+
// 过期结论参考(show/compare.md「— ✓ 12d」):只在该条件缺席这道题、且记录里存在与该
|
|
588
|
+
// experiment 当前基准不可比的历史判定时才给,口径与 experiment-table 两档占位行一致——
|
|
589
|
+
// `fresh` 为 true 时不给参考,读者已声明只看新执行。它只写进 references,不进 cells,
|
|
590
|
+
// 因此天然不参与 totals / delta / 配对覆盖的任何一个数。
|
|
591
|
+
let references;
|
|
592
|
+
if (!fresh) {
|
|
593
|
+
for (const condition of conditions) {
|
|
594
|
+
if (cells[condition] !== undefined)
|
|
595
|
+
continue;
|
|
596
|
+
const experimentIds = experimentIdsByCondition.get(condition);
|
|
597
|
+
if (!experimentIds || experimentIds.size === 0)
|
|
598
|
+
continue;
|
|
599
|
+
const candidates = historyAttempts.filter((a) => experimentIds.has(a.experimentId) &&
|
|
600
|
+
a.evalId === evalId &&
|
|
601
|
+
a.run.configHash !== anchorConfigHashByExperiment.get(a.experimentId));
|
|
602
|
+
const reference = staleReferenceOf(candidates);
|
|
603
|
+
if (reference) {
|
|
604
|
+
references ??= {};
|
|
605
|
+
references[condition] = reference;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
rows.push({ key: evalId, flipped, cells, ...(delta ? { delta } : {}), ...(references ? { references } : {}) });
|
|
562
610
|
}
|
|
563
611
|
// 各条件自身覆盖面:分母是该条件有结果的 eval 数,不看其它条件是否也覆盖了这道题。
|
|
564
612
|
const totals = {};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { TableContent, TableContentRow } from "../definition/cell.ts";
|
|
2
2
|
import type { DeltaData, StabilityMatrixCell, StabilityMatrixData } from "../model/types.ts";
|
|
3
|
-
|
|
3
|
+
import { type ReportLocale } from "../model/locale.ts";
|
|
4
|
+
export declare function deltaTableContent(data: DeltaData, locale?: ReportLocale): TableContent;
|
|
4
5
|
export interface StabilityContentRow extends TableContentRow {
|
|
5
6
|
readonly evalId: string;
|
|
6
7
|
/** 全部条件历史执行中通过次数为 0 且执行数 > 0。 */
|