niceeval 0.11.1 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/README.md +4 -2
- package/README.zh.md +4 -2
- package/dist/agents/types.d.ts +2 -2
- package/dist/o11y/types.d.ts +3 -3
- package/dist/report/assets/colors.d.ts +1 -1
- package/dist/report/assets/colors.js +1 -1
- package/dist/report/components/attempt-detail/AttemptSource.js +1 -2
- package/dist/report/components/attempt-detail/compute.d.ts +4 -4
- package/dist/report/components/attempt-detail/compute.js +6 -6
- package/dist/report/components/attempt-detail/faces.d.ts +1 -1
- package/dist/report/components/attempt-detail/faces.js +3 -3
- package/dist/report/components/entity-lists/AttemptList.d.ts +3 -3
- package/dist/report/components/entity-lists/AttemptList.js +3 -3
- package/dist/report/components/entity-lists/ExperimentList.js +2 -2
- package/dist/report/components/entity-lists/compute.js +1 -1
- package/dist/report/components/entity-lists/faces.js +1 -1
- package/dist/report/components/entity-lists/index.d.ts +1 -1
- package/dist/report/components/entity-lists/index.js +1 -1
- package/dist/report/components/metric-views/chart-math.d.ts +2 -2
- package/dist/report/components/metric-views/chart-math.js +2 -2
- package/dist/report/components/metric-views/compute.d.ts +2 -3
- package/dist/report/components/metric-views/compute.js +3 -4
- package/dist/report/components/shared.d.ts +1 -1
- package/dist/report/components/site-components/compute.d.ts +5 -5
- package/dist/report/components/site-components/compute.js +6 -6
- package/dist/report/components/site-components/index.d.ts +7 -7
- package/dist/report/components/site-components/index.js +7 -7
- package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
- package/dist/report/components/site-components/scope-warnings.js +3 -3
- package/dist/report/components/site-components/snapshot-diagnostics.js +1 -1
- package/dist/report/components/summaries/compute.d.ts +1 -1
- package/dist/report/components/summaries/compute.js +2 -2
- package/dist/report/components/summaries/faces.js +1 -1
- package/dist/report/components/summaries/index.d.ts +1 -1
- package/dist/report/components/summaries/index.js +1 -1
- package/dist/report/index.js +1 -1
- package/dist/report/model/aggregate.d.ts +2 -2
- package/dist/report/model/aggregate.js +2 -2
- package/dist/report/model/format.d.ts +2 -2
- package/dist/report/model/format.js +2 -2
- package/dist/report/model/metrics.js +3 -3
- package/dist/report/model/types.d.ts +11 -12
- package/dist/results/locator.js +1 -1
- package/dist/results/select.d.ts +4 -4
- package/dist/results/select.js +5 -5
- package/dist/results/types.d.ts +8 -8
- package/dist/runner/types.d.ts +14 -14
- package/dist/sandbox/resolve.d.ts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +21 -0
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/report-components.mdx +156 -74
- package/docs-site/zh/tutorials/custom-reports.mdx +112 -22
- package/docs-site/zh/tutorials/sandbox-agent.mdx +3 -4
- package/docs-site/zh/tutorials/sandbox-providers.mdx +15 -15
- package/docs-site/zh/tutorials/theming.mdx +169 -0
- package/package.json +2 -1
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/bub-install-spec.ts +26 -8
- package/src/agents/bub.ts +40 -15
- package/src/agents/builtin.ts +11 -1
- package/src/agents/coding-cli-versions.ts +67 -0
- package/src/agents/hermes.ts +235 -0
- package/src/agents/index.ts +4 -0
- package/src/agents/langgraph.test.ts +1 -1
- package/src/agents/manifest.ts +1 -1
- package/src/agents/openai-compat.test.ts +1 -1
- package/src/agents/openclaw.ts +103 -24
- package/src/agents/opencode.ts +183 -0
- package/src/agents/sdk-streams.test.ts +1 -1
- package/src/agents/shared.ts +13 -2
- package/src/agents/types.ts +2 -2
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/o11y/cost.ts +1 -1
- package/src/o11y/parsers/bub.test.ts +1 -1
- package/src/o11y/parsers/bub.ts +1 -1
- package/src/o11y/parsers/codex.test.ts +1 -1
- package/src/o11y/parsers/codex.ts +1 -1
- package/src/o11y/parsers/hermes.ts +198 -0
- package/src/o11y/parsers/openclaw.ts +25 -3
- package/src/o11y/parsers/opencode.ts +295 -0
- package/src/o11y/types.ts +3 -3
- package/src/report/assets/colors.ts +1 -1
- package/src/report/assets/styles.css +17 -15
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptConversation.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiff.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptError.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptSource.tsx +2 -3
- package/src/report/components/attempt-detail/AttemptSummary.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTrace.tsx +1 -1
- package/src/report/components/attempt-detail/UsageTable.tsx +1 -1
- package/src/report/components/attempt-detail/attempt-components.test.tsx +1 -1
- package/src/report/components/attempt-detail/compute.ts +6 -6
- package/src/report/components/attempt-detail/faces.ts +3 -3
- package/src/report/components/attempt-detail/index.tsx +1 -1
- package/src/report/components/entity-lists/AttemptList.tsx +3 -3
- package/src/report/components/entity-lists/ExperimentList.tsx +2 -2
- package/src/report/components/entity-lists/compute.ts +1 -1
- package/src/report/components/entity-lists/faces.ts +1 -1
- package/src/report/components/entity-lists/index.tsx +1 -1
- package/src/report/components/metric-views/DeltaTable.tsx +1 -2
- package/src/report/components/metric-views/StabilityMatrix.tsx +1 -1
- package/src/report/components/metric-views/chart-math.test.ts +1 -1
- package/src/report/components/metric-views/chart-math.ts +2 -2
- package/src/report/components/metric-views/compute.ts +3 -4
- package/src/report/components/shared.ts +1 -1
- package/src/report/components/site-components/CopyFixPrompt.tsx +1 -1
- package/src/report/components/site-components/HeroCard.tsx +1 -1
- package/src/report/components/site-components/PoweredBy.tsx +1 -1
- package/src/report/components/site-components/ScopeWarnings.tsx +1 -1
- package/src/report/components/site-components/SnapshotDiagnostics.tsx +1 -1
- package/src/report/components/site-components/TraceWaterfall.tsx +1 -1
- package/src/report/components/site-components/compute.ts +6 -6
- package/src/report/components/site-components/index.tsx +7 -7
- package/src/report/components/site-components/scope-warnings.ts +3 -3
- package/src/report/components/site-components/snapshot-diagnostics.ts +1 -1
- package/src/report/components/summaries/ScopeSummary.tsx +2 -2
- package/src/report/components/summaries/compute.ts +2 -2
- package/src/report/components/summaries/faces.ts +1 -1
- package/src/report/components/summaries/index.tsx +1 -1
- package/src/report/index.ts +1 -1
- package/src/report/model/aggregate.ts +2 -2
- package/src/report/model/format.ts +2 -2
- package/src/report/model/metrics.ts +3 -3
- package/src/report/model/types.ts +12 -13
- package/src/results/annotated-source.test.ts +1 -1
- package/src/results/attempt-evidence.test.ts +1 -1
- package/src/results/copy.ts +3 -3
- package/src/results/format.ts +3 -3
- package/src/results/host-equivalence.test.ts +6 -6
- package/src/results/index.ts +1 -1
- package/src/results/locator.test.ts +2 -2
- package/src/results/locator.ts +1 -1
- package/src/results/open.ts +2 -2
- package/src/results/publish.ts +1 -1
- package/src/results/results.test.ts +6 -323
- package/src/results/select.test.ts +438 -0
- package/src/results/select.ts +6 -6
- package/src/results/skipped-notice.ts +1 -1
- package/src/results/truncate.ts +2 -2
- package/src/results/types.ts +9 -9
- package/src/results/writer.ts +4 -4
- package/src/runner/attempt.test.ts +2 -2
- package/src/runner/attempt.ts +6 -6
- package/src/runner/eval-source.test.ts +1 -1
- package/src/runner/eval-source.ts +1 -1
- package/src/runner/reporters/artifacts.ts +1 -1
- package/src/runner/run.ts +2 -2
- package/src/runner/timing.ts +1 -1
- package/src/runner/types.ts +14 -14
- package/src/sandbox/docker-agent-image.ts +36 -0
- package/src/sandbox/e2b-agent-template.test.ts +6 -21
- package/src/sandbox/e2b-agent-template.ts +44 -11
- package/src/sandbox/index.ts +8 -0
- package/src/sandbox/official-baselines.test.ts +175 -0
- package/src/sandbox/resolve.ts +1 -1
- package/src/sandbox/types.ts +1 -1
- package/src/scoring/diff.ts +1 -1
- package/src/scoring/types.ts +1 -1
- package/src/shared/facts.ts +1 -1
- package/src/show/index.ts +1 -1
- package/src/show/render.ts +1 -1
- package/src/show/show.test.ts +3 -3
- package/src/view/app/App.tsx +3 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +1 -1
- package/src/view/data.test.ts +2 -2
- package/src/view/data.ts +1 -1
- package/src/view/site-base.test.ts +51 -0
- package/src/view/site.ts +16 -0
- package/src/view/styles.css +1 -2
- package/src/view/template.html +1 -0
- package/src/view/view-report.test.ts +1 -1
package/src/results/copy.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// copySnapshots:把选中快照按格式感知地复制到另一个目录(定稿见 docs/feature/
|
|
1
|
+
// copySnapshots:把选中快照按格式感知地复制到另一个目录(定稿见 docs/feature/record/library.md「复制与瘦身」)。
|
|
2
2
|
//
|
|
3
3
|
// 发布场景的原语:只带指定 artifact、只带选中快照的全部 attempt,布局知识不外泄。
|
|
4
4
|
// artifact 复制忠实于源(原字节,不重新序列化、不改写);snapshot.json / result.json
|
|
@@ -20,7 +20,7 @@ import type { ArtifactKind, AttemptHandle, Scope, Snapshot, SnapshotMeta } from
|
|
|
20
20
|
|
|
21
21
|
/** 缺省携带的 artifact:commands / events / trace / o11y / agentSetup / sources;diff 不截断、
|
|
22
22
|
* 可达百 MB,缺省不带。commands 缺省带的理由单独交代:失败命令证据是 errored attempt 的主要
|
|
23
|
-
* 下钻面,不能默认发布拷贝时静默删掉(见 docs/feature/
|
|
23
|
+
* 下钻面,不能默认发布拷贝时静默删掉(见 docs/feature/record/library.md「复制与瘦身」)。 */
|
|
24
24
|
const DEFAULT_PUBLISH_ARTIFACTS: ArtifactKind[] = ["commands", "events", "trace", "o11y", "agentSetup", "sources"];
|
|
25
25
|
const VALID_ARTIFACTS: ArtifactKind[] = ["commands", "events", "trace", "o11y", "agentSetup", "diff", "sources"];
|
|
26
26
|
|
|
@@ -234,7 +234,7 @@ async function findArtifactFiles(
|
|
|
234
234
|
break;
|
|
235
235
|
}
|
|
236
236
|
} catch {
|
|
237
|
-
// 缺文件跳过:某类数据为空本来就不生成对应 JSON(docs/feature/
|
|
237
|
+
// 缺文件跳过:某类数据为空本来就不生成对应 JSON(docs/feature/record/architecture.md)。
|
|
238
238
|
}
|
|
239
239
|
}
|
|
240
240
|
}
|
package/src/results/format.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// Results Format 的布局与版本知识,规则见 docs/feature/
|
|
1
|
+
// Results Format 的布局与版本知识,规则见 docs/feature/record/architecture.md。
|
|
2
2
|
//
|
|
3
|
-
// 按 docs/feature/
|
|
3
|
+
// 按 docs/feature/record/library.md,这份知识只住在本库:写入面(writer.ts / copy.ts)与
|
|
4
4
|
// 读取面(open.ts)共用这里的目录规则与版本判定;src/runner/reporters/artifacts.ts
|
|
5
5
|
// 是 writer 的薄壳,view(src/view/data.ts)经 openResults 消费,不自带布局知识。
|
|
6
6
|
|
|
@@ -57,7 +57,7 @@ export type SnapshotClassification =
|
|
|
57
57
|
| { kind: "not-a-report" };
|
|
58
58
|
|
|
59
59
|
/**
|
|
60
|
-
* 版本判定与最小形状校验(docs/feature/
|
|
60
|
+
* 版本判定与最小形状校验(docs/feature/record/architecture.md「版本不匹配时的读取行为」):
|
|
61
61
|
* - 带 format 信封:format 不是 niceeval.results → 无关 JSON;schemaVersion 非数字 → malformed;
|
|
62
62
|
* schemaVersion 与当前不同 → 不兼容(不解析、不迁移、不降级),带 schemaVersion 与完整 producer;
|
|
63
63
|
* schemaVersion 相同时校验 experimentId / agent / startedAt 均为 string,不满足 → malformed。
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/
|
|
2
|
-
// show / view 两宿主的 Scope 选择等价契约(docs/feature/
|
|
3
|
-
// docs/feature/
|
|
1
|
+
// cases: docs/engineering/testing/unit/record.md
|
|
2
|
+
// show / view 两宿主的 Scope 选择等价契约(docs/feature/record/architecture.md「Selection 是计算入口」;
|
|
3
|
+
// docs/feature/record/library.md「选择快照」)。
|
|
4
4
|
//
|
|
5
5
|
// 守护的不变量:同一结果根、同一组范围参数下,两扇门(niceeval show 的 text 面、niceeval view 的
|
|
6
6
|
// web 面)传给 selectCurrentResults(results, scope) 同形的 scope({ experiment, patterns }),必须
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// docs/engineering/testing/e2e/report.md 的读面 CLI 行为(§4)与渲染面(§5)验收。
|
|
13
13
|
//
|
|
14
14
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
15
|
-
// 依据 docs/feature/
|
|
15
|
+
// 依据 docs/feature/record/architecture.md 的稳定磁盘契约(与 show.test.ts / view/data.test.ts 同一写法)。
|
|
16
16
|
|
|
17
17
|
import { afterEach, describe, expect, it } from "vitest";
|
|
18
18
|
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
@@ -42,7 +42,7 @@ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}):
|
|
|
42
42
|
return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
/** 实验目录名清洗:与 docs/feature/
|
|
45
|
+
/** 实验目录名清洗:与 docs/feature/record/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
|
|
46
46
|
function cleanDirName(id: string): string {
|
|
47
47
|
return id.replace(/[^\w.@-]/g, "_");
|
|
48
48
|
}
|
|
@@ -150,7 +150,7 @@ function normalizeWarning(w: ScopeWarning): NormWarning {
|
|
|
150
150
|
/**
|
|
151
151
|
* Selection 的「按 experiment × eval」视图现在从 `selection.attempts` 分组重建,不再读
|
|
152
152
|
* `snapshot.evals`——真实 Snapshot 各自持有完整(未按现刻水位收窄的)evals,只有物化的
|
|
153
|
-
* `Scope.attempts` 才是这次选择的真正结果(docs/feature/
|
|
153
|
+
* `Scope.attempts` 才是这次选择的真正结果(docs/feature/record/library.md「官方现刻水位」)。
|
|
154
154
|
*/
|
|
155
155
|
function normalizeSelection(selection: Scope): NormSelection {
|
|
156
156
|
const byExperiment = new Map<string, Map<string, NormAttempt[]>>();
|
package/src/results/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// niceeval/results —— 实验结果数据的读写库(定稿见 docs/feature/
|
|
1
|
+
// niceeval/results —— 实验结果数据的读写库(定稿见 docs/feature/record/library.md、docs/feature/record/architecture.md)。
|
|
2
2
|
//
|
|
3
3
|
// 读:openResults(实验 → 快照 → eval → attempt 分层、skipped、latest() / current() Scope);
|
|
4
4
|
// 写:createResultsWriter(快照声明 + attempt 增量落盘 + finish 补 completedAt);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/
|
|
2
|
-
// AttemptLocator 单测(定稿见 docs/feature/
|
|
1
|
+
// cases: docs/engineering/testing/unit/record.md
|
|
2
|
+
// AttemptLocator 单测(定稿见 docs/feature/record/library.md「按 locator 寻址一个 attempt」、docs/concepts.md「Attempt 定位符」):
|
|
3
3
|
// 确定性编码、字段敏感(任一身份字段变化都换 locator)、@ 前缀与定长 base36 格式、decode 的语法
|
|
4
4
|
// 校验分支、大批量下的实际无撞车、buildLocatorIndex 的撞车检测(用可注入的 encode 强制制造撞车,
|
|
5
5
|
// 不依赖真实哈希小概率碰撞)与「相同身份重复出现不算撞车」(--resume 携带条目跨快照复现同一身份的场景)、
|
package/src/results/locator.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// AttemptLocator:Attempt 的不透明、可复制、可发布的短标识(定稿见 docs/feature/
|
|
1
|
+
// AttemptLocator:Attempt 的不透明、可复制、可发布的短标识(定稿见 docs/feature/record/library.md
|
|
2
2
|
// 「按 locator 寻址一个 attempt」、docs/concepts.md「Attempt 定位符」、docs/feature/reports/view.md 的 `#/attempt/@<locator>` 路由)。
|
|
3
3
|
//
|
|
4
4
|
// locator 由不可变身份元组 {experimentId, 快照 startedAt, evalId, attempt 下标(0-indexed)}
|
package/src/results/open.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// openResults:扫描结果目录,返回「实验 → 快照 → eval → attempt」的类型化层次
|
|
2
|
-
// (定稿见 docs/feature/
|
|
2
|
+
// (定稿见 docs/feature/record/library.md「读:openResults」、docs/feature/record/architecture.md「读取规则」)。
|
|
3
3
|
//
|
|
4
4
|
// 三条铁律:
|
|
5
5
|
// - 忠实磁盘:快照与实验归组只切片,不合并、不聚合、不去重;合并/聚合永远发生在消费方。
|
|
@@ -476,7 +476,7 @@ function makeAttempt(snapshot: Snapshot, snapshotDir: string, attemptDir: string
|
|
|
476
476
|
snapshot,
|
|
477
477
|
locator: record.locator as AttemptLocator,
|
|
478
478
|
// 携带条目投影:artifactBase 有值就是本快照 `--resume` 合入的上一轮终态结果
|
|
479
|
-
// (docs/feature/
|
|
479
|
+
// (docs/feature/sample/library.md「时效:新执行与历史执行」)。
|
|
480
480
|
carried: Boolean(record.artifactBase),
|
|
481
481
|
commands: lazyArtifact<FailedCommandEvidence[]>(candidates, "commands", record.commands),
|
|
482
482
|
events: lazyArtifact<StreamEvent[]>(candidates, "events", record.events),
|
package/src/results/publish.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// 发布预算(见 docs/feature/
|
|
1
|
+
// 发布预算(见 docs/feature/record/library.md「复制与瘦身:copySnapshots」)。
|
|
2
2
|
// 结果数据分两类:.niceeval/ 是本地事实根,不是默认可提交目录;任何要离开本机的拷贝是
|
|
3
3
|
// 发布拷贝,经 copySnapshots 这一条管线产出。管线只做选择、归拢与整文件大小预检,
|
|
4
4
|
// 不改写内容——保密边界由格式在采集侧划定(env 值与命令 stdout/stderr 不进结果文件)。
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/
|
|
1
|
+
// cases: docs/engineering/testing/unit/record.md
|
|
2
2
|
// niceeval/results 的单测:临时目录里构造最小 snapshot.json / result.json / artifact fixture,
|
|
3
|
-
// 覆盖定稿契约(docs/feature/
|
|
4
|
-
// 懒加载与 artifactBase 回退、skipped 三种原因、
|
|
5
|
-
// Selection.filter 修剪、dedupeAttempts 身份键、writer(独占目录、并发快照互不干扰、
|
|
3
|
+
// 覆盖定稿契约(docs/feature/record/library.md、docs/feature/record/architecture.md):分层读取(含快照级字段注入)、
|
|
4
|
+
// 懒加载与 artifactBase 回退、skipped 三种原因、writer(独占目录、并发快照互不干扰、
|
|
6
5
|
// snapshot.json 键形状、writeAttempt/writeAttemptFor、finish 幂等)、copySnapshots(布局、
|
|
7
6
|
// knownEvalIds 补记、artifacts 词干列表重算)。
|
|
8
7
|
// 读取面 fixture 的目录名/artifact 路径手写(不 import 库的路径函数),让测试独立于实现充当格式基准。
|
|
@@ -304,296 +303,6 @@ describe("openResults · skipped", () => {
|
|
|
304
303
|
});
|
|
305
304
|
});
|
|
306
305
|
|
|
307
|
-
// ───────────────────────── latest() Selection 与警告 ─────────────────────────
|
|
308
|
-
|
|
309
|
-
describe("results.latest() · Selection", () => {
|
|
310
|
-
it("每个实验取最新快照;experiments 前缀过滤同 CLI 语义(尾斜杠等价)", async () => {
|
|
311
|
-
const root = await makeRoot();
|
|
312
|
-
await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
313
|
-
await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
314
|
-
const tuesday = await writeSnapshot(root, "mid_b", "s2", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
315
|
-
|
|
316
|
-
const results = await openResults(root);
|
|
317
|
-
const latest = results.latest();
|
|
318
|
-
expect(latest.snapshots.map((s) => s.experimentId)).toEqual(["mid/a", "mid/b"]);
|
|
319
|
-
expect(latest.snapshots[1].dir).toBe(tuesday);
|
|
320
|
-
|
|
321
|
-
expect(results.latest({ experiments: "mid/a" }).snapshots).toHaveLength(1);
|
|
322
|
-
expect(results.latest({ experiments: "mid/" }).snapshots).toHaveLength(2);
|
|
323
|
-
expect(results.latest({ experiments: ["mid/a", "mid/b"] }).snapshots).toHaveLength(2);
|
|
324
|
-
expect(results.latest({ experiments: "other" }).snapshots).toHaveLength(0);
|
|
325
|
-
expect(results.latest({ experiments: "mid/a" }).snapshots[0].experimentId).toBe("mid/a"); // 不误配 "mid/ab"
|
|
326
|
-
});
|
|
327
|
-
|
|
328
|
-
it("coverage:最新快照覆盖 < 已知并集时,missingEvalIds 列出缺的题;分母是 knownEvalIds", async () => {
|
|
329
|
-
const root = await makeRoot();
|
|
330
|
-
const mondayDir = await writeSnapshot(root, "midterm", "2026-07-01T08-00-00-000Z", meta({ experimentId: "midterm/bub-gpt-5.4", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
331
|
-
await writeResultFile(mondayDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1 }));
|
|
332
|
-
await writeResultFile(mondayDir, "algebra/q2/a1", record({ id: "algebra/q2", attempt: 1 }));
|
|
333
|
-
await writeResultFile(mondayDir, "algebra/q3/a1", record({ id: "algebra/q3", attempt: 1 }));
|
|
334
|
-
|
|
335
|
-
const fridayDir = await writeSnapshot(root, "midterm", "2026-07-05T08-00-00-000Z", meta({ experimentId: "midterm/bub-gpt-5.4", agent: "bub", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }));
|
|
336
|
-
await writeResultFile(fridayDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1 }));
|
|
337
|
-
|
|
338
|
-
const latest = (await openResults(root)).latest();
|
|
339
|
-
expect(latest.snapshots).toHaveLength(1);
|
|
340
|
-
// partial-coverage / stale-snapshot 不再是 warnings —— 覆盖缺口是行级事实,物化在 coverage 上。
|
|
341
|
-
expect(latest.warnings.filter((w) => w.kind !== "unreadable-snapshot")).toHaveLength(0);
|
|
342
|
-
const coverage = latest.coverage.find((c) => c.experimentId === "midterm/bub-gpt-5.4")!;
|
|
343
|
-
expect(coverage.knownEvalIds).toEqual(["algebra/q1", "algebra/q2", "algebra/q3"]);
|
|
344
|
-
expect(coverage.missingEvalIds).toEqual(["algebra/q2", "algebra/q3"]);
|
|
345
|
-
});
|
|
346
|
-
|
|
347
|
-
it("unfinished-snapshot:选中快照缺 completedAt", async () => {
|
|
348
|
-
const root = await makeRoot();
|
|
349
|
-
const dir = await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }));
|
|
350
|
-
|
|
351
|
-
const latest = (await openResults(root)).latest();
|
|
352
|
-
const warn = latest.warnings.find((w) => w.kind === "unfinished-snapshot")!;
|
|
353
|
-
expect(warn).toMatchObject({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir });
|
|
354
|
-
expect(warn.message).toContain("has no completedAt");
|
|
355
|
-
});
|
|
356
|
-
|
|
357
|
-
it("Selection.filter 只删不换:快照删减,幸存实验的 coverage / 警告保留、其余丢弃", async () => {
|
|
358
|
-
const root = await makeRoot();
|
|
359
|
-
const aDir = await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
360
|
-
await writeResultFile(aDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
361
|
-
await writeResultFile(aDir, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
362
|
-
const bDir = await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
363
|
-
await writeResultFile(bDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
364
|
-
await writeResultFile(bDir, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
365
|
-
|
|
366
|
-
// 两个实验的最新快照都只重跑了 q1(残缺,进 coverage.missingEvalIds)且都未收尾(进 warnings)。
|
|
367
|
-
const aDir2 = await writeSnapshot(root, "mid_a", "s2", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z" }));
|
|
368
|
-
await writeResultFile(aDir2, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
369
|
-
const bDir2 = await writeSnapshot(root, "mid_b", "s2", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-02T08:00:00.000Z" }));
|
|
370
|
-
await writeResultFile(bDir2, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
371
|
-
|
|
372
|
-
const latest = (await openResults(root)).latest();
|
|
373
|
-
expect(latest.coverage.map((c) => c.experimentId).sort()).toEqual(["mid/a", "mid/b"]);
|
|
374
|
-
expect(latest.coverage.find((c) => c.experimentId === "mid/a")!.missingEvalIds).toEqual(["q2"]);
|
|
375
|
-
expect(latest.warnings.filter((w) => w.kind === "unfinished-snapshot")).toHaveLength(2);
|
|
376
|
-
|
|
377
|
-
const filtered = latest.filter((s) => s.experimentId !== "mid/b");
|
|
378
|
-
expect(filtered.snapshots.map((s) => s.experimentId)).toEqual(["mid/a"]);
|
|
379
|
-
expect(filtered.coverage.map((c) => c.experimentId)).toEqual(["mid/a"]);
|
|
380
|
-
expect(filtered.warnings.map((w) => ("experimentId" in w ? w.experimentId : undefined))).toEqual(["mid/a"]);
|
|
381
|
-
// 原 Selection 不被改动。
|
|
382
|
-
expect(latest.snapshots).toHaveLength(2);
|
|
383
|
-
expect(latest.coverage).toHaveLength(2);
|
|
384
|
-
expect(latest.warnings).toHaveLength(2);
|
|
385
|
-
});
|
|
386
|
-
|
|
387
|
-
it("current() 下同一 experiment 有两个贡献 Snapshot 时,filter 删除其中一个只删该来源的水位,不整实验全留或全删", async () => {
|
|
388
|
-
const root = await makeRoot();
|
|
389
|
-
// 周一:q1、q2 真实执行。
|
|
390
|
-
const mondayDir = await writeSnapshot(root, "multi_e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "multi/e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
391
|
-
await writeResultFile(mondayDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
392
|
-
await writeResultFile(mondayDir, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
393
|
-
// 周二:只重跑 q1;current() 从周一补齐 q2,两个真实快照都进 Scope.snapshots。
|
|
394
|
-
const tuesdayDir = await writeSnapshot(root, "multi_e", "2026-07-02T08-00-00-000Z", meta({ experimentId: "multi/e", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
395
|
-
await writeResultFile(tuesdayDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
396
|
-
|
|
397
|
-
const current = (await openResults(root)).current();
|
|
398
|
-
expect(current.snapshots.map((s) => s.startedAt).sort()).toEqual(["2026-07-01T08:00:00.000Z", "2026-07-02T08:00:00.000Z"]);
|
|
399
|
-
expect(current.attempts.map((a) => a.evalId).sort()).toEqual(["q1", "q2"]);
|
|
400
|
-
expect(current.coverage.find((c) => c.experimentId === "multi/e")!.missingEvalIds).toEqual([]);
|
|
401
|
-
|
|
402
|
-
// 删掉周一(q2 唯一来源),周二(q1 来源)保留:不是整实验全留或全删。
|
|
403
|
-
const filtered = current.filter((s) => s.startedAt !== "2026-07-01T08:00:00.000Z");
|
|
404
|
-
expect(filtered.snapshots.map((s) => s.startedAt)).toEqual(["2026-07-02T08:00:00.000Z"]);
|
|
405
|
-
expect(filtered.attempts.map((a) => a.evalId)).toEqual(["q1"]);
|
|
406
|
-
// knownEvalIds(分母)不变,只有被删来源独占贡献的 q2 转入 missingEvalIds。
|
|
407
|
-
const filteredCoverage = filtered.coverage.find((c) => c.experimentId === "multi/e")!;
|
|
408
|
-
expect(filteredCoverage.knownEvalIds).toEqual(["q1", "q2"]);
|
|
409
|
-
expect(filteredCoverage.missingEvalIds).toEqual(["q2"]);
|
|
410
|
-
// 原 Scope 不被改动。
|
|
411
|
-
expect(current.snapshots).toHaveLength(2);
|
|
412
|
-
expect(current.attempts).toHaveLength(2);
|
|
413
|
-
|
|
414
|
-
// 反过来删掉周二(q1 唯一来源),周一(q2 来源)保留。
|
|
415
|
-
const otherWay = current.filter((s) => s.startedAt !== "2026-07-02T08:00:00.000Z");
|
|
416
|
-
expect(otherWay.snapshots.map((s) => s.startedAt)).toEqual(["2026-07-01T08:00:00.000Z"]);
|
|
417
|
-
expect(otherWay.attempts.map((a) => a.evalId)).toEqual(["q2"]);
|
|
418
|
-
expect(otherWay.coverage.find((c) => c.experimentId === "multi/e")!.missingEvalIds).toEqual(["q1"]);
|
|
419
|
-
|
|
420
|
-
// attempt.snapshot 原样指回真实来源(不是过滤时重建的新对象)。
|
|
421
|
-
for (const attempt of filtered.attempts) {
|
|
422
|
-
expect(filtered.snapshots).toContain(attempt.snapshot);
|
|
423
|
-
}
|
|
424
|
-
});
|
|
425
|
-
});
|
|
426
|
-
|
|
427
|
-
// ───────────────────────── unreadable-snapshot 警告 ─────────────────────────
|
|
428
|
-
|
|
429
|
-
describe("results.latest() / results.current() · unreadable-snapshot", () => {
|
|
430
|
-
it("malformed 快照产生 warning 且其余快照照常计入;无关 JSON 不产生 warning", async () => {
|
|
431
|
-
const root = await makeRoot();
|
|
432
|
-
const okDir = await writeSnapshot(root, "ok-exp", "2026-07-04T08-00-00-000Z-oooo", meta({ experimentId: "ok", agent: "bub", startedAt: "2026-07-04T08:00:00.000Z", completedAt: "2026-07-04T08:10:00.000Z" }));
|
|
433
|
-
await writeResultFile(okDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
434
|
-
|
|
435
|
-
const badDir = join(root, "bad-exp", "2026-07-02T08-00-00-000Z-zzzz");
|
|
436
|
-
await mkdir(badDir, { recursive: true });
|
|
437
|
-
await writeFile(join(badDir, "snapshot.json"), "not json {", "utf-8");
|
|
438
|
-
|
|
439
|
-
const alienDir = join(root, "alien-exp", "2026-07-06T08-00-00-000Z-alien");
|
|
440
|
-
await mkdir(alienDir, { recursive: true });
|
|
441
|
-
await writeFile(join(alienDir, "summary.json"), JSON.stringify({ hello: 1 }), "utf-8");
|
|
442
|
-
|
|
443
|
-
const results = await openResults(root);
|
|
444
|
-
const latest = results.latest();
|
|
445
|
-
// 其余快照照常计入,不被坏落盘拖垮。
|
|
446
|
-
expect(latest.snapshots.map((s) => s.experimentId)).toEqual(["ok"]);
|
|
447
|
-
|
|
448
|
-
const unreadable = latest.warnings.filter((w) => w.kind === "unreadable-snapshot");
|
|
449
|
-
expect(unreadable).toHaveLength(1); // 无关 JSON(alien)不产生 warning
|
|
450
|
-
expect(unreadable[0]).toMatchObject({ kind: "unreadable-snapshot", dir: badDir, reason: "malformed" });
|
|
451
|
-
expect(unreadable[0].message).toContain(badDir);
|
|
452
|
-
expect(unreadable[0].message).toMatch(/inspect .*snapshot\.json/);
|
|
453
|
-
expect((unreadable[0] as { command?: string }).command).toBeUndefined();
|
|
454
|
-
|
|
455
|
-
// results.current() 同一份事实(show / view 报告槽走的是它)。
|
|
456
|
-
const current = results.current();
|
|
457
|
-
expect(current.warnings.filter((w) => w.kind === "unreadable-snapshot")).toHaveLength(1);
|
|
458
|
-
});
|
|
459
|
-
|
|
460
|
-
it("incompatible(schemaVersion 不兼容,niceeval producer)的 warning 带版本化 command", async () => {
|
|
461
|
-
const root = await makeRoot();
|
|
462
|
-
const oldDir = join(root, "old-exp", "2026-06-01T08-00-00-000Z");
|
|
463
|
-
await mkdir(oldDir, { recursive: true });
|
|
464
|
-
await writeFile(
|
|
465
|
-
join(oldDir, "snapshot.json"),
|
|
466
|
-
JSON.stringify({
|
|
467
|
-
format: RESULTS_FORMAT,
|
|
468
|
-
schemaVersion: RESULTS_SCHEMA_VERSION - 1,
|
|
469
|
-
producer: { name: "niceeval", version: "0.4.6" },
|
|
470
|
-
experimentId: "old",
|
|
471
|
-
agent: "bub",
|
|
472
|
-
startedAt: "2026-06-01T08:00:00.000Z",
|
|
473
|
-
}),
|
|
474
|
-
"utf-8",
|
|
475
|
-
);
|
|
476
|
-
|
|
477
|
-
const latest = (await openResults(root)).latest();
|
|
478
|
-
const warn = latest.warnings.find((w) => w.kind === "unreadable-snapshot")!;
|
|
479
|
-
expect(warn).toMatchObject({ kind: "unreadable-snapshot", dir: oldDir, reason: "incompatible-version" });
|
|
480
|
-
expect((warn as { command?: string }).command).toBe(`npx niceeval@0.4.6 show --results ${root}`);
|
|
481
|
-
expect(warn.message).toContain("npx niceeval@0.4.6 show --results");
|
|
482
|
-
});
|
|
483
|
-
|
|
484
|
-
it("非实验作用域:Selection.filter 收窄后 unreadable-snapshot warning 仍在", async () => {
|
|
485
|
-
const root = await makeRoot();
|
|
486
|
-
const aDir = await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
487
|
-
await writeResultFile(aDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
488
|
-
|
|
489
|
-
const badDir = join(root, "bad-exp", "2026-07-02T08-00-00-000Z-zzzz");
|
|
490
|
-
await mkdir(badDir, { recursive: true });
|
|
491
|
-
await writeFile(join(badDir, "snapshot.json"), "not json {", "utf-8");
|
|
492
|
-
|
|
493
|
-
const latest = (await openResults(root)).latest();
|
|
494
|
-
expect(latest.warnings.filter((w) => w.kind === "unreadable-snapshot")).toHaveLength(1);
|
|
495
|
-
|
|
496
|
-
// filter 只删快照,不删非实验作用域的警告 —— 即便过滤条件与该 warning 无关的实验都不复存在。
|
|
497
|
-
const filtered = latest.filter((s) => s.experimentId === "mid/a");
|
|
498
|
-
expect(filtered.snapshots).toHaveLength(1);
|
|
499
|
-
expect(filtered.warnings.filter((w) => w.kind === "unreadable-snapshot")).toHaveLength(1);
|
|
500
|
-
});
|
|
501
|
-
});
|
|
502
|
-
|
|
503
|
-
// ───────────────────────── ScopeWarning 联合成员恰为三种(回归锁死) ─────────────────────────
|
|
504
|
-
|
|
505
|
-
// 类型契约(编译期,随 pnpm typecheck):ScopeWarning 联合成员恰为三种——多一个 kind 而没同步
|
|
506
|
-
// 改这里,default 分支的 never 赋值编译不过;少一个则对应 case 编译不过。kind 全集不是运行时
|
|
507
|
-
// 行为,没有运行时断言可写(docs/engineering/testing/unit/README.md「类型契约」)。
|
|
508
|
-
function assertScopeWarningKindExhaustive(kind: import("./types.ts").ScopeWarning["kind"]): void {
|
|
509
|
-
switch (kind) {
|
|
510
|
-
case "unfinished-snapshot":
|
|
511
|
-
case "missing-startedAt":
|
|
512
|
-
case "unreadable-snapshot":
|
|
513
|
-
return;
|
|
514
|
-
default: {
|
|
515
|
-
const exhausted: never = kind;
|
|
516
|
-
void exhausted;
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
}
|
|
520
|
-
void assertScopeWarningKindExhaustive;
|
|
521
|
-
|
|
522
|
-
// ───────────────────────── 时效:attempt.carried 与 fresh 口径 ─────────────────────────
|
|
523
|
-
|
|
524
|
-
describe("时效:carried 投影与 fresh 口径", () => {
|
|
525
|
-
it("latest({ fresh: true }):排除携带条目,被排除的题进 coverage.missingEvalIds", async () => {
|
|
526
|
-
const root = await makeRoot();
|
|
527
|
-
const monday = await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
528
|
-
await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
529
|
-
|
|
530
|
-
// 周二:q1 携带合入(artifactBase 指回周一),q2 真实执行。
|
|
531
|
-
const tuesday = await writeSnapshot(root, "e", "2026-07-02T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
532
|
-
await writeResultFile(tuesday, "q1/a1", record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:00:00.000Z", artifactBase: "e/2026-07-01T08-00-00-000Z/q1/a1" }));
|
|
533
|
-
await writeResultFile(tuesday, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
534
|
-
|
|
535
|
-
const results = await openResults(root);
|
|
536
|
-
const notFresh = results.latest();
|
|
537
|
-
expect(notFresh.snapshots[0]!.evals.map((e) => e.id).sort()).toEqual(["q1", "q2"]);
|
|
538
|
-
expect(notFresh.coverage.find((c) => c.experimentId === "e")!.missingEvalIds).toEqual([]);
|
|
539
|
-
|
|
540
|
-
const fresh = results.latest({ fresh: true });
|
|
541
|
-
expect(fresh.attempts.map((a) => a.evalId)).toEqual(["q2"]); // q1 是携带条目,被排除
|
|
542
|
-
expect(fresh.coverage.find((c) => c.experimentId === "e")!.missingEvalIds).toEqual(["q1"]);
|
|
543
|
-
// 真实 Snapshot 原样保留:q1 仍在它自己的 evals 里,fresh 没有克隆/裁剪来源对象。
|
|
544
|
-
expect(fresh.snapshots[0]!.evals.map((e) => e.id).sort()).toEqual(["q1", "q2"]);
|
|
545
|
-
});
|
|
546
|
-
|
|
547
|
-
it("current({ fresh: true }):同时排除携带条目与跨快照拼入的历史执行,区分力 fixture ——「只排携带 / 只排旧快照 / 两者都排」三种候选算法各给出不同答案", async () => {
|
|
548
|
-
const root = await makeRoot();
|
|
549
|
-
// 周一:q1、q2 真实执行。
|
|
550
|
-
const monday = await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
551
|
-
await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
552
|
-
await writeResultFile(monday, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
553
|
-
|
|
554
|
-
// 周二:q1 携带合入(仍是这次快照里"最新"的一份,但 carried=true);q3 真实执行;q2 本次没有
|
|
555
|
-
// 任何 attempt —— current() 从周一补齐它,补齐来的 q2 attempt 所属快照早于本实验在 Scope 中
|
|
556
|
-
// 最新快照(周二),是「跨快照拼入」的历史执行,即使它自己不是携带条目。
|
|
557
|
-
const tuesday = await writeSnapshot(root, "e", "2026-07-02T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
558
|
-
await writeResultFile(tuesday, "q1/a1", record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:00:00.000Z", artifactBase: "e/2026-07-01T08-00-00-000Z/q1/a1" }));
|
|
559
|
-
await writeResultFile(tuesday, "q3/a1", record({ id: "q3", attempt: 1 }));
|
|
560
|
-
|
|
561
|
-
const results = await openResults(root);
|
|
562
|
-
const notFresh = results.current();
|
|
563
|
-
expect(notFresh.attempts.map((a) => a.evalId).sort()).toEqual(["q1", "q2", "q3"]);
|
|
564
|
-
expect(notFresh.coverage.find((c) => c.experimentId === "e")!.missingEvalIds).toEqual([]);
|
|
565
|
-
// 两个真实来源都在场:q2 唯一来自周一,q1/q3 来自周二——不是一个合并对象。
|
|
566
|
-
expect(notFresh.snapshots.map((s) => s.startedAt).sort()).toEqual(["2026-07-01T08:00:00.000Z", "2026-07-02T08:00:00.000Z"]);
|
|
567
|
-
|
|
568
|
-
const fresh = results.current({ fresh: true });
|
|
569
|
-
// 只有 q3(本次快照真实执行、非携带)是新执行;q1(携带)与 q2(跨快照拼入)都被排除。
|
|
570
|
-
expect(fresh.attempts.map((a) => a.evalId)).toEqual(["q3"]);
|
|
571
|
-
expect(fresh.coverage.find((c) => c.experimentId === "e")!.missingEvalIds.sort()).toEqual(["q1", "q2"]);
|
|
572
|
-
// 周一(q2 唯一来源)不再贡献任何 fresh attempt,退出 snapshots;周二仍在,即便它自己的
|
|
573
|
-
// evals 里仍然原样带着被排除的 q1(真实 Snapshot 不被裁剪)。
|
|
574
|
-
expect(fresh.snapshots.map((s) => s.startedAt)).toEqual(["2026-07-02T08:00:00.000Z"]);
|
|
575
|
-
expect(fresh.snapshots[0]!.evals.map((e) => e.id).sort()).toEqual(["q1", "q3"]);
|
|
576
|
-
});
|
|
577
|
-
|
|
578
|
-
it("fresh 清空全部 attempt 后 Scope 只剩 coverage 事实;filter 不拿 snapshots 当 coverage 的存续代理", async () => {
|
|
579
|
-
const root = await makeRoot();
|
|
580
|
-
const monday = await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
581
|
-
await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
582
|
-
// 周二唯一的条目是携带复印件:fresh 把它排除后,这个实验没有任何新执行。
|
|
583
|
-
const tuesday = await writeSnapshot(root, "e", "2026-07-02T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
584
|
-
await writeResultFile(tuesday, "q1/a1", record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:00:00.000Z", artifactBase: "e/2026-07-01T08-00-00-000Z/q1/a1" }));
|
|
585
|
-
|
|
586
|
-
const fresh = (await openResults(root)).current({ fresh: true });
|
|
587
|
-
expect(fresh.attempts).toEqual([]);
|
|
588
|
-
const coverage = fresh.coverage.find((c) => c.experimentId === "e")!;
|
|
589
|
-
expect(coverage.missingEvalIds).toEqual(["q1"]);
|
|
590
|
-
// coverage-only 实验(零 snapshot 贡献)经 filter 后覆盖缺口不静默消失——榜单占位行的上游事实。
|
|
591
|
-
expect(fresh.filter(() => true).coverage).toEqual(fresh.coverage);
|
|
592
|
-
});
|
|
593
|
-
});
|
|
594
|
-
|
|
595
|
-
// ───────────────────────── 身份键去重 ─────────────────────────
|
|
596
|
-
|
|
597
306
|
function fakeSnapshot(over: { experimentId: string; startedAt: string; dir: string }): Snapshot {
|
|
598
307
|
return {
|
|
599
308
|
agent: "bub",
|
|
@@ -623,32 +332,6 @@ function fakeAttempt(snapshot: Snapshot, result: EvalResult): AttemptHandle {
|
|
|
623
332
|
};
|
|
624
333
|
}
|
|
625
334
|
|
|
626
|
-
describe("dedupeAttempts", () => {
|
|
627
|
-
it("按 (experimentId, evalId, attempt, startedAt) 去重,保留最新快照里的那份;缺 startedAt 不去重并出 missing-startedAt", () => {
|
|
628
|
-
const monday = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir: "/tmp/e/monday" });
|
|
629
|
-
const tuesday = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-02T08:00:00.000Z", dir: "/tmp/e/tuesday" });
|
|
630
|
-
|
|
631
|
-
const a1 = fakeAttempt(monday, record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" }));
|
|
632
|
-
const a1Resumed = fakeAttempt(tuesday, record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" })); // resume 原样合入
|
|
633
|
-
const a2Mon = fakeAttempt(monday, record({ id: "q2", attempt: 1, startedAt: "2026-07-01T08:02:00.000Z" }));
|
|
634
|
-
const a2Tue = fakeAttempt(tuesday, record({ id: "q2", attempt: 1, startedAt: "2026-07-02T08:02:00.000Z" })); // 重跑,新 startedAt
|
|
635
|
-
const a3Mon = fakeAttempt(monday, record({ id: "q3", attempt: 1 })); // 缺 startedAt(携带条目缺锚的极端情况)
|
|
636
|
-
const a3Tue = fakeAttempt(tuesday, record({ id: "q3", attempt: 1 }));
|
|
637
|
-
|
|
638
|
-
const { attempts, warnings } = dedupeAttempts([a1, a2Mon, a3Mon, a1Resumed, a2Tue, a3Tue]);
|
|
639
|
-
expect(attempts).toHaveLength(5);
|
|
640
|
-
const q1 = attempts.filter((a) => a.evalId === "q1");
|
|
641
|
-
expect(q1).toHaveLength(1);
|
|
642
|
-
expect(q1[0].snapshot).toBe(tuesday); // 保留最新快照
|
|
643
|
-
expect(attempts.filter((a) => a.evalId === "q2")).toHaveLength(2);
|
|
644
|
-
expect(attempts.filter((a) => a.evalId === "q3")).toHaveLength(2);
|
|
645
|
-
|
|
646
|
-
expect(warnings).toHaveLength(2);
|
|
647
|
-
expect(warnings[0]).toMatchObject({ kind: "missing-startedAt", experimentId: "e", evalId: "q3" });
|
|
648
|
-
expect(warnings[0].message).toContain("has no startedAt");
|
|
649
|
-
});
|
|
650
|
-
});
|
|
651
|
-
|
|
652
335
|
// ───────────────────────── writer ─────────────────────────
|
|
653
336
|
|
|
654
337
|
describe("createResultsWriter", () => {
|
|
@@ -769,7 +452,7 @@ describe("createResultsWriter", () => {
|
|
|
769
452
|
expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
|
|
770
453
|
});
|
|
771
454
|
|
|
772
|
-
// cases: docs/engineering/testing/unit/
|
|
455
|
+
// cases: docs/engineering/testing/unit/record.md「Usage、facts 与失败命令证据落盘」
|
|
773
456
|
it("commands:落成 commands.json(不内联进 result.json),artifacts 含 commands;懒加载读回原样往返;没有非零命令的 attempt 恒 null;copySnapshots 缺省携带", async () => {
|
|
774
457
|
const root = await makeRoot();
|
|
775
458
|
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "0.12.0" } });
|
|
@@ -836,7 +519,7 @@ describe("createResultsWriter", () => {
|
|
|
836
519
|
});
|
|
837
520
|
|
|
838
521
|
it("每个 Snapshot 各自独立封口:两个 Experiment 各自不同的 completedAt 与 diagnostics 不串味,空 diagnostics 省略字段", async () => {
|
|
839
|
-
// cases: docs/engineering/testing/unit/
|
|
522
|
+
// cases: docs/engineering/testing/unit/record.md「落盘格式」——snap.finish() 唯一一次补
|
|
840
523
|
// completedAt 与快照级 diagnostics。
|
|
841
524
|
const root = await makeRoot();
|
|
842
525
|
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
@@ -882,7 +565,7 @@ describe("createResultsWriter", () => {
|
|
|
882
565
|
});
|
|
883
566
|
|
|
884
567
|
it("facts:attempt 级随 writeAttempt 落进 result.json、experiment 级随 finish() 落进 snapshot.json,两级原样读回不合并,空 facts 省略字段", async () => {
|
|
885
|
-
// cases: docs/engineering/testing/unit/
|
|
568
|
+
// cases: docs/engineering/testing/unit/record.md「Usage、facts 与失败命令证据落盘」——
|
|
886
569
|
// 两级 facts 各自独立落盘与读回,读取面不把 attempt facts 与 snapshot facts 混在一起。
|
|
887
570
|
const root = await makeRoot();
|
|
888
571
|
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|