niceeval 0.4.0 → 0.4.1

package/docs/cli.md

@@ -61,6 +61,8 @@ niceeval exp compare --force                  # 忽略指纹缓存,全量重跑
 niceeval exp compare --tag <tag>              # 按单个标签过滤
 ```
 
+不带 `--force` 时,续跑携带的基线是跨全部历史 run、每 `(experimentId, evalId)` 取最新一份(`src/view/loader.ts` 的 `loadLatestResultsPerEval`):上次 passed 且 fingerprint 匹配的直接携入新 summary,只真跑失败/errored/缺失的。所以「补齐一组实验」就是重跑 `niceeval exp <组>` 本身。注意带 eval-id 位置参数时,summary 只含本次计划内的 eval——补跑别用位置参数,否则产出部分快照(见 memory 的 rerun-with-eval-filter-partial-snapshot)。
+
 ## 评分与退出
 
 ```sh

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "niceeval",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Agent-native eval tool — eval agents, services, functions, and coding-agent fixtures",
   "type": "module",
   "license": "MIT",

package/src/cli.ts

@@ -17,7 +17,7 @@ import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
 import { JUnit } from "./runner/reporters/json.ts";
 import { Live as LiveReporter, type LiveRow } from "./runner/reporters/live.ts";
 import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
-import { buildView, startViewServer, loadMostRecentResults, IncompatibleResultsError } from "./view/index.ts";
+import { buildView, startViewServer, loadLatestResultsPerEval, IncompatibleResultsError } from "./view/index.ts";
 import { t } from "./i18n/index.ts";
 import { formatThrown } from "./util.ts";
 import type { Config, DiscoveredExperiment, Reporter } from "./types.ts";
@@ -435,7 +435,7 @@ async function main(): Promise<void> {
   const sandboxRecs = agentRuns.map((r) => sandboxRecommendedConcurrency(r.sandbox));
   const sandboxDefaultConcurrency = sandboxRecs.length > 0 ? Math.min(...sandboxRecs) : 10;
 
-  const priorResults = flags.force ? undefined : await loadMostRecentResults(join(cwd, ".niceeval"));
+  const priorResults = flags.force ? undefined : await loadLatestResultsPerEval(join(cwd, ".niceeval"));
 
   const summary = await runEvals({
     config,

package/src/runner/run.ts

@@ -70,7 +70,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     for (const r of opts.priorResults) {
       if (!r.experimentId || !priorRunKeys.has(`${r.experimentId}|${r.id}`)) continue;
       // artifactsDir 是相对"本次 run 自己目录"的路径,新 summary 换了目录就会失效,必须清掉。
-      // artifactBase 不一样:它已经是 loadMostRecentResults(经 withViewRefs)拼好的、
+      // artifactBase 不一样:它已经是 loadLatestResultsPerEval(经 withViewRefs)拼好的、
       // 相对稳定的 .niceeval 根的路径,指向旧 run 的产物目录依然可解析,原样带过来
       // ——不然 view 就再也找不到这条携带结果的源码/转录/trace 了。
       carriedResults.push({ ...r, artifactsDir: undefined });

package/src/view/index.ts

@@ -13,7 +13,7 @@ export {
   IncompatibleResultsError,
   incompatibleHint,
   incompatibleViewCommand,
-  loadMostRecentResults,
+  loadLatestResultsPerEval,
   loadSummaries,
   type IncompatibleRun,
   type ScanResult,

package/src/view/loader.test.ts

@@ -0,0 +1,61 @@
+// loadLatestResultsPerEval:续跑携带基线必须是跨历史「每 (experimentId, evalId) 最新一份」,
+// 而不是最近一个 run —— 否则部分补跑 run 会把基线清空,`exp <组>` 补齐失效。
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { loadLatestResultsPerEval } from "./loader.ts";
+import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION } from "../runner/types.ts";
+
+let root: string;
+
+afterEach(async () => {
+  if (root) await rm(root, { recursive: true, force: true });
+});
+
+async function writeRun(dir: string, startedAt: string, results: object[]): Promise<void> {
+  await mkdir(join(root, dir), { recursive: true });
+  await writeFile(
+    join(root, dir, "summary.json"),
+    JSON.stringify({ format: RESULTS_FORMAT, schemaVersion: RESULTS_SCHEMA_VERSION, startedAt, results }),
+  );
+}
+
+function result(experimentId: string, id: string, outcome: string, extra: object = {}): object {
+  return { experimentId, id, agent: "a", outcome, ...extra };
+}
+
+describe("loadLatestResultsPerEval", () => {
+  it("部分补跑 run 只遮蔽它跑过的 eval,其它 eval 仍取自更早的全量 run", async () => {
+    root = await mkdtemp(join(tmpdir(), "niceeval-loader-"));
+    await writeRun("2026-01-01T00-00-00", "2026-01-01T00:00:00.000Z", [
+      result("exp/a", "e1", "passed"),
+      result("exp/a", "e2", "errored"),
+      result("exp/b", "e1", "passed"),
+    ]);
+    // 部分补跑:只重跑了 exp/a 的 e2
+    await writeRun("2026-01-02T00-00-00", "2026-01-02T00:00:00.000Z", [result("exp/a", "e2", "passed")]);
+
+    const results = (await loadLatestResultsPerEval(root)) as Array<{ experimentId: string; id: string; outcome: string }>;
+    const byKey = new Map(results.map((r) => [`${r.experimentId}|${r.id}`, r.outcome]));
+    expect(byKey.get("exp/a|e1")).toBe("passed"); // 来自旧全量 run,没被部分 run 冲掉
+    expect(byKey.get("exp/a|e2")).toBe("passed"); // 来自补跑 run(最新)
+    expect(byKey.get("exp/b|e1")).toBe("passed");
+    expect(results).toHaveLength(3);
+  });
+
+  it("同 (experiment, eval) 多 attempt 整批取自含它的最新 run,不跨 run 混装", async () => {
+    root = await mkdtemp(join(tmpdir(), "niceeval-loader-"));
+    await writeRun("2026-01-01T00-00-00", "2026-01-01T00:00:00.000Z", [
+      result("exp/a", "e1", "passed", { fingerprint: "old" }),
+    ]);
+    await writeRun("2026-01-02T00-00-00", "2026-01-02T00:00:00.000Z", [
+      result("exp/a", "e1", "failed", { fingerprint: "new" }),
+      result("exp/a", "e1", "passed", { fingerprint: "new" }),
+    ]);
+
+    const results = (await loadLatestResultsPerEval(root)) as Array<{ fingerprint: string }>;
+    expect(results).toHaveLength(2);
+    expect(results.every((r) => r.fingerprint === "new")).toBe(true);
+  });
+});

package/src/view/loader.ts

@@ -87,11 +87,27 @@ export class NotAReportError extends Error {
   }
 }
 
-/** 读最近一次运行的所有 EvalResult，供 --resume 跳过已通过的 eval。 */
-export async function loadMostRecentResults(root = ".niceeval"): Promise<EvalResult[]> {
-  const { loaded } = await loadSummaries(root);
-  // loadSummaries 已按 startedAt 降序，第一个是最新的
-  return loaded[0]?.summary.results ?? [];
+/**
+ * 读跨历史「每 (experimentId, evalId) 最新一份」的 EvalResult,供续跑携带已通过结果。
+ * 只看最近一个 run 不行:部分补跑(位置参数只跑几道题)会把携带基线换成那个部分 run,
+ * 之后重跑任何实验都携带不到东西,`exp <组>` 的「补齐缺失」语义随之失效。
+ * 同一 (experimentId, evalId) 的多个 attempt 整批取自含它的最新 run,不跨 run 混装。
+ */
+export async function loadLatestResultsPerEval(root = ".niceeval"): Promise<EvalResult[]> {
+  const { loaded } = await loadSummaries(root); // 已按 startedAt 降序
+  const claimed = new Set<string>();
+  const results: EvalResult[] = [];
+  for (const { summary } of loaded) {
+    const takenThisRun = new Set<string>();
+    for (const r of summary.results ?? []) {
+      const key = `${r.experimentId ?? `${r.agent}|${r.model ?? ""}`}|${r.id}`;
+      if (claimed.has(key)) continue;
+      takenThisRun.add(key);
+      results.push(r);
+    }
+    for (const key of takenThisRun) claimed.add(key);
+  }
+  return results;
 }
 
 /** 服务/解析工件的根目录:输入是目录就用它,是文件就用其所在目录。 */