niceeval 0.12.0 → 0.12.1-canary.22

package/docs-site/zh/examples/index.mdx

@@ -42,6 +42,13 @@ description: "按被测对象选择可运行的 NiceEval 示例：Agent Framewor
 - [查看在线报告](https://niceeval.com/showcase/memory)
 - [查看项目源码](https://github.com/CorrectRoadH/coding-agent-memory-evals)
 
+### Terminal-Bench
+
+[terminal-bench](https://github.com/NiceEval/terminal-bench) 用 NiceEval 跑经过审核的 Harbor Terminal-Bench 题包，判分仍走原版 `run-tests.sh`，并把各 coding agent 与模型并排比较。
+
+- [查看在线报告](https://niceeval.com/showcase/terminal-bench)
+- [查看项目源码](https://github.com/NiceEval/terminal-bench)
+
 ## Examples 的收录边界
 
 - 单段 API 用法放进任务教程或 Reference。

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "niceeval",
-  "version": "0.12.0",
+  "version": "0.12.1-canary.22",
   "description": "Agent-native eval tool — eval agents, services, functions, and coding-agent fixtures",
   "type": "module",
   "license": "MIT",

package/src/runner/accept.test.ts

@@ -186,14 +186,24 @@ describe("acceptPreparedAttempt", () => {
     roots.push(root);
     const currentManifest = { algorithmVersion: 2, coverageVersion: 1, config: {}, source: {}, data: {} };
     const sources = [makeSource(root, { id: "e" }), makeSource(root, { id: "f" })];
+    // 模拟 prepareAcceptTarget 的真实形状:每条 locator 的 currentExperiment.selectedEvalIds
+    // 只有自己那一题(cliPatterns 单 id),sandboxPlansByEval 也只有自己。
     const prepared = await Promise.all(sources.map((source) => prepareAcceptedAttempt({
       recordRoot: root,
       source,
       pair: makePair({}, source.evalId),
       currentFingerprint: `current-${source.evalId}`,
       currentManifest,
-      currentConfigHash: `config-${source.evalId}`,
-      knownEvalIds: ["e", "f"],
+      currentConfigHash: "shared-config",
+      currentExperiment: {
+        attempts: 1,
+        earlyExit: true,
+        selectedEvalIds: [source.evalId],
+        sandboxLayer: {},
+        sandboxPlansByEval: { [source.evalId]: { plan: source.evalId } },
+        agentInstalls: [],
+      },
+      knownEvalIds: [source.evalId],
       now: () => "2026-01-02T00:00:00.000Z",
     })));
 
@@ -206,6 +216,22 @@ describe("acceptPreparedAttempt", () => {
       accepted.map((entry) => entry.sourceLocator),
     );
     expect(accepted[0]!.record.experiments[0]!.runs).toHaveLength(1);
+
+    // 快照级覆盖声明必须是整组,不能只剩 groupFirst 的单题——否则 currentSample / view 塌成 1 题。
+    const runMeta = JSON.parse(
+      await readFile(join(accepted[0]!.run.dir, "run.json"), "utf-8"),
+    ) as {
+      experiment?: { selectedEvalIds?: string[]; sandboxPlansByEval?: globalThis.Record<string, unknown> };
+      knownEvalIds?: string[];
+    };
+    expect(runMeta.experiment?.selectedEvalIds?.slice().sort()).toEqual(["e", "f"]);
+    expect(Object.keys(runMeta.experiment?.sandboxPlansByEval ?? {}).sort()).toEqual(["e", "f"]);
+    expect(runMeta.knownEvalIds?.slice().sort()).toEqual(["e", "f"]);
+
+    // 读面:currentSample 必须看到两条,不能按错误收窄的 selectedEvalIds 只留第一条。
+    const { currentSample } = await import("../sample/index.ts");
+    const sample = currentSample(accepted[0]!.record);
+    expect(sample.attempts.map((a) => a.evalId).sort()).toEqual(["e", "f"]);
   });
 
   it("批量 prepare 中任一条失败时不创建 snapshot", async () => {

package/src/runner/accept.ts

@@ -520,7 +520,7 @@ export async function writeAcceptedAttempts(
   // 返回值必须按调用方传入的 preparedAttempts 顺序还原;分组打乱了处理顺序,用引用做索引。
   const locatorByPrepared = new Map<PreparedAcceptedAttempt, AttemptLocator>();
   for (const [experimentId, group] of groups) {
-    // 快照级字段(agent/model/configHash/currentExperiment/name)按本组取,不再从全批 first 拿——
+    // 快照级字段(agent/model/configHash/name)按本组取,不再从全批 first 拿——
     // 否则跨 experiment 批次会把另一个 experiment 的身份写进这个 experiment 的 run.json。
     const groupFirst = group[0]!;
     // manifests 是这个 experiment 自己的袋子;不同 experiment 各自独立,不共享同一个对象,
@@ -529,15 +529,21 @@ export async function writeAcceptedAttempts(
     const knownEvalIds = new Set<string>();
     for (const prepared of group) {
       manifests[prepared.source.evalId] = prepared.currentManifest;
+      knownEvalIds.add(prepared.source.evalId);
       for (const evalId of prepared.knownEvalIds ?? []) knownEvalIds.add(evalId);
     }
+    // prepare 阶段每条 locator 单独按「只选中自己那一题」重算指纹,currentExperiment.selectedEvalIds
+    // 因此只有一个 id。快照级覆盖声明必须是本 experiment 组**全部**接受的题——currentSample
+    // 按 selectedEvalIds 过滤贡献范围,只写 groupFirst 会让批量 accept 的 view/show 塌成 1 题
+    // (MemoryBench: 36 条 result.json 在盘、首页 1/36)。
+    const experiment = experimentForAcceptedGroup(group);
     const snapshot = await writer.run({
       experimentId,
       agent: groupFirst.pair.run.agent.name,
       ...(groupFirst.pair.run.model !== undefined ? { model: groupFirst.pair.run.model } : {}),
       startedAt: now,
       configHash: groupFirst.currentConfigHash,
-      ...(groupFirst.currentExperiment === undefined ? {} : { experiment: groupFirst.currentExperiment }),
+      ...(experiment === undefined ? {} : { experiment }),
       ...(knownEvalIds.size === 0 ? {} : { knownEvalIds: [...knownEvalIds] }),
       manifests,
       ...(groupFirst.name === undefined ? {} : { name: groupFirst.name }),
@@ -570,6 +576,39 @@ export async function writeAcceptedAttempts(
   });
 }
 
+/**
+ * 把同 experiment 一批 accept 的覆盖声明合成快照级 `ExperimentRunInfo`。
+ * 单条 prepare 的 `currentExperiment.selectedEvalIds` / `sandboxPlansByEval` 只含自己那题
+ * (指纹重算需要);封口时必须扩成整组,否则 Sample 读面按 selectedEvalIds 过滤会丢掉其余题。
+ */
+function experimentForAcceptedGroup(
+  group: readonly PreparedAcceptedAttempt[],
+): EvalResult["experiment"] | undefined {
+  const bases = group
+    .map((prepared) => prepared.currentExperiment)
+    .filter((experiment): experiment is NonNullable<typeof experiment> => experiment !== undefined);
+  const base = bases[0];
+  if (base === undefined) return undefined;
+
+  const selectedEvalIds = new Set<string>(base.selectedEvalIds);
+  const sandboxPlansByEval: globalThis.Record<string, JsonValue> = {
+    ...base.sandboxPlansByEval,
+  };
+  for (const prepared of group) {
+    selectedEvalIds.add(prepared.source.evalId);
+    const plans = prepared.currentExperiment?.sandboxPlansByEval;
+    if (plans === undefined) continue;
+    for (const [evalId, plan] of Object.entries(plans)) {
+      sandboxPlansByEval[evalId] = plan;
+    }
+  }
+  return {
+    ...base,
+    selectedEvalIds: [...selectedEvalIds],
+    sandboxPlansByEval,
+  };
+}
+
 function acceptedResultFor(
   prepared: PreparedAcceptedAttempt,
   locator: AttemptLocator,