niceeval 0.4.3 → 0.4.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "niceeval",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "Agent-native eval tool — eval agents, services, functions, and coding-agent fixtures",
   "type": "module",
   "license": "MIT",

package/src/cli.ts

@@ -11,6 +11,7 @@ import { pathToFileURL } from "node:url";
 import { parseArgs as nodeParseArgs } from "node:util";
 import { discoverEvals, discoverExperiments, makeFilter } from "./runner/discover.ts";
 import { runEvals, type AgentRun } from "./runner/run.ts";
+import { runWho } from "./runner/types.ts";
 import { stopAllSandboxes, liveSandboxCount } from "./sandbox/registry.ts";
 import { sandboxRecommendedConcurrency } from "./sandbox/resolve.ts";
 import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
@@ -372,9 +373,9 @@ async function main(): Promise<void> {
       // TTY 模式:用 live display 替换 Console reporter,把 attempt log 路由到状态表行尾
       const liveRows: LiveRow[] = [];
       for (const agentRun of agentRuns) {
-        const who = agentRun.model
-          ? `${agentRun.agent.name}/${agentRun.model}`
-          : agentRun.agent.name;
+        // who 必须与 attempt.ts 的进度上报同源(runWho):曾用 agent/model,同 agent 同 model
+        // 的实验变体(xxx 与 xxx--agents-md)会被折叠成一行,0/2 看起来像同一 eval 跑两次。
+        const who = runWho(agentRun);
         const matched = evals.filter((e) => agentRun.evalFilter(e.id));
         for (const evalDef of matched) {
           liveRows.push({ evalId: evalDef.id, who, total: agentRun.runs });

package/src/i18n/en.ts

@@ -156,6 +156,7 @@ export const en = {
   "runner.timeout": "attempt timed out ({{timeoutMs}}ms)\nRecent progress:\n{{recentLogs}}",
   "runner.traceSelected": " -> kept {{count}} semantic spans",
   "runner.resumeCarry": "  · reusing {{carried}} passing results from last run, re-running {{retry}} evals\n",
+  "runner.resumeCarryDetail": "      carried [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
   "sandbox.missingSpec": "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").",

package/src/i18n/zh-CN.ts

@@ -153,6 +153,7 @@ export const zhCN = {
   "runner.timeout": "attempt 超时({{timeoutMs}}ms)\n最近进度:\n{{recentLogs}}",
   "runner.traceSelected": " → 留 {{count}}(按语义)",
   "runner.resumeCarry": "  · 复用上次 {{carried}} 个通过的结果,重跑 {{retry}} 个 eval\n",
+  "runner.resumeCarryDetail": "      复用 [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "使用 remote agent(不创建沙箱)…",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend not implemented; use docker, vercel, or e2b",
   "sandbox.missingSpec": "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。",

package/src/runner/attempt.ts

@@ -36,6 +36,7 @@ import type {
   Telemetry,
   TraceSpan,
 } from "../types.ts";
+import { runWho } from "./types.ts";
 import type { AgentRun, Attempt, RunOptions } from "./types.ts";
 
 export function runAttemptEffect(
@@ -72,7 +73,7 @@ export function runAttemptEffect(
 
   // 流式进度打到宿主 stderr(结果走 stdout,互不干扰)。容器主日志【不】放这些进度标记 ——
   // 那里留给 agent 的原始输出(adapter 给 agent 命令开 { stream: true })。
-  const who = run.model ? `${run.agent.name}/${run.model}` : run.agent.name;
+  const who = runWho(run);
   // 同时保留最近 20 条进度消息,timeout 时嵌入 error 字段方便定位卡在哪一步。
   const recentLogs: string[] = [];
   const log = (m: string) => {

package/src/runner/run.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from "vitest";
+import { judgeProbeTargets } from "./run.ts";
+import type { JudgeConfig } from "../types.ts";
+
+// judge 预检的目标收敛:只探测「实际要跑、且源码里出现 judge 字样」的 eval 的生效配置。
+// 这是对 memory/judge-config-precheck-hard-fails-without-key 的修复守护——
+// 全局配了 judge 但选中的 eval 都不用时,不能再因 judge key / 端点问题拦下整次运行。
+describe("judgeProbeTargets", () => {
+  const configJudge: JudgeConfig = { model: "gpt-5.4" };
+
+  it("skips probing when no selected eval mentions judge", () => {
+    const evals = [
+      { source: `t.check(t.reply, includes("2"));`, judge: undefined },
+      { source: `await t.sandbox.exec("pnpm test");`, judge: undefined },
+    ];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([]);
+  });
+
+  it("probes config-level judge when a selected eval mentions judge", () => {
+    const evals = [
+      { source: `t.judge.autoevals.closedQA("did it summarize?");`, judge: undefined },
+      { source: `t.check(t.reply, includes("ok"));`, judge: undefined },
+    ];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([configJudge]);
+  });
+
+  it("resolves eval-level judge over config-level, like attempt resolution", () => {
+    const evalJudge: JudgeConfig = { model: "deepseek-v4", baseUrl: "http://localhost:8787/v1" };
+    const evals = [{ source: `t.judge.autoevals.factuality("2")`, judge: evalJudge }];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([evalJudge]);
+  });
+
+  it("dedupes identical effective configs across evals", () => {
+    const evals = [
+      { source: `t.judge.autoevals.closedQA("a")`, judge: undefined },
+      { source: `t.judge.autoevals.closedQA("b")`, judge: undefined },
+    ];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([configJudge]);
+  });
+
+  it("returns nothing when judge is used but no config exists (runtime env fallback)", () => {
+    const evals = [{ source: `t.judge.autoevals.closedQA("a")`, judge: undefined }];
+    expect(judgeProbeTargets(evals, undefined)).toEqual([]);
+  });
+
+  it("does not match judge as part of a longer identifier", () => {
+    const evals = [{ source: `const prejudged = true;`, judge: undefined }];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([]);
+  });
+});

package/src/runner/run.ts

@@ -2,6 +2,7 @@
 // 职责只有编排:指纹缓存在 fingerprint.ts,单 attempt 生命周期在 attempt.ts,
 // reporter 编排 / 汇总在 report.ts,Sandbox 适配器在 remote-sandbox.ts。
 
+import { readFile } from "node:fs/promises";
 import { Effect, Cause, Duration, Exit } from "effect";
 import { probeJudge } from "../scoring/judge.ts";
 import { t } from "../i18n/index.ts";
@@ -14,34 +15,47 @@ import type { Attempt, RunOptions } from "./types.ts";
 
 export type { AgentRun, RunOptions } from "./types.ts";
 
+/** 收集本次要探测的 judge 配置:只看「实际要跑、且源码里出现 judge 字样」的 eval 的生效
+ *  配置(evalDef.judge ?? config.judge,与 attempt.ts 的 resolveJudge 一致),按
+ *  model|baseUrl|apiKeyEnv 去重。要跑的 eval 都不用 judge 时返回空 —— 全局配了 judge
+ *  也不探测,纯确定性断言的运行不再被 judge key / 端点问题拦下。
+ *  源码扫描是启发式:judge 调用藏在 import 的 helper 里时会漏判,漏判只是退回旧行为
+ *  (评分时才报 judge 错误,损失 fail fast),不影响正确性。 */
+export function judgeProbeTargets(
+  evals: Array<{ source: string; judge: JudgeConfig | undefined }>,
+  configJudge: JudgeConfig | undefined,
+): JudgeConfig[] {
+  const seen = new Set<string>();
+  const toProbe: JudgeConfig[] = [];
+  for (const e of evals) {
+    const jc = e.judge ?? configJudge;
+    if (!jc || !/\bjudge\b/.test(e.source)) continue;
+    const key = `${jc.model ?? ""}|${jc.baseUrl ?? ""}|${jc.apiKeyEnv ?? ""}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    toProbe.push(jc);
+  }
+  return toProbe;
+}
+
 export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   const startedAt = new Date().toISOString();
   const t0 = Date.now();
 
-  // 预检 judge:有显式配置时在第一步验证 API key + 端点可达,避免跑完 agent 才发现 judge 不通。
-  {
-    const seen = new Set<string>();
-    const toProbe: JudgeConfig[] = [];
-    for (const jc of [opts.config.judge, ...opts.evals.map((e) => e.judge)]) {
-      if (!jc) continue;
-      const key = `${jc.model ?? ""}|${jc.baseUrl ?? ""}|${jc.apiKeyEnv ?? ""}`;
-      if (seen.has(key)) continue;
-      seen.add(key);
-      toProbe.push(jc);
+  // 按 sourcePath 缓存文件内容,fingerprint 与 judge 预检共用:
+  // 矩阵大时(实验 × eval)规划阶段不做串行重复文件读。
+  const sourceCache = new Map<string, Promise<string>>();
+  const readSource = (path: string): Promise<string> => {
+    let p = sourceCache.get(path);
+    if (!p) {
+      p = readFile(path, "utf-8");
+      sourceCache.set(path, p);
     }
-    if (toProbe.length > 0) {
-      process.stderr.write(t("runner.judgePrecheck"));
-      for (const jc of toProbe) {
-        const err = await probeJudge(jc, opts.signal);
-        if (err) throw new Error(err);
-      }
-    }
-  }
+    return p;
+  };
 
   const plannedFingerprints = new Map<string, string>();
   {
-    // 并行 + 按 sourcePath 缓存:矩阵大时(实验 × eval)规划阶段不做串行重复文件读。
-    const sourceCache = new Map<string, Promise<string>>();
     const jobs: Promise<void>[] = [];
     for (const run of opts.agentRuns) {
       for (const evalDef of opts.evals.filter((e) => run.evalFilter(e.id))) {
@@ -100,9 +114,39 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     }
   }
 
+  // 预检 judge:验证 API key + 端点可达,避免跑完 agent 才发现 judge 不通。
+  // 放在 attempts 展开之后,fail fast 只对会真正触发 judge 的运行生效
+  // (目标收集逻辑见 judgeProbeTargets;全部结果携入、attempts 为空时也自然跳过)。
+  {
+    const uniqueEvals = [...new Map(attempts.map((a) => [a.evalDef.id, a.evalDef])).values()];
+    const sources = await Promise.all(uniqueEvals.map((e) => readSource(e.sourcePath)));
+    const toProbe = judgeProbeTargets(
+      uniqueEvals.map((e, i) => ({ source: sources[i] ?? "", judge: e.judge })),
+      opts.config.judge,
+    );
+    if (toProbe.length > 0) {
+      process.stderr.write(t("runner.judgePrecheck"));
+      for (const jc of toProbe) {
+        const err = await probeJudge(jc, opts.signal);
+        if (err) throw new Error(err);
+      }
+    }
+  }
+
   if (carriedResults.length > 0) {
     const retryCount = new Set(attempts.map((a) => `${a.run.experimentId ?? ""}|${a.evalDef.id}`)).size;
     process.stderr.write(t("runner.resumeCarry", { carried: carriedResults.length, retry: retryCount }));
+    // 按 experiment 分组列出被复用(跳过)的 eval:不列清单的话,用户只看到数量,
+    // 无法核对「跳过的是不是我以为已经过了的那些」。同一 key 多个 run 去重。
+    const carriedByExperiment = new Map<string, Set<string>>();
+    for (const r of carriedResults) {
+      const ids = carriedByExperiment.get(r.experimentId!) ?? new Set<string>();
+      ids.add(r.id);
+      carriedByExperiment.set(r.experimentId!, ids);
+    }
+    for (const [experiment, ids] of [...carriedByExperiment].sort(([a], [b]) => a.localeCompare(b))) {
+      process.stderr.write(t("runner.resumeCarryDetail", { experiment, evals: [...ids].sort().join(", ") }));
+    }
   }
 
   // onRunStart 报「本次实际要跑的 eval」(过滤 + 去重),不是发现到的全部 —— 否则计数误导。

package/src/runner/types.ts

@@ -211,6 +211,16 @@ export interface PriceOverride {
 
 // ───────────────────────── 调度编排 ─────────────────────────
 
+/**
+ * 进度行 / 日志里标识一个 run 配置的短名。有 experiment 时用其 basename(唯一,
+ * 能区分同 agent 同 model 的实验变体,如 xxx 与 xxx--agents-md;与汇总表口径一致);
+ * 无 experiment 时退回 agent/model。live display 以它作行聚合 key,两处必须同源。
+ */
+export function runWho(run: Pick<AgentRun, "agent" | "model" | "experimentId">): string {
+  if (run.experimentId) return run.experimentId.split("/").pop()!;
+  return run.model ? `${run.agent.name}/${run.model}` : run.agent.name;
+}
+
 /** 一个 (agent, model, flags) 的运行配置 —— 由 CLI / 实验展开。 */
 export interface AgentRun {
   agent: Agent;