niceeval 0.9.0 → 0.10.0

package/src/results/select.ts

@@ -16,7 +16,7 @@ import type {
   Snapshot,
 } from "./types.ts";
 import type { ExperimentRunInfo, JsonValue } from "../types.ts";
-import { evalPrefixPredicate } from "../shared/aggregate.ts";
+import { evalPrefixPredicate, matchExperimentSelector } from "../shared/aggregate.ts";
 
 /**
  * Results.latest() 的实现:每个实验取最新一次快照(= exp.snapshots[0]),生成挑选警告。
@@ -81,7 +81,7 @@ export function selectLatest(
 
 /** selectCurrentResults 的范围输入:experiment id 前缀与 eval id 前缀,都可缺省。 */
 export interface ResultScope {
-  /** experiment id 前缀(--experiment),分段匹配语义同 filterExperiments。 */
+  /** experiment id 前缀(--exp),分段匹配语义同 filterExperiments。 */
   experiment?: string | string[];
   /** eval id 前缀(位置参数),收窄 Scope 覆盖的 eval;覆盖警告分母同步收窄到范围内。 */
   patterns?: string[];
@@ -136,6 +136,15 @@ export function deepEqualJson(a: unknown, b: unknown): boolean {
   return false;
 }
 
+/**
+ * 一个快照实际选中的 eval id 全集:优先读落盘的 `experiment.selectedEvalIds`(niceeval 自己的
+ * 写入面必有此字段);第三方 harness 未实现该字段时退化为该快照实际写出的 `evals`,不把整份
+ * 来源排除在外(见 docs/feature/results/architecture.md「selectedEvalIds」)。
+ */
+export function selectedEvalIdsOf(snapshot: Snapshot): readonly string[] {
+  return snapshot.experiment?.selectedEvalIds ?? snapshot.evals.map((ev) => ev.id);
+}
+
 /**
  * 两个宿主(show / view)共用的现刻水位选择器:每个 experiment × eval 取「包含该 eval 的
  * 最新快照」里的全部 attempt,跨 run 合成。results.latest() 只挑「每实验最新快照」,带 eval
@@ -166,8 +175,12 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
     const taken = new Map<string, { ev: Eval; snapshot: Snapshot }>();
     for (const snapshot of exp.snapshots) {
       if (!deepEqualJson(comparabilityConfigOf(snapshot), baseline)) continue;
+      // 一个来源快照只贡献它自己选中的 eval——不在其 selectedEvalIds(或第三方退化后的实际
+      // evals)内的历史 attempt 不进入合成结果,即使它恰好出现在 snapshot.evals 里。第三方
+      // 无该字段时 selectedEvalIdsOf 退化为快照实际 evals,这里的过滤天然是 no-op。
+      const selectedIds = new Set(selectedEvalIdsOf(snapshot));
       for (const ev of snapshot.evals) {
-        if (!match(ev.id) || taken.has(ev.id)) continue;
+        if (!selectedIds.has(ev.id) || !match(ev.id) || taken.has(ev.id)) continue;
         taken.set(ev.id, { ev, snapshot });
       }
     }
@@ -184,12 +197,17 @@ export function selectCurrentResults(results: Results, scope: ResultScope = {}):
     }
     const evals = picks.map((p) => p.ev);
     const base = exp.latest;
+    // 合成快照的 selectedEvalIds 重建为最终 picks 的有序 id 列表——不是照抄某一来源快照
+    // (通常是最新快照)的局部选择。base.experiment 缺失(第三方无 ExperimentRunInfo)时不
+    // 伪造一份不完整投影,继续靠 report 侧 fallback(见 selectedEvalIdsOf)。
+    const experiment =
+      base.experiment !== undefined ? { ...base.experiment, selectedEvalIds: evals.map((ev) => ev.id) } : undefined;
     snapshots.push({
       experimentId: exp.id,
       startedAt,
       agent: base.agent,
       ...(base.model !== undefined ? { model: base.model } : {}),
-      ...(base.experiment !== undefined ? { experiment: base.experiment } : {}),
+      ...(experiment !== undefined ? { experiment } : {}),
       ...(base.name !== undefined ? { name: base.name } : {}),
       producer: base.producer,
       schemaVersion: base.schemaVersion,
@@ -368,12 +386,18 @@ export function isNewerSnapshot(a: Snapshot, b: Snapshot): boolean {
   return a.dir.localeCompare(b.dir) > 0;
 }
 
-/** experiment id 分段前缀过滤(--experiment / latest({ experiments }) 同一语义);包内使用,不进公共 barrel。 */
+/**
+ * experiment 选择器过滤(--exp / latest({ experiments }) 同一语义,与 `niceeval exp` 位置参数
+ * 共用 matchExperimentSelector,见 docs/feature/experiments/cli.md「实验选择器怎样解析」);
+ * 包内使用,不进公共 barrel。
+ */
 export function filterExperiments(experiments: Experiment[], filter?: string | string[]): Experiment[] {
   if (filter === undefined) return experiments;
   // 允许 "compare/" 这种带尾斜杠的写法,与 "compare" 等价;分段匹配不误配 "compare2"。
   const prefixes = (Array.isArray(filter) ? filter : [filter]).map((p) => p.replace(/\/+$/, ""));
-  return experiments.filter((exp) => prefixes.some((p) => exp.id === p || exp.id.startsWith(p + "/")));
+  const ids = experiments.map((exp) => exp.id);
+  const matched = new Set(prefixes.flatMap((p) => matchExperimentSelector(ids, p)));
+  return experiments.filter((exp) => matched.has(exp.id));
 }
 
 /**

package/src/runner/attempt.test.ts

@@ -100,7 +100,7 @@ async function runOnce(
     // 自定义 provider:create() 直接返回内存 fake,绕开真实沙箱 provider。
     sandbox: defineSandbox({ name: "fake-provider", create: async () => asSandbox(box) }),
     timeoutMs: 5_000,
-    evalFilter: () => true,
+    selectedEvalIds: [evalDef.id],
   };
   const attempt: Attempt = { evalDef, run, attempt: 0, key: "fake/eval", fingerprint: "" };
   const config: Config = {};
@@ -275,6 +275,53 @@ describe("runAttemptEffect · onPhase 回调随 enterPhase 同步触发", () =>
   });
 });
 
+// eval.teardown 的触发条件是「eval.setup 时点走到过」,不是「setup 声明且成功」(成对触发规则,
+// 见 docs/runner.md「环境预置不进运行器,但按顺序调它」)。时点在 attempt.ts 里于调用
+// evalDef.setup 之前就置位,所以 setup 抛错、乃至压根没声明 setup,都不豁免 teardown。
+describe("runAttemptEffect · eval.teardown 的触发规则", () => {
+  it("eval.setup 抛错时,eval.teardown 仍被调用(半初始化现场同样要扫尾)", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-eval-setup-throws",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    let teardownCalls = 0;
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, {
+      evalDefOverrides: {
+        setup: async () => {
+          throw new Error("boom-from-eval-setup");
+        },
+        teardown: async () => {
+          teardownCalls += 1;
+        },
+      },
+    });
+
+    expect(result.error?.message).toContain("boom-from-eval-setup");
+    expect(result.error?.phase).toBe("eval.setup");
+    expect(teardownCalls).toBe(1);
+  });
+
+  it("未声明 eval.setup 时,eval.teardown 依然触发(时点走到不依赖 setup 是否声明)", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-no-eval-setup",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    let teardownCalls = 0;
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, {
+      evalDefOverrides: {
+        teardown: async () => {
+          teardownCalls += 1;
+        },
+      },
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(teardownCalls).toBe(1);
+  });
+});
+
 describe("runAttemptEffect · 主链与 Scope 收尾的计时边界", () => {
   it("sandbox.stop 只计入收尾,主链 phase 合计不超过 durationMs", async () => {
     const agent = defineSandboxAgent({

package/src/runner/attempt.ts

@@ -8,6 +8,7 @@ import { readFile as readSourceFile } from "node:fs/promises";
 import { Effect, Cause, Duration } from "effect";
 import { createSandbox, resolveSandbox } from "../sandbox/resolve.ts";
 import { stopSandbox, unregisterSandbox } from "../sandbox/registry.ts";
+import { withCleanupTimeout } from "./cleanup-timeout.ts";
 import { KEEPABLE_PROVIDERS, nativeEnterCommand, suspendSandbox } from "../sandbox/keep.ts";
 import { keptEntryId, updateKeptEntry, writeKeptEntry } from "../sandbox/keep-registry.ts";
 import { createTraceReceiver, type TraceReceiver } from "../o11y/otlp/receiver.ts";
@@ -31,7 +32,6 @@ import type { CapturedEvalSource } from "./eval-source.ts";
 import type {
   AgentContext,
   AgentSetupManifest,
-  Cleanup,
   Config,
   DiagnosticInput,
   DiffArtifact,
@@ -456,7 +456,7 @@ export function runAttemptEffect(
  *  生命周期阶段(极早期就挂、还没跨进任何阶段时兜底 `eval.run`——phase 是必填字段,不留空);
  *  code 目前只对确定已知的类别赋稳定码,其余走 `"unexpected-error"`——provider 专属的限流码
  *  分类留在各 provider 的 `classifyProvisionError`,没有中性入口能在这里复算,不猜一个可能错的码。 */
-function errorFromThrown(e: unknown, phase: LifecyclePhase | undefined): AttemptError {
+export function errorFromThrown(e: unknown, phase: LifecyclePhase | undefined): AttemptError {
   const { message, stack, cause } = describeError(e);
   return {
     code: "unexpected-error",
@@ -551,22 +551,21 @@ async function runAttemptBody(
     progress: feedback.progress,
     diagnostic: feedback.diagnostic,
   };
-  let agentCleanup: Cleanup | void = undefined;
   let agentDidSetup = false;
+  /** agent.setup 时点已走到(未声明 setup 也置位)——agent.teardown 的触发条件(成对触发规则)。 */
+  let agentSetupReached = false;
   /** agent.setup 写进沙箱的安装清单(装了 Skill / plugin / MCP 的沙箱型 adapter 才有)。 */
   let agentSetup: AgentSetupManifest | undefined;
-  // EvalDef.setup() 返回的 cleanup 闭包;finally 里按 LIFO 跑(见下)。
-  let evalCleanup: Cleanup | void = undefined;
+  /** eval.setup 时点已走到(分类账锚点之后;未声明 setup 也置位)——eval.teardown 的触发条件。 */
+  let evalSetupReached = false;
   // 变更分类账(仅沙箱型;workspace.baseline 阶段建立)。
   let ledger: ChangeLedger | undefined;
-  // SandboxSpec.setup() 返回的 cleanup 闭包,按调用顺序收集;finally 里 LIFO 跑(见下)。
-  const sandboxCleanups: Cleanup[] = [];
   try {
     if (usesSandbox) {
       // 沙箱级生命周期钩子(SandboxSpec.setup):环境预置层,先于 workspace 上传 / git 基线 /
       // eval.setup 跑——改动进 git 基线,不会被误算进 agent 产出的 diff。按追加顺序依次执行;
-      // 单个抛错走下面的执行错误路径(与 eval.setup / agent.setup 同一条),已跑过的 cleanup /
-      // sandbox.teardown 钩子仍在 finally 里跑(见 catch/finally)。
+      // 单个抛错走下面的执行错误路径(与 eval.setup / agent.setup 同一条),后续 setup 不再执行,
+      // sandbox.teardown 钩子链仍在 finally 里完整跑(见 catch/finally)——半初始化的沙箱同样要扫尾。
       if (sandboxSetupHooks.length > 0) {
         enterPhase("sandbox.setup");
         log(t("runner.startSandboxSetup"));
@@ -582,8 +581,17 @@ async function runAttemptBody(
         });
         if (hookNode) recorder.pushParent(hookNode);
         try {
-          const cleanup = await hook(sandbox, hookCtx);
-          if (typeof cleanup === "function") sandboxCleanups.push(cleanup);
+          // 迁移护栏:tsx 用户没有类型检查,旧式「setup 返回 cleanup」会被静默忽略——命中即报
+          // 清晰错误(attempt errored),指向成对 teardown 写法。下方 eval.setup / agent.setup 同。
+          const returned = (await hook(sandbox, hookCtx)) as unknown;
+          if (typeof returned === "function") {
+            throw new Error(
+              t("runner.setupReturnedCleanup", {
+                layer: `SandboxSpec.setup() hook #${i}`,
+                hint: "SandboxSpec.teardown(fn)",
+              }).trimEnd(),
+            );
+          }
         } catch (e) {
           if (hookNode) hookNode.failed = true;
           throw e;
@@ -600,20 +608,41 @@ async function runAttemptBody(
       ledger = await createChangeLedger(sandbox, evalDef.diff);
 
       // eval 级 setup(starter prep:npm install / 装系统依赖等)。命令默认非 root;
-      // setup 里需要 root 的(apt/pip)自己传 { root: true }。
+      // setup 里需要 root 的(apt/pip)自己传 { root: true }。时点在此走到——
+      // teardown 的触发条件是时点,不是「setup 声明且成功」(成对触发规则,见
+      // docs/runner.md「环境预置不进运行器,但按顺序调它」)。
+      evalSetupReached = true;
       if (evalDef.setup) {
         enterPhase("eval.setup");
         log(t("runner.evalSetup"));
-        evalCleanup = await evalDef.setup(withEvalLocalPaths(sandbox, evalDef.baseDir), hookCtx);
+        const returned = (await evalDef.setup(withEvalLocalPaths(sandbox, evalDef.baseDir), hookCtx)) as unknown;
+        if (typeof returned === "function") {
+          throw new Error(
+            t("runner.setupReturnedCleanup", {
+              layer: `EvalDef.setup (${evalDef.id})`,
+              hint: "EvalDef.teardown",
+            }).trimEnd(),
+          );
+        }
       }
     }
 
     // agent 自己的 lifecycle:装 CLI、写 config(每个沙箱一次,不在每轮 send 里)。
+    // 时点在此走到(未声明 setup 也算)——agent.teardown 据此触发。
+    agentSetupReached = true;
     if (run.agent.setup) {
       enterPhase("agent.setup");
       log(t("runner.startAgentSetup"));
       agentDidSetup = true;
-      agentCleanup = await run.agent.setup(sandbox, attemptCtx);
+      const returned = (await run.agent.setup(sandbox, attemptCtx)) as unknown;
+      if (typeof returned === "function") {
+        throw new Error(
+          t("runner.setupReturnedCleanup", {
+            layer: `Agent.setup (${run.agent.name})`,
+            hint: "Agent.teardown",
+          }).trimEnd(),
+        );
+      }
     }
 
     // 安装 manifest:adapter 在 setup 收尾写进沙箱的固定路径,核心只把它抬成 attempt artifact
@@ -856,14 +885,17 @@ async function runAttemptBody(
     // (不计入 durationMs 口径,见 docs/feature/results/architecture.md)。执行序与 LifecyclePhase
     // 闭集声明一致:eval.teardown → agent.teardown → sandbox.teardown;各段可独立标 failed。
     // 沙箱 stop / 接收器 close 不在这里 —— 由 runAttemptEffect 的 Scope 在本函数返回后回收,
-    // 并经 finalizer 计成 sandbox.stop。没有对应 cleanup 的段直接跳过,不产生空阶段。
-    const evalCleanupFn = typeof evalCleanup === "function" ? evalCleanup : undefined;
-    if (evalCleanupFn) {
+    // 并经 finalizer 计成 sandbox.stop。触发规则统一是「同层 setup 时点走到过」(setup / test
+    // 抛错不豁免,见 docs/runner.md「环境预置不进运行器,但按顺序调它」);没有对应 teardown
+    // 的段直接跳过,不产生空阶段。
+    if (evalSetupReached && evalDef.teardown) {
       enterPhase("eval.teardown");
       await recorder
         .measureClosing("eval.teardown", async () => {
           try {
-            await evalCleanupFn();
+            // 收尾可调用体一律有界(docs/cli.md「中断:三级响应」的有界性前提):挂起的 teardown
+            // 到点按本段失败语义收束,后续段照常执行,收尾链不能无限拖住退出。下同。
+            await withCleanupTimeout(() => evalDef.teardown!(withEvalLocalPaths(sandbox, evalDef.baseDir), hookCtx));
           } catch (e) {
             // 收尾失败只是 diagnostic,不改判定 —— 挂到 attempt.diagnostics(见 finally 末尾并入)。
             diagnostics.push(teardownDiagnostic("eval.teardown", e));
@@ -872,14 +904,12 @@ async function runAttemptBody(
         })
         .catch(() => {});
     }
-    const agentCleanupFn = typeof agentCleanup === "function" ? agentCleanup : undefined;
-    if (agentCleanupFn !== undefined || (agentDidSetup && run.agent.teardown !== undefined)) {
+    if (agentSetupReached && run.agent.teardown) {
       enterPhase("agent.teardown");
       await recorder
         .measureClosing("agent.teardown", async () => {
           try {
-            if (agentCleanupFn) await agentCleanupFn();
-            if (agentDidSetup) await run.agent.teardown?.(sandbox, attemptCtx);
+            await withCleanupTimeout(() => run.agent.teardown!(sandbox, attemptCtx));
           } catch (e) {
             diagnostics.push(teardownDiagnostic("agent.teardown", e));
             throw e;
@@ -887,33 +917,22 @@ async function runAttemptBody(
         })
         .catch(() => {});
     }
-    if (usesSandbox && (sandboxCleanups.length > 0 || sandboxTeardownHooks.length > 0)) {
+    if (usesSandbox && sandboxTeardownHooks.length > 0) {
       enterPhase("sandbox.teardown");
       await recorder
         .measureClosing("sandbox.teardown", async () => {
           const before = diagnostics.length;
-          // sandbox.setup 返回的 cleanup:LIFO(后 setup 先 cleanup)。
-          for (let i = sandboxCleanups.length - 1; i >= 0; i--) {
+          // sandbox.teardown 钩子:按追加的逆序执行(LIFO),沙箱销毁前最后一步。setup 链中途
+          // 抛错不影响本链完整走完——半初始化的沙箱同样要扫尾,钩子自己对未初始化状态防御。
+          // 逐钩子有界:一个挂起的钩子不阻塞链上其余钩子拿到执行机会。
+          log(t("runner.startSandboxTeardown"));
+          for (let i = sandboxTeardownHooks.length - 1; i >= 0; i--) {
             try {
-              await sandboxCleanups[i]();
+              await withCleanupTimeout(() => sandboxTeardownHooks[i](sandbox, hookCtx));
             } catch (e) {
               diagnostics.push(teardownDiagnostic("sandbox.teardown", e));
             }
           }
-          // sandbox.teardown 钩子:按追加的逆序执行,沙箱销毁前最后一步。
-          if (sandboxTeardownHooks.length > 0) {
-            log(t("runner.startSandboxTeardown"));
-            for (let i = sandboxTeardownHooks.length - 1; i >= 0; i--) {
-              try {
-                const teardownCleanup = await sandboxTeardownHooks[i](sandbox, hookCtx);
-                // SandboxHook 类型允许返回 Cleanup(与 setup 同一签名);teardown 返回的
-                // cleanup 没有更晚的挂点,立即执行。
-                if (typeof teardownCleanup === "function") await teardownCleanup();
-              } catch (e) {
-                diagnostics.push(teardownDiagnostic("sandbox.teardown", e));
-              }
-            }
-          }
           if (diagnostics.length > before) throw new Error("sandbox teardown diagnostics");
         })
         .catch(() => {});
@@ -1017,17 +1036,18 @@ async function collectSources(
 
 /** 解析后运行配置的穷尽投影(ExperimentRunInfo,见 docs/feature/results/architecture.md):
  *  agent/model 只在快照顶层,这里不复制;sandbox 只经 provider 的公开参数投影落盘。 */
-function experimentRunInfo(run: AgentRun, configSandbox?: Config["sandbox"]): EvalResult["experiment"] {
+export function experimentRunInfo(run: AgentRun, configSandbox?: Config["sandbox"]): EvalResult["experiment"] {
   return {
     ...(run.description !== undefined ? { description: run.description } : {}),
     ...(run.reasoningEffort !== undefined ? { reasoningEffort: run.reasoningEffort } : {}),
     ...(Object.keys(run.flags).length > 0 ? { flags: run.flags } : {}),
+    ...(run.labels !== undefined && Object.keys(run.labels).length > 0 ? { labels: run.labels } : {}),
     runs: run.runs,
     earlyExit: run.earlyExit,
     ...(run.timeoutMs !== undefined ? { timeoutMs: run.timeoutMs } : {}),
     ...(run.budget !== undefined ? { budget: run.budget } : {}),
     ...(run.maxConcurrency !== undefined ? { maxConcurrency: run.maxConcurrency } : {}),
-    selectedEvalIds: run.selectedEvalIds ?? [],
+    selectedEvalIds: [...run.selectedEvalIds],
     ...(run.evalFilterFingerprint !== undefined ? { evalFilterFingerprint: run.evalFilterFingerprint } : {}),
     ...sandboxProjection(run, configSandbox),
   };

package/src/runner/cleanup-timeout.test.ts

@@ -0,0 +1,16 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+// (「cleanup 执行有界」行;sandbox/cases.md 的「收尾链每个可调用体各有清理超时」行同样由本文件证明机制)
+// bug: memory/force-exit-skips-experiment-teardown.md
+import { describe, expect, it } from "vitest";
+import { withCleanupTimeout } from "./cleanup-timeout.ts";
+
+describe("withCleanupTimeout", () => {
+  it("挂起的可调用体到点抛超时错误(消息含超时毫秒数)", async () => {
+    await expect(withCleanupTimeout(() => new Promise<never>(() => {}), 20)).rejects.toThrow(/20ms/);
+  });
+
+  it("按时完成的可调用体原样透传返回值;抛错原样透传,不被包装", async () => {
+    await expect(withCleanupTimeout(async () => "ok", 1_000)).resolves.toBe("ok");
+    await expect(withCleanupTimeout(() => Promise.reject(new Error("boom")), 1_000)).rejects.toThrow("boom");
+  });
+});

package/src/runner/cleanup-timeout.ts

@@ -0,0 +1,23 @@
+// 收尾可调用体的有界执行:eval/agent/sandbox 各段的 cleanup、钩子与实验级 cleanup 共用。
+// 有界性是「强清 = 加速收尾」设计的前提(docs/cli.md「中断:三级响应」):任何一个挂起的收尾
+// 都不能无限拖住退出,到点按该段自己的失败语义收束(teardown-failed / experiment-teardown-failed
+// 诊断),超时后遗留的 promise 悬空,随进程退出消亡。
+
+import { t } from "../i18n/index.ts";
+
+/** 单个收尾可调用体的清理超时(docs 声明值,provider stop 另有自己的 8s 超时)。 */
+export const CLEANUP_TIMEOUT_MS = 30_000;
+
+export async function withCleanupTimeout<T>(fn: () => Promise<T> | T, timeoutMs = CLEANUP_TIMEOUT_MS): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      Promise.resolve().then(fn),
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(() => reject(new Error(t("runner.cleanupTimeout", { timeoutMs }))), timeoutMs);
+      }),
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}

package/src/runner/discover.ts

@@ -103,8 +103,7 @@ export async function discoverExperiments(root: string): Promise<DiscoveredExper
       .replace(/\.experiment$/, "")
       .split(sep)
       .join("/");
-    const group = id.includes("/") ? id.split("/")[0]! : "";
-    out.push({ ...def, id, group });
+    out.push({ ...def, id });
   }
   return out;
 }

package/src/runner/eval-selection.test.ts

@@ -0,0 +1,187 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+// 纯选题边界:EvalDescriptor 投影 + resolveExperimentEvals 求值(定稿见
+// docs/feature/eval/library.md「EvalDescriptor」、docs/feature/experiments/library.md「evals」)。
+
+import { describe, expect, it } from "vitest";
+import { evalDescriptorOf, resolveExperimentEvals, selectedEvalsForRun } from "./eval-selection.ts";
+import type { DiscoveredEval, EvalDescriptor } from "./types.ts";
+
+const source = { path: "evals/fake.eval.ts", content: "export default { test() {} };\n", sha256: "fake" };
+
+function makeEval(id: string, overrides: Partial<DiscoveredEval> = {}): DiscoveredEval {
+  return {
+    id,
+    baseDir: "/project/evals",
+    sourcePath: `/project/evals/${id}.eval.ts`,
+    source,
+    test() {},
+    ...overrides,
+  };
+}
+
+describe("evalDescriptorOf", () => {
+  it("显式白名单投影:不暴露 sourcePath / baseDir / test 等内部字段", () => {
+    const evalDef = makeEval("coding/fix-button", {
+      description: "fix the button",
+      tags: ["coding", "frontend"],
+      environment: "node-22",
+      metadata: { owner: "team-a" },
+    });
+    const descriptor = evalDescriptorOf(evalDef);
+    expect(descriptor).toEqual({
+      id: "coding/fix-button",
+      description: "fix the button",
+      tags: ["coding", "frontend"],
+      environment: "node-22",
+      metadata: { owner: "team-a" },
+    });
+    expect(descriptor).not.toHaveProperty("sourcePath");
+    expect(descriptor).not.toHaveProperty("baseDir");
+    expect(descriptor).not.toHaveProperty("source");
+    expect(descriptor).not.toHaveProperty("test");
+  });
+
+  it("未声明 tags 得到只读空数组;冻结,mutation 不影响下一次读取", () => {
+    const evalDef = makeEval("bare/eval");
+    const d1 = evalDescriptorOf(evalDef);
+    expect(d1.tags).toEqual([]);
+    expect(Object.isFrozen(d1)).toBe(true);
+    expect(Object.isFrozen(d1.tags)).toBe(true);
+
+    // 每次投影都是新对象,一次调用拿到的引用被 mutate(TS 层本应拒绝,这里模拟无类型 JS)
+    // 不会影响下一次对同一 evalDef 的投影。
+    const mutable = d1 as unknown as { tags: string[] };
+    expect(() => mutable.tags.push("x")).toThrow(); // 冻结数组:push 直接抛错
+    const d2 = evalDescriptorOf(evalDef);
+    expect(d2.tags).toEqual([]);
+  });
+});
+
+describe("resolveExperimentEvals", () => {
+  const codingFixButton = makeEval("coding/fix-button", { tags: ["coding", "frontend"], environment: "node-22" });
+  const researchGpu = makeEval("research/gpu-literature", { tags: ["research"], environment: "gpu" });
+  const evals = [codingFixButton, researchGpu];
+
+  it("谓词同时可读 id / tags / environment / metadata,只命中匹配的 eval", () => {
+    const { selectedEvalIds } = resolveExperimentEvals({
+      experimentId: "exp/coding-only",
+      selector: (e) => e.id.startsWith("coding/") && e.tags.includes("coding") && e.environment !== "gpu",
+      cliPatterns: [],
+      evals,
+    });
+    expect(selectedEvalIds).toEqual(["coding/fix-button"]);
+  });
+
+  it('"*" 或省略 selector 全选;数组按裸前缀;CLI patterns 与 selector 取交集', () => {
+    expect(
+      resolveExperimentEvals({ experimentId: "e", selector: "*", cliPatterns: [], evals }).selectedEvalIds,
+    ).toEqual(["coding/fix-button", "research/gpu-literature"]);
+    expect(
+      resolveExperimentEvals({ experimentId: "e", selector: undefined, cliPatterns: [], evals }).selectedEvalIds,
+    ).toEqual(["coding/fix-button", "research/gpu-literature"]);
+    expect(
+      resolveExperimentEvals({ experimentId: "e", selector: ["coding/"], cliPatterns: [], evals }).selectedEvalIds,
+    ).toEqual(["coding/fix-button"]);
+    // selector 选中两条,CLI 追加前缀再收窄到一条:两层是交集。
+    expect(
+      resolveExperimentEvals({ experimentId: "e", selector: "*", cliPatterns: ["research/"], evals }).selectedEvalIds,
+    ).toEqual(["research/gpu-literature"]);
+  });
+
+  it("零命中时 selectedEvalIds 为空(不报错)", () => {
+    const { selectedEvals, selectedEvalIds } = resolveExperimentEvals({
+      experimentId: "e",
+      selector: () => false,
+      cliPatterns: [],
+      evals,
+    });
+    expect(selectedEvals).toEqual([]);
+    expect(selectedEvalIds).toEqual([]);
+  });
+
+  it("返回顺序 = discovery 稳定顺序,不随谓词命中顺序改变;id 去重", () => {
+    const a = makeEval("z/a");
+    const b = makeEval("a/b");
+    const { selectedEvalIds } = resolveExperimentEvals({
+      experimentId: "e",
+      selector: "*",
+      cliPatterns: [],
+      evals: [a, b],
+    });
+    expect(selectedEvalIds).toEqual(["z/a", "a/b"]); // discovery 顺序,不是字典序
+  });
+
+  it("每个候选 eval 只求值谓词恰好一次", () => {
+    let calls = 0;
+    resolveExperimentEvals({
+      experimentId: "e",
+      selector: () => {
+        calls += 1;
+        return true;
+      },
+      cliPatterns: [],
+      evals,
+    });
+    expect(calls).toBe(evals.length);
+  });
+
+  it("谓词返回非 boolean:报错携带 experiment id 与 eval id", () => {
+    expect(() =>
+      resolveExperimentEvals({
+        experimentId: "exp/bad-return",
+        selector: () => "yes" as unknown as boolean,
+        cliPatterns: [],
+        evals: [codingFixButton],
+      }),
+    ).toThrow(/exp\/bad-return.*coding\/fix-button/s);
+  });
+
+  it("谓词返回 Promise:不按 truthy 接受,报错说明必须同步", () => {
+    expect(() =>
+      resolveExperimentEvals({
+        experimentId: "exp/async",
+        selector: () => Promise.resolve(true) as unknown as boolean,
+        cliPatterns: [],
+        evals: [codingFixButton],
+      }),
+    ).toThrow(/Promise/);
+  });
+
+  it("谓词抛错:错误携带 experiment id + eval id,并保留原 cause", () => {
+    const original = new Error("boom");
+    let thrown: unknown;
+    try {
+      resolveExperimentEvals({
+        experimentId: "exp/throws",
+        selector: (): boolean => {
+          throw original;
+        },
+        cliPatterns: [],
+        evals: [codingFixButton],
+      });
+    } catch (e) {
+      thrown = e;
+    }
+    expect(thrown).toBeInstanceOf(Error);
+    expect((thrown as Error).message).toMatch(/exp\/throws/);
+    expect((thrown as Error).message).toMatch(/coding\/fix-button/);
+    expect((thrown as Error).cause).toBe(original);
+  });
+});
+
+describe("selectedEvalsForRun", () => {
+  it("按已解析的 selectedEvalIds 取 eval,保持 discovery 顺序", () => {
+    const a = makeEval("a");
+    const b = makeEval("b");
+    const c = makeEval("c");
+    const picked = selectedEvalsForRun([a, b, c], { selectedEvalIds: ["c", "a"] });
+    expect(picked.map((e) => e.id)).toEqual(["a", "c"]); // 顺序随 all,不随 selectedEvalIds 声明序
+  });
+});
+
+describe("EvalDescriptor 类型可从公开入口导入", () => {
+  it("类型层守卫:占位断言(真正的推断检查在 pnpm typecheck)", () => {
+    const descriptor: EvalDescriptor = { id: "x", tags: [] };
+    expect(descriptor.id).toBe("x");
+  });
+});