niceeval 0.6.1 → 0.7.0

package/src/context/context.test.ts

@@ -1,6 +1,8 @@
+// cases: docs/engineering/unit-tests/eval/cases.md
 import { describe, expect, it } from "vitest";
 import { createEvalContext, type ContextState } from "./context.ts";
-import { includes } from "../expect/index.ts";
+import { commandSucceeded, includes } from "../expect/index.ts";
+import { completeCoverage, resolveAgentCoverage } from "../scoring/coverage.ts";
 import type { Agent, AgentContext, InputRequest, Sandbox, StreamEvent, Turn, TurnInput } from "../types.ts";
 
 // 计算工具 + 最终回复"1 + 1 = **2** 哦!😊"——复现截图里的场景:助手回复明明包含 "2",
@@ -10,6 +12,7 @@ function calculatorAgent(): Agent {
     name: "calculator",
     // 测试注入了真实的 fake sandbox,kind: "sandbox" 让 t.sandbox 过沙箱能力守卫。
     kind: "sandbox",
+    coverage: completeCoverage,
     async send(_input: TurnInput, ctx: AgentContext): Promise<Turn> {
       ctx.session.capture("sess-1");
       const events: StreamEvent[] = [
@@ -84,11 +87,12 @@ describe("createEvalContext / TestContext live state", () => {
       scripts: state.late.scripts,
       usage: { inputTokens: 0, outputTokens: 0 },
       status: "completed",
+      coverage: resolveAgentCoverage(completeCoverage),
       readFile: async () => undefined,
     });
-    expect(result.passed).toBe(true);
-    expect(result.score).toBe(1);
-    expect(result.evidence).toBeUndefined();
+    expect(result.outcome).toBe("passed");
+    expect(result.outcome === "passed" ? result.score : -1).toBe(1);
+    expect(result.outcome === "passed" ? result.evidence : "?").toBeUndefined();
   });
 
   it("t.check(...) attaches the actually-checked value as evidence when it fails", async () => {
@@ -103,10 +107,41 @@ describe("createEvalContext / TestContext live state", () => {
       scripts: state.late.scripts,
       usage: { inputTokens: 0, outputTokens: 0 },
       status: "completed",
+      coverage: resolveAgentCoverage(completeCoverage),
       readFile: async () => undefined,
     });
-    expect(result.passed).toBe(false);
-    expect(result.evidence).toBe("1 + 1 = **2** 哦!😊");
+    expect(result.outcome).toBe("failed");
+    expect(result.outcome === "failed" ? result.received : undefined).toBe("1 + 1 = **2** 哦!😊");
+  });
+
+  it("t.check(CommandResult, …) 失败时 received 塌成退出码+输出尾部,evidence 是命令行", async () => {
+    const { context, state } = makeContext(calculatorAgent());
+    const commandResult = {
+      stdout: `\n> test\n> pytest tests/\n${"collecting …\n".repeat(200)}1 failed, 5 passed in 12.72s\n`,
+      stderr: "",
+      exitCode: 1,
+      command: "npm run test",
+    };
+    context.check(commandResult, commandSucceeded());
+
+    const [result] = await state.collector.finalize({
+      events: [],
+      facts: { toolCalls: [], subagentCalls: [], inputRequests: [], parked: false, messageCount: 0, compactions: 0 },
+      diff: state.late.diff,
+      scripts: state.late.scripts,
+      usage: { inputTokens: 0, outputTokens: 0 },
+      status: "completed",
+      coverage: resolveAgentCoverage(completeCoverage),
+      readFile: async () => undefined,
+    });
+    expect(result.outcome).toBe("failed");
+    if (result.outcome !== "failed") return;
+    const [firstLine] = result.received!.split("\n");
+    expect(firstLine).toMatch(/^exit 1 · "…/);
+    expect(firstLine).toContain("1 failed, 5 passed in 12.72s"); // 摘要面只保留的这一行自含失败计数
+    expect(firstLine!.length).toBeLessThan(200);
+    expect(result.received).toContain("output tail:"); // 更长尾部保留换行,attempt 首页展开
+    expect(result.evidence).toBe("npm run test");
   });
 
   it("t.events reflects the turn's events after send(), not an empty snapshot", async () => {
@@ -147,6 +182,7 @@ function scriptedAgent(turns: Turn[]): Agent & { received: TurnInput[] } {
   const agent: Agent = {
     name: "scripted",
     kind: "remote",
+    coverage: completeCoverage,
     async send(input: TurnInput) {
       received.push(input);
       const turn = turns[Math.min(i, turns.length - 1)] as Turn;
@@ -208,6 +244,8 @@ describe("t.respond() / t.respondAll(): structured InputResponse", () => {
     await context.send("apply two edits");
 
     await expect(context.respond("approve")).rejects.toThrow(/字符串回答无法对位|cannot be matched/);
+    // 报错但响应已发出同样违约:agent 只收到最初那一次 send,没有第二次输入。
+    expect(agent.received).toHaveLength(1);
   });
 
   it("object form { request, optionId } disambiguates when multiple requests are pending", async () => {
@@ -299,6 +337,7 @@ function baseScoringContext(state: ContextState) {
     scripts: state.late.scripts,
     usage: { inputTokens: 0, outputTokens: 0 },
     status: "completed" as const,
+    coverage: resolveAgentCoverage(completeCoverage),
     readFile: async () => undefined,
   };
 }
@@ -314,7 +353,7 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
 
     const [result] = await state.collector.finalize(baseScoringContext(state));
     expect(result.name).toBe("parked");
-    expect(result.passed).toBe(true);
+    expect(result.outcome).toBe("passed");
   });
 
   it("turn.noFailedActions() looks only at this turn's own tool calls", async () => {
@@ -333,7 +372,7 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
 
     const [result] = await state.collector.finalize(baseScoringContext(state));
     expect(result.name).toBe("noFailedActions");
-    expect(result.passed).toBe(false);
+    expect(result.outcome).toBe("failed");
   });
 
   it("turn.maxTokens()/turn.maxCost() read this turn's own Turn.usage, not the session total", async () => {
@@ -347,9 +386,9 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
 
     const [tokens, cost] = await state.collector.finalize(baseScoringContext(state));
     expect(tokens.name).toBe("maxTokens(1000)");
-    expect(tokens.passed).toBe(true);
+    expect(tokens.outcome).toBe("passed");
     expect(cost.name).toBe("maxCost(0.01)");
-    expect(cost.passed).toBe(false);
+    expect(cost.outcome).toBe("failed");
   });
 
   it("turn.loadedSkill() reads the skill.loaded event scoped to this turn", async () => {
@@ -364,7 +403,7 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
     turn.loadedSkill("pdf-export");
 
     const [result] = await state.collector.finalize(baseScoringContext(state));
-    expect(result.passed).toBe(true);
+    expect(result.outcome).toBe("passed");
   });
 
   // Skill 加载是一等事件,不是「名字叫 load_skill 的工具调用」——adapter 负责归一
@@ -382,6 +421,6 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
     turn.loadedSkill("pdf-export");
 
     const [result] = await state.collector.finalize(baseScoringContext(state));
-    expect(result.passed).toBe(false);
+    expect(result.outcome).toBe("failed");
   });
 });

package/src/context/context.ts

@@ -4,13 +4,15 @@
 import { readFile } from "node:fs/promises";
 import { basename, extname } from "node:path";
 import { SessionManager, RunSession, lastAssistantText } from "./session.ts";
-import { AssertionCollector, computePassed } from "../scoring/collector.ts";
+import { AssertionCollector, computePassed, unavailable } from "../scoring/collector.ts";
+import type { ResolvedCoverage } from "../scoring/coverage.ts";
 import { deepEqual, validateSchema } from "../scoring/match.ts";
 import type { Spec } from "../scoring/collector.ts";
 import * as Scoped from "../scoring/scoped.ts";
 import { buildJudge } from "../scoring/judge.ts";
 import { EvalSkipped, EvalRequirementFailed, TurnFailed } from "./control-flow.ts";
 import { deriveRunFacts } from "../o11y/derive.ts";
+import { diffIsEmpty, diffMatches, emptyDiffData } from "../scoring/diff.ts";
 import { t } from "../i18n/index.ts";
 import { resolveLocalPath } from "../sandbox/paths.ts";
 import { brief } from "../util.ts";
@@ -73,6 +75,15 @@ export interface ContextDeps {
   otel?: import("../o11y/otlp/turn-otel.ts").AgentOtelChannel;
   /** Eval definition directory; used to resolve host-side relative fixture paths. */
   evalBaseDir?: string;
+  /** runner 绑定的作用域反馈(t.progress / t.diagnostic 与 adapter ctx 共用实现);
+   *  省略时(测试直调)progress 退回 log、diagnostic 静默丢弃。 */
+  feedback?: import("../types.ts").ScopedFeedback;
+  /** adapter send 在飞时的通知(errored 归因到嵌套的 `agent.run` 阶段用);透传给 SessionManager。 */
+  onSendActive?: (active: boolean) => void;
+  /** 变更分类账的 send 窗口钩子(仅沙箱型);透传给 SessionManager(见 SessionDeps.ledgerHooks)。 */
+  ledgerHooks?: import("./session.ts").SessionDeps["ledgerHooks"];
+  /** 每轮 send 的墙钟包络回报(runner 挂 turn 时间树节点);透传给 SessionManager。 */
+  onTurn?: import("./session.ts").SessionDeps["onTurn"];
 }
 
 /**
@@ -80,7 +91,7 @@ export interface ContextDeps {
  * 这是唯一仍需要构造证据之外强制检查的能力——`t.sandbox.file`/`t.sandbox.fileChanged()` 等直接读沙箱
  * 文件系统,没有沙箱就没有东西可读,不报错会静默返回空结果。其余能力(多轮对话、
  * 工具断言……)都不再问卷式声明,由「做没做到」的构造证据决定,见
- * docs-site/zh/concepts/adapter.mdx「能力从哪来」一节。
+ * docs-site/zh/explanation/adapter.mdx「能力从哪来」一节。
  */
 function capabilityGuard(agentName: string, cap: string, method: string): () => never {
   return () => {
@@ -100,12 +111,16 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     log: deps.log,
     telemetry: deps.telemetry,
     otel: deps.otel,
+    feedback: deps.feedback,
+    onSendActive: deps.onSendActive,
+    onTurn: deps.onTurn,
+    ledgerHooks: deps.ledgerHooks,
   });
   const collector = new AssertionCollector();
   const state: ContextState = {
     collector,
     manager,
-    late: { diff: { generatedFiles: {}, deletedFiles: [] }, scripts: {} },
+    late: { diff: emptyDiffData(), scripts: {} },
   };
 
   async function resolveValue(value: unknown, sc: ScoringContext): Promise<unknown> {
@@ -113,8 +128,36 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     return value;
   }
 
-  /** 断言失败时给 view 看的「实际被检查了什么」,而不是重复 matcher 自己的名字。 */
+  /** CommandResult 形状的值(duck-type,不 import sandbox 域):received 按「退出码 + 输出尾部」投影。 */
+  function asCommandResult(value: unknown): { stdout: string; stderr: string; exitCode: number; command?: string } | undefined {
+    if (!value || typeof value !== "object") return undefined;
+    const v = value as { stdout?: unknown; stderr?: unknown; exitCode?: unknown };
+    return typeof v.stdout === "string" && typeof v.stderr === "string" && typeof v.exitCode === "number"
+      ? (value as { stdout: string; stderr: string; exitCode: number; command?: string })
+      : undefined;
+  }
+
+  /** 断言失败时给 view 看的「实际被检查了什么」,而不是重复 matcher 自己的名字。
+   *  按值的形状落成人可读事实(而不是留一坨 JSON 给渲染层解析):CommandResult 的第一行是
+   *  `exit N · "…输出尾部摘要"`(stdout+stderr 合并折单行,信号常收在末尾——pytest / vitest
+   *  的 failed 计数都在最后几行;榜单与 --eval 标注这类单行面只保留这一行),随后附原样保留
+   *  换行的更长尾部——runner 不另存 eval 侧命令的输出,这条记录就是它唯一的家,attempt 首页
+   *  与 result.json 靠它给出「更进一步」;文件引用带 `// path` 头;其余走通用 JSON 预览。 */
   function previewCheckedValue(value: unknown): string {
+    const cmd = asCommandResult(value);
+    if (cmd) {
+      const combined = `${cmd.stdout}\n${cmd.stderr}`.trim();
+      const folded = combined.replace(/\s+/g, " ").trim();
+      const summary = folded.length > 160 ? `…${folded.slice(-159)}` : folded;
+      const headline = summary.length > 0 ? `exit ${cmd.exitCode} · "${summary}"` : `exit ${cmd.exitCode}`;
+      if (combined.length <= 160) return headline;
+      let tail = combined.slice(-3600);
+      if (combined.length > 3600) {
+        const firstBreak = tail.indexOf("\n");
+        if (firstBreak >= 0) tail = tail.slice(firstBreak + 1); // 不从半行开始
+      }
+      return `${headline}\noutput tail:\n${tail}`;
+    }
     if (value && typeof value === "object" && typeof (value as { path?: unknown }).path === "string") {
       const content = (value as { content?: unknown }).content;
       if (typeof content === "string") return brief(`// ${(value as { path: string }).path}\n${content}`, 4000);
@@ -122,13 +165,17 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     return brief(value, 4000);
   }
 
+  /** 失败断言的 evidence:被检查值自带命令摘要(CommandResult.command)时就是「命令行本身」。 */
+  function checkedValueEvidence(value: unknown): string | undefined {
+    const command = asCommandResult(value)?.command;
+    return typeof command === "string" && command.length > 0 ? command : undefined;
+  }
+
+  // agent 归因 diff 的只读视图:get = 最后触及窗口的终态;matches 扫触及路径与各窗口内容。
   const diffView: DiffView = {
-    get: (path) => state.late.diff.generatedFiles[path],
-    isEmpty: () =>
-      Object.keys(state.late.diff.generatedFiles).length === 0 &&
-      state.late.diff.deletedFiles.length === 0,
-    matches: (re) =>
-      Object.entries(state.late.diff.generatedFiles).some(([p, c]) => re.test(p) || re.test(c)),
+    get: (path) => state.late.diff.get(path),
+    isEmpty: () => diffIsEmpty(state.late.diff),
+    matches: (re) => diffMatches(state.late.diff, re),
   };
 
   const sandboxAssertions = {
@@ -168,6 +215,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     getEvents: () => readonly StreamEvent[],
     getStatus: () => "completed" | "failed" | "waiting",
     getUsage: () => Usage,
+    getCoverage: () => ResolvedCoverage,
   ) {
     return collector.record({
       ...spec,
@@ -179,6 +227,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
           facts: deriveRunFacts(events),
           status: getStatus(),
           usage: getUsage(),
+          coverage: getCoverage(),
         });
       },
     });
@@ -195,24 +244,34 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
   }
 
   function makeSessionHandle(session: RunSession): SessionHandle {
-    const scoped = (spec: Spec) =>
-      recordScoped(spec, () => session.events, () => session.lastStatus, () => session.usage);
+    // session 作用域 = 记录断言时快照(见 docs/feature/scoring/architecture/scopes.md):
+    // 之后该 session 再发生的轮次不改变这条断言的评估材料;只看最后一轮用 send() 的 TurnHandle。
+    const scoped = (spec: Spec) => {
+      const events = session.events.slice();
+      const status = session.lastStatus;
+      const usage = { ...session.usage };
+      const coverage = session.coverage;
+      return recordScoped(spec, () => events, () => status, () => usage, () => coverage);
+    };
 
     const handle: SessionHandle = {
-      send: async (text) => makeTurnHandle(await manager.send(session, text), collector, deps, text),
-      sendFile: async (path, text) =>
-        makeTurnHandle(await manager.send(session, text ?? "", [await readInputFile(path)]), collector, deps, text ?? ""),
+      send: async (input) => {
+        const text = typeof input === "string" ? input : input.text;
+        const files = typeof input === "string" ? undefined : input.files;
+        const turn = await manager.send(session, text, files);
+        return makeTurnHandle(turn, collector, deps, text, manager.resolveTurnCoverage(turn));
+      },
+      sendFile: async (path, text) => {
+        const turn = await manager.send(session, text ?? "", [await readInputFile(path)]);
+        return makeTurnHandle(turn, collector, deps, text ?? "", manager.resolveTurnCoverage(turn));
+      },
       requireInputRequest: (filter) => requireInputRequest(session, filter),
       respond: async (...answers) => {
         if (answers.length === 0) throw new Error(t("hitl.respondEmpty"));
         const built = buildRespondInput(session, answers);
         session.pendingInputRequests.length = 0;
-        return makeTurnHandle(
-          await manager.send(session, built.text, undefined, built.responses),
-          collector,
-          deps,
-          built.text,
-        );
+        const turn = await manager.send(session, built.text, undefined, built.responses);
+        return makeTurnHandle(turn, collector, deps, built.text, manager.resolveTurnCoverage(turn));
       },
       respondAll: async (optionId) => {
         if (session.pendingInputRequests.length === 0) {
@@ -226,7 +285,8 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
         }));
         session.pendingInputRequests.length = 0;
         const input = requests.map(() => optionId).join("\n");
-        return makeTurnHandle(await manager.send(session, input, undefined, responses), collector, deps, input);
+        const turn = await manager.send(session, input, undefined, responses);
+        return makeTurnHandle(turn, collector, deps, input, manager.resolveTurnCoverage(turn));
       },
       get reply() {
         return session.lastMessage;
@@ -251,7 +311,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
       notEvent: (type) => scoped(Scoped.notEventOfType(type)),
       calledSubagent: (name, match) => scoped(Scoped.calledSubagent(name, match)),
       eventOrder: (types) => scoped(Scoped.eventOrder(types)),
-      eventsSatisfy: (predicate, label) => scoped(Scoped.eventsSatisfy(predicate, label)),
+      eventsSatisfy: (label, predicate) => scoped(Scoped.eventsSatisfy(label, predicate)),
       maxTokens: (max) => scoped(Scoped.maxTokens(max)),
       maxCost: (usd) => scoped(Scoped.maxCost(usd)),
       get usage() {
@@ -266,6 +326,12 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
 
   const primary = makeSessionHandle(manager.primary);
 
+  // t 作用域 = 整个 attempt:全部 session(含 t.newSession() 开的)的全部轮次,finalize 时对
+  // 聚合结果求值(见 docs/feature/scoring/architecture/scopes.md)。newSession 的事件进入这里,
+  // 但不进入主 session 的即时 t.reply / t.events 读取视图;t.judge 默认材料仍是主 session 对话。
+  const aggregateScoped = (spec: Spec) =>
+    recordScoped(spec, () => manager.allEvents, () => manager.lastStatus, () => manager.usage, () => manager.coverage);
+
   // 沙箱能力守卫:非沙箱型 agent(kind !== "sandbox")把文件系统类动作替换成「一调用就报清晰错误」。
   // 其余能力(多轮对话、工具断言……)不再问卷式声明——没接 ctx.session 续接存取器的 agent
   // 每轮各是新对话,没吐 action.* 事件的 agent 上正断言自然不命中,负断言按事件完整性证明
@@ -288,6 +354,12 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     model: deps.model,
     reasoningEffort: deps.reasoningEffort,
     flags: deps.flags,
+    // 作用域反馈:scope 固定为 eval.run(runner 按当前阶段归因,eval 不能冒充其它阶段)。
+    progress: (u: import("../types.ts").ProgressUpdate) =>
+      deps.feedback
+        ? deps.feedback.progress(u)
+        : deps.log(u.current !== undefined && u.total !== undefined ? `${u.message} (${u.current}/${u.total})` : u.message),
+    diagnostic: (d: import("../types.ts").DiagnosticInput) => deps.feedback?.diagnostic(d),
     log: deps.log,
     skip: (reason: string) => {
       if (reason.trim().length === 0) throw new Error(t("context.skipEmpty"));
@@ -297,16 +369,22 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
 
     check: (value: unknown, assertion: ValueAssertion) => {
       // evaluate 读 spec 自己的 severity/threshold(而不是捕获记录时的快照):
-      // 句柄的 .gate()/.atLeast() 会事后改写 spec,evidence 判定必须和 finalize 同一口径。
+      // 句柄的 .gate()/.atLeast() 会事后改写 spec,received 判定必须和 finalize 同一口径。
       const spec: Spec = {
         name: assertion.name,
         severity: assertion.severity,
         threshold: assertion.threshold,
+        ...(assertion.isOptional ? { optional: true as const } : {}),
         evaluate: async (sc) => {
           const resolved = await resolveValue(value, sc);
           const score = await assertion.score(resolved);
           if (computePassed(spec.severity, spec.threshold, score)) return score;
-          return { score, evidence: previewCheckedValue(resolved) };
+          return {
+            score,
+            expected: assertion.expected,
+            received: previewCheckedValue(resolved),
+            ...(checkedValueEvidence(resolved) !== undefined ? { evidence: checkedValueEvidence(resolved) } : {}),
+          };
         },
       };
       return collector.record(spec);
@@ -321,13 +399,43 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
         name: assertion.name,
         severity: "gate",
         threshold: assertion.threshold,
-        evaluate: () => score,
+        evaluate: () =>
+          passed
+            ? score
+            : {
+                score,
+                expected: assertion.expected,
+                received: previewCheckedValue(v),
+                ...(checkedValueEvidence(v) !== undefined ? { evidence: checkedValueEvidence(v) } : {}),
+              },
       });
       if (!passed) throw new EvalRequirementFailed(assertion.name);
       return value;
     },
 
     sandbox: sandboxHandle,
+
+    // 作用域断言(t 级:聚合全部 session)。这些描述符盖过 primary 的同名方法——
+    // t.send/t.reply/t.events 仍是主 session 的即时视图,断言聚合与读取视图是两回事。
+    succeeded: () => aggregateScoped(Scoped.succeeded()),
+    parked: () => aggregateScoped(Scoped.parked()),
+    messageIncludes: (token: string | RegExp) => aggregateScoped(Scoped.messageIncludes(token)),
+    calledTool: (name: string, match?: import("../types.ts").ToolMatch) => aggregateScoped(Scoped.calledTool(name, match)),
+    notCalledTool: (name: string, match?: import("../types.ts").ToolMatch) => aggregateScoped(Scoped.notCalledTool(name, match)),
+    toolOrder: (names: string[]) => aggregateScoped(Scoped.toolOrder(names)),
+    usedNoTools: () => aggregateScoped(Scoped.usedNoTools()),
+    maxToolCalls: (max: number) => aggregateScoped(Scoped.maxToolCalls(max)),
+    loadedSkill: (skill: string) => aggregateScoped(Scoped.loadedSkill(skill)),
+    noFailedActions: () => aggregateScoped(Scoped.noFailedActions()),
+    event: (type: StreamEvent["type"], opts?: { count?: number }) => aggregateScoped(Scoped.eventOfType(type, opts)),
+    notEvent: (type: StreamEvent["type"]) => aggregateScoped(Scoped.notEventOfType(type)),
+    calledSubagent: (name: string, match?: import("../types.ts").SubagentMatch) =>
+      aggregateScoped(Scoped.calledSubagent(name, match)),
+    eventOrder: (types: StreamEvent["type"][]) => aggregateScoped(Scoped.eventOrder(types)),
+    eventsSatisfy: (label: string, predicate: (events: readonly StreamEvent[]) => boolean) =>
+      aggregateScoped(Scoped.eventsSatisfy(label, predicate)),
+    maxTokens: (max: number) => aggregateScoped(Scoped.maxTokens(max)),
+    maxCost: (usd: number) => aggregateScoped(Scoped.maxCost(usd)),
   };
   const context = Object.defineProperties(
     {},
@@ -364,7 +472,13 @@ function mimeTypeFor(path: string): string {
   }
 }
 
-function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: ContextDeps, input: string): TurnHandle {
+function makeTurnHandle(
+  turn: Turn,
+  collector: AssertionCollector,
+  deps: ContextDeps,
+  input: string,
+  coverage: ResolvedCoverage,
+): TurnHandle {
   const message = lastAssistantText(turn.events) ?? "";
   const facts = deriveRunFacts(turn.events);
   const usage = turn.usage ?? { inputTokens: 0, outputTokens: 0 };
@@ -379,6 +493,7 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
           facts,
           status: turn.status,
           usage,
+          coverage,
         }),
     });
 
@@ -404,13 +519,28 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
       collector.record({
         name: "outputEquals",
         severity: "gate",
-        evaluate: () => (deepEqual(turn.data, value) ? 1 : 0),
+        evaluate: () => {
+          if (deepEqual(turn.data, value)) return 1;
+          // 正断言:data 通道非 complete 且这一轮根本没给 data,「没采到」不能算成「没输出」。
+          if (turn.data === undefined && coverage.data.status !== "complete") {
+            const c = coverage.data;
+            return unavailable(`coverage:data=${c.status}${c.reason ? ` (${c.reason})` : ""}`);
+          }
+          return { score: 0, expected: brief(value, 800), received: brief(turn.data, 800) };
+        },
       }),
     outputMatches: (schema) =>
       collector.record({
         name: "outputMatches",
         severity: "gate",
-        evaluate: async () => ((await validateSchema(turn.data, schema)) ? 1 : 0),
+        evaluate: async () => {
+          if (await validateSchema(turn.data, schema)) return 1;
+          if (turn.data === undefined && coverage.data.status !== "complete") {
+            const c = coverage.data;
+            return unavailable(`coverage:data=${c.status}${c.reason ? ` (${c.reason})` : ""}`);
+          }
+          return { score: 0, received: brief(turn.data, 800) };
+        },
       }),
     messageIncludes: (token) => scoped(Scoped.messageIncludes(token)),
     succeeded: () => scoped(Scoped.succeeded()),
@@ -426,7 +556,7 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
     notEvent: (type) => scoped(Scoped.notEventOfType(type)),
     calledSubagent: (name, match) => scoped(Scoped.calledSubagent(name, match)),
     eventOrder: (types) => scoped(Scoped.eventOrder(types)),
-    eventsSatisfy: (predicate, label) => scoped(Scoped.eventsSatisfy(predicate, label)),
+    eventsSatisfy: (label, predicate) => scoped(Scoped.eventsSatisfy(label, predicate)),
     maxTokens: (max) => scoped(Scoped.maxTokens(max)),
     maxCost: (usd) => scoped(Scoped.maxCost(usd)),
     judge: buildJudge({
@@ -526,7 +656,9 @@ function inputRequestMatches(request: InputRequest, filter?: InputRequestFilter)
   if (filter.display !== undefined && !stringMatches(request.display ?? "", filter.display)) return false;
   if (filter.action !== undefined && !stringMatches(request.action ?? "", filter.action)) return false;
   if (filter.optionIds !== undefined) {
+    // 「恰好提供这组选项」:集合完全一致(顺序无关),不是子集包含——写少一个选项不算命中。
     const optionIds = new Set((request.options ?? []).map((o) => o.id));
+    if (optionIds.size !== filter.optionIds.length) return false;
     if (!filter.optionIds.every((id) => optionIds.has(id))) return false;
   }
   if (filter.input !== undefined && !partialObjectMatches(request.input, filter.input)) return false;

package/src/context/session.test.ts

@@ -1,10 +1,11 @@
+// cases: docs/engineering/unit-tests/eval/cases.md
 import { describe, expect, it } from "vitest";
 
 import { createAgentSession, SessionManager } from "./session.ts";
 import type { Agent, Sandbox, StreamEvent, Turn, TurnInput } from "../types.ts";
 
 // createAgentSession() 是 ctx.session 的实现——一条会话线的存取器(见
-// docs-site/zh/concepts/adapter.mdx 的 AgentSession 契约)。这里直接测存取器本身;
+// docs-site/zh/explanation/adapter.mdx 的 AgentSession 契约)。这里直接测存取器本身;
 // 端到端的「同一条线同一个 ctx.session」由 SessionManager / RunSession 保证。
 
 function fakeSandbox(): Sandbox {