niceeval 0.6.1 → 0.6.2

package/src/scoring/judge.test.ts

@@ -0,0 +1,142 @@
+// cases: docs/engineering/unit-tests/scoring/cases.md
+// judge 解析与请求材料的单测:端点/凭据/模型解析结果必须进入真实请求,低分过不了 .gate()。
+// fixture judge client = 截获 globalThis.fetch(autoevals 底层 openai client 走全局 fetch),
+// 不起 HTTP server、不 spawn CLI。契约见 docs/feature/scoring/library/judge.md 与
+// docs-site/zh/concepts/judge.mdx 的解析优先级表;用例登记在
+// docs/engineering/unit-tests/scoring/cases.md 的 Judge 分区。
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { AssertionCollector } from "./collector.ts";
+import { buildJudge } from "./judge.ts";
+import { computeVerdict } from "./verdict.ts";
+import { resolveAgentCoverage, completeCoverage } from "./coverage.ts";
+import { emptyDiffData } from "./diff.ts";
+import { deriveRunFacts } from "../o11y/derive.ts";
+import type { JudgeConfig, ScoringContext } from "../types.ts";
+
+function ctx(): ScoringContext {
+  return {
+    events: [],
+    facts: deriveRunFacts([]),
+    diff: emptyDiffData(),
+    scripts: {},
+    usage: { inputTokens: 0, outputTokens: 0 },
+    status: "completed",
+    coverage: resolveAgentCoverage(completeCoverage),
+    readFile: async () => undefined,
+  };
+}
+
+interface CapturedRequest {
+  url: string;
+  authorization: string | null;
+  body: { model?: string; messages?: Array<{ role: string; content: string }> };
+}
+
+/** 截获全局 fetch:记录请求,回一个 ClosedQA 选 "N"(score 0)的 chat completion。 */
+function stubJudgeFetch(): CapturedRequest[] {
+  const captured: CapturedRequest[] = [];
+  vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+    const url = input instanceof Request ? input.url : String(input);
+    const headers = new Headers(input instanceof Request && !init?.headers ? input.headers : init?.headers);
+    const rawBody = init?.body ?? (input instanceof Request ? await input.text() : undefined);
+    captured.push({
+      url,
+      authorization: headers.get("authorization"),
+      body: typeof rawBody === "string" ? JSON.parse(rawBody) : {},
+    });
+    const payload = {
+      id: "chatcmpl-fixture",
+      object: "chat.completion",
+      created: 0,
+      model: "fixture",
+      choices: [
+        {
+          index: 0,
+          finish_reason: "tool_calls",
+          message: {
+            role: "assistant",
+            content: null,
+            tool_calls: [
+              {
+                id: "call_1",
+                type: "function",
+                function: {
+                  name: "select_choice",
+                  arguments: JSON.stringify({ choice: "N", reasons: "拒绝识图,答非所问" }),
+                },
+              },
+            ],
+          },
+        },
+      ],
+    };
+    return new Response(JSON.stringify(payload), { status: 200, headers: { "content-type": "application/json" } });
+  });
+  return captured;
+}
+
+function judgeWith(judge: JudgeConfig | undefined) {
+  const collector = new AssertionCollector();
+  const ns = buildJudge({
+    record: (spec) => collector.record(spec),
+    judge,
+    getOutput: () => "很抱歉,我目前使用的模型不支持图像输入,无法查看你发送的图片。",
+    getInput: () => "这张图片里有什么?主要是什么颜色?",
+  });
+  return { collector, ns };
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  vi.unstubAllEnvs();
+});
+
+describe("judge 端点/凭据/模型解析进入真实请求", () => {
+  it("NICEEVAL_JUDGE_BASE/KEY 落在请求 URL 与 Bearer 头;config model 压过 NICEEVAL_JUDGE_MODEL;score 0 过不了 .gate(0.7)", async () => {
+    vi.stubEnv("NICEEVAL_JUDGE_BASE", "http://judge.fixture.internal/v1");
+    vi.stubEnv("NICEEVAL_JUDGE_KEY", "fixture-key");
+    vi.stubEnv("NICEEVAL_JUDGE_MODEL", "env-model");
+    const captured = stubJudgeFetch();
+
+    const { collector, ns } = judgeWith({ model: "config-model" });
+    ns.autoevals.closedQA("助手是否描述了这张图片的内容,而不是答非所问?").gate(0.7);
+    const [result] = await collector.finalize(ctx());
+
+    expect(captured).toHaveLength(1);
+    expect(captured[0]!.url).toBe("http://judge.fixture.internal/v1/chat/completions");
+    expect(captured[0]!.authorization).toBe("Bearer fixture-key");
+    expect(captured[0]!.body.model).toBe("config-model");
+    // 请求材料:被评的 output 与 rubric 都要真的送到裁判面前。
+    const material = JSON.stringify(captured[0]!.body.messages ?? []);
+    expect(material).toContain("不支持图像输入");
+    expect(material).toContain("助手是否描述了这张图片的内容");
+
+    // 裁判给 0 分,.gate(0.7) 是硬要求:该条 failed,折叠后整个 attempt failed(与 --strict 无关)。
+    expect(result).toMatchObject({ severity: "gate", threshold: 0.7, outcome: "failed", score: 0 });
+    expect(computeVerdict({ assertions: [result!] })).toBe("failed");
+  });
+
+  it("单次 { model } 压过 judge config", async () => {
+    vi.stubEnv("NICEEVAL_JUDGE_KEY", "fixture-key");
+    const captured = stubJudgeFetch();
+
+    const { collector, ns } = judgeWith({ model: "config-model" });
+    ns.autoevals.closedQA("是否切题?", { model: "call-model" });
+    await collector.finalize(ctx());
+
+    expect(captured[0]!.body.model).toBe("call-model");
+  });
+
+  it("config 缺席时回落到 NICEEVAL_JUDGE_MODEL", async () => {
+    vi.stubEnv("NICEEVAL_JUDGE_KEY", "fixture-key");
+    vi.stubEnv("NICEEVAL_JUDGE_MODEL", "env-model");
+    const captured = stubJudgeFetch();
+
+    const { collector, ns } = judgeWith(undefined);
+    ns.autoevals.closedQA("是否切题?");
+    await collector.finalize(ctx());
+
+    expect(captured[0]!.body.model).toBe("env-model");
+  });
+});

package/src/scoring/judge.ts

@@ -6,7 +6,7 @@
 // closedQA / factuality / summarizes 直接用 autoevals 库(braintrust)。
 
 import { ClosedQA, Factuality, Summary } from "autoevals";
-import type { EvalScore } from "./collector.ts";
+import { unavailable, type EvalScore, type EvalUnavailable } from "./collector.ts";
 import type { AssertionHandle, AutoevalsNamespace, JudgeConfig, JudgeNamespace, ScoringContext } from "../types.ts";
 import { getEnv } from "../util.ts";
 import { t } from "../i18n/index.ts";
@@ -43,7 +43,7 @@ export interface JudgeDeps {
   record(spec: {
     name: string;
     severity: "soft";
-    evaluate(ctx: ScoringContext): Promise<EvalScore>;
+    evaluate(ctx: ScoringContext): Promise<EvalScore | EvalUnavailable>;
   }): AssertionHandle;
   judge: JudgeConfig | undefined;
   getOutput: () => string;
@@ -52,17 +52,6 @@ export interface JudgeDeps {
   signal?: AbortSignal;
 }
 
-/** 没解析到 judge key 时返回的 no-op 命名空间:judge 断言静默跳过(不记录)。 */
-function noOpJudge(): JudgeNamespace {
-  const handle: AssertionHandle = {
-    atLeast: () => handle,
-    gate: () => handle,
-  };
-  const skip = () => handle;
-  const noOpAutoevals: AutoevalsNamespace = { closedQA: skip, factuality: skip, summarizes: skip };
-  return { autoevals: noOpAutoevals };
-}
-
 /** 预检显式配置的 judge:验证 model + API key 存在,并发最小请求确认端点可达。
  *  返回错误描述字符串,可达则返回 undefined。*/
 export async function probeJudge(judge: JudgeConfig, signal?: AbortSignal): Promise<string | undefined> {
@@ -99,11 +88,12 @@ export async function probeJudge(judge: JudgeConfig, signal?: AbortSignal): Prom
   return undefined;
 }
 
-/** 构造 t.judge 命名空间。每个方法 record 一条延迟 soft 断言。 */
+/** 构造 t.judge 命名空间。每个方法 record 一条延迟 soft 断言。
+ *  没解析到模型或 API key 时【不静默、不抛错】:该条断言照常记录,finalize 时落成
+ *  `outcome: "unavailable"`(带机器可读 reason)——rubric 写了就必须留下记录,评不了的
+ *  结论按 Severity 与 Verdict 的折叠规则使 attempt errored(除非作者链 `.optional()`)。 */
 export function buildJudge(deps: JudgeDeps): JudgeNamespace {
   const resolved = resolveJudge(deps.judge);
-  // 没 key 就静默跳过 judge —— eval 不必再手动 gate「环境里有没有 judge key」。
-  if (!resolved.apiKey) return noOpJudge();
 
   const materialFor = async (ctx: ScoringContext, on?: string): Promise<string> => {
     if (on) {
@@ -124,16 +114,23 @@ export function buildJudge(deps: JudgeDeps): JudgeNamespace {
 
   // 三个 autoevals 方法只差评分器和材料字段名,共享行为(record spec / 材料构造 /
   // 分数归一 / evidence)单一出处。model 解析:单次 { model } → judge config →
-  // NICEEVAL_JUDGE_MODEL;都没有是配置错误,调用点即报(不静默跳过,会藏住误配)。
+  // NICEEVAL_JUDGE_MODEL;没解析到模型或 key 时该条记 unavailable(带 reason),
+  // 绝不静默消失、也不在调用点崩——评不了的折叠交给 Severity 与 Verdict 规则。
   const makeAutoeval =
     (kind: "closedQA" | "factuality" | "summarizes", scorer: Scorer, payloadKey: "criteria" | "expected") =>
     (reference: string, opts?: { on?: string; model?: string }) => {
       const model = opts?.model ?? resolved.model;
-      if (!model) throw new Error(t("judge.modelMissing"));
       return deps.record({
         name: `judge:autoevals:${kind}`,
         severity: "soft",
         evaluate: async (ctx) => {
+          if (!model) {
+            return unavailable("judge-model-unresolved (no model in config, NICEEVAL_JUDGE_MODEL unset)");
+          }
+          if (!resolved.apiKey) {
+            const envHint = deps.judge?.apiKeyEnv ?? "NICEEVAL_JUDGE_KEY / OPENAI_API_KEY";
+            return unavailable(`judge-key-unresolved (${envHint} unset)`);
+          }
           const output = await materialFor(ctx, opts?.on);
           const result = await scorer({
             input: deps.getInput(),

package/src/scoring/scoped.ts

@@ -1,8 +1,26 @@
 // 作用域断言:读标准事件流的派生事实(toolCalls / parked …)、diff、脚本结果。
 // 每个 builder 产一个延迟 Spec,context 负责 record。规则覆盖不到的奇怪断言可直接落 events。
-
-import type { Spec } from "./collector.ts";
-import type { DiffData, ScoringContext, StreamEvent, SubagentCall, SubagentMatch, ToolCall, ToolMatch } from "../types.ts";
+//
+// 证据覆盖的三值折叠(见 docs/feature/scoring/architecture/evidence.md):
+// - 正断言:找到匹配即通过(证据存在就是证据);没找到且所需通道非 complete(含 unknown)
+//   记 unavailable——「没采到」不能算成「Agent 没做」;complete 通道上没找到才是 failed。
+// - 负断言:找到反例即 failed(反例是确凿证据);没找到反例且通道非 complete 记 unavailable——
+//   空流证明不了「没发生」。
+// - 上限断言:实测已超限即 failed(partial 只会少采,超限是确凿的);未超限且通道非 complete
+//   记 unavailable——缺证据不能按零聚合。
+
+import { unavailable, type EvalUnavailable, type Spec } from "./collector.ts";
+import type { CoverageChannel } from "./coverage.ts";
+import type { ScoringContext, StreamEvent, SubagentCall, SubagentMatch, ToolCall, ToolMatch } from "../types.ts";
+
+// ── 覆盖折叠 ──
+
+/** 所需通道非 complete 时返回 unavailable(带机器可读 reason),complete 返回 undefined。 */
+function coverageGap(ctx: ScoringContext, channel: CoverageChannel): EvalUnavailable | undefined {
+  const c = ctx.coverage[channel];
+  if (c.status === "complete") return undefined;
+  return unavailable(`coverage:${channel}=${c.status}${c.reason ? ` (${c.reason})` : ""}`);
+}
 
 // ── 工具匹配小语言 ──
 
@@ -49,7 +67,7 @@ function toolMatches(tc: ToolCall, name: string, match?: ToolMatch): boolean {
   return true;
 }
 
-// ── evidence:把调用的出入参带回断言结果,view 展开可见,不用翻原始事件流 ──
+// ── received:把调用的出入参带回断言结果,view 展开可见,不用翻原始事件流 ──
 
 function briefJson(value: unknown, max = 800): string {
   let s: string;
@@ -95,18 +113,50 @@ function subagentMatches(call: SubagentCall, name: string, match?: SubagentMatch
   return true;
 }
 
+/** ToolMatch 的期望描述(`≥1 call matching input.city = "Brooklyn"` 之类)。 */
+function describeToolExpectation(name: string, match?: ToolMatch): string {
+  const conditions: string[] = [];
+  if (match?.input) {
+    for (const [k, v] of Object.entries(match.input)) conditions.push(`input.${k} = ${briefJson(v, 120)}`);
+  }
+  if (match?.status) conditions.push(`status = ${match.status}`);
+  const cond = conditions.length ? ` matching ${conditions.join(", ")}` : "";
+  const count = match?.count !== undefined ? `exactly ${match.count} calls of ${name}` : `≥1 call of ${name}`;
+  return `${count}${cond}`;
+}
+
 // ── builders ──
 
 export function succeeded(): Spec {
   return {
     name: "succeeded",
     severity: "gate",
-    evaluate: (ctx) => (ctx.status !== "failed" && !ctx.facts.parked ? 1 : 0),
+    evaluate: (ctx) => {
+      // status 通道非 complete(恒 completed 的映射)时,末态不可信,通过与失败都评不了。
+      const gap = coverageGap(ctx, "status");
+      if (gap) return gap;
+      const ok = ctx.status !== "failed" && !ctx.facts.parked;
+      if (ok) return 1;
+      return {
+        score: 0,
+        received: ctx.facts.parked
+          ? `${ctx.facts.inputRequests.length || 1} unanswered input request`
+          : `status: ${ctx.status}`,
+      };
+    },
   };
 }
 
 export function parked(): Spec {
-  return { name: "parked", severity: "gate", evaluate: (ctx) => (ctx.facts.parked ? 1 : 0) };
+  return {
+    name: "parked",
+    severity: "gate",
+    evaluate: (ctx) => {
+      const gap = coverageGap(ctx, "status");
+      if (gap) return gap;
+      return ctx.facts.parked ? 1 : { score: 0, received: `status: ${ctx.status} (no pending input request)` };
+    },
+  };
 }
 
 export function messageIncludes(token: string | RegExp): Spec {
@@ -120,8 +170,15 @@ export function messageIncludes(token: string | RegExp): Spec {
         .map((e) => e.text)
         .join("\n");
       const ok = token instanceof RegExp ? token.test(text) : text.includes(token);
-      // 失败时把实际被扫的助手文本带回来(和 t.check 的口径一致);命中时行色已说明一切。
-      return ok ? 1 : { score: 0, evidence: text ? (text.length > 4000 ? text.slice(0, 4000) + "…" : text) : undefined };
+      if (ok) return 1;
+      // 正断言:非 complete 通道上没找到记 unavailable,不判失败。
+      const gap = coverageGap(ctx, "messages");
+      if (gap) return gap;
+      return {
+        score: 0,
+        expected: token instanceof RegExp ? `matches ${token}` : `contains ${JSON.stringify(token)}`,
+        received: text ? (text.length > 4000 ? text.slice(0, 4000) + "…" : text) : "(no assistant messages)",
+      };
     },
   };
 }
@@ -133,11 +190,24 @@ export function calledTool(name: string, match?: ToolMatch): Spec {
     evaluate: (ctx) => {
       const matched = ctx.facts.toolCalls.filter((tc) => toolMatches(tc, name, match));
       const n = matched.length;
-      const score = match?.count !== undefined ? (n === match.count ? 1 : 0) : n >= 1 ? 1 : 0;
+      const ok = match?.count !== undefined ? n === match.count : n >= 1;
       // 命中给命中调用的出入参;没命中给同名调用(条件不满足的近失);再没有就列出实际调过的工具。
       const sameName = ctx.facts.toolCalls.filter((tc) => tc.name === name || tc.originalName === name);
       const shown = matched.length ? matched : sameName.length ? sameName : ctx.facts.toolCalls;
-      return { score, evidence: describeCalls(shown) };
+      if (ok) return { score: 1, received: describeCalls(shown) };
+      // 精确 count 且实测已超出:partial 只会少采,超出是确凿失败;其余未命中按覆盖折叠。
+      const definitiveOvershoot = match?.count !== undefined && n > match.count;
+      if (!definitiveOvershoot) {
+        const gap = coverageGap(ctx, "actions");
+        if (gap) return gap;
+      }
+      return {
+        score: 0,
+        expected: describeToolExpectation(name, match),
+        received:
+          describeCalls(shown) ??
+          `${ctx.facts.toolCalls.length} tool calls, none matching`,
+      };
     },
   };
 }
@@ -148,7 +218,11 @@ export function notCalledTool(name: string, match?: ToolMatch): Spec {
     severity: "gate",
     evaluate: (ctx) => {
       const matched = ctx.facts.toolCalls.filter((tc) => toolMatches(tc, name, match));
-      return { score: matched.length === 0 ? 1 : 0, evidence: describeCalls(matched) };
+      // 负断言:找到反例即 failed(证据确凿),没找到时空流证明不了「没发生」。
+      if (matched.length > 0) return { score: 0, received: describeCalls(matched) };
+      const gap = coverageGap(ctx, "actions");
+      if (gap) return gap;
+      return 1;
     },
   };
 }
@@ -162,8 +236,15 @@ export function toolOrder(names: string[]): Spec {
       for (const tc of ctx.facts.toolCalls) {
         if (i < names.length && (tc.name === names[i] || tc.originalName === names[i])) i++;
       }
+      if (i === names.length) return 1;
+      const gap = coverageGap(ctx, "actions");
+      if (gap) return gap;
       const actual = ctx.facts.toolCalls.map((tc) => tc.originalName ?? tc.name).join(" → ");
-      return { score: i === names.length ? 1 : 0, evidence: actual || undefined };
+      return {
+        score: 0,
+        expected: names.join(" → "),
+        received: actual ? `${actual} (missing ${names[i]})` : "(no tool calls)",
+      };
     },
   };
 }
@@ -172,10 +253,12 @@ export function usedNoTools(): Spec {
   return {
     name: "usedNoTools",
     severity: "gate",
-    evaluate: (ctx) => ({
-      score: ctx.facts.toolCalls.length === 0 ? 1 : 0,
-      evidence: describeCalls(ctx.facts.toolCalls),
-    }),
+    evaluate: (ctx) => {
+      if (ctx.facts.toolCalls.length > 0) return { score: 0, received: describeCalls(ctx.facts.toolCalls) };
+      const gap = coverageGap(ctx, "actions");
+      if (gap) return gap;
+      return 1;
+    },
   };
 }
 
@@ -183,10 +266,14 @@ export function maxToolCalls(max: number): Spec {
   return {
     name: `maxToolCalls(${max})`,
     severity: "gate",
-    evaluate: (ctx) => ({
-      score: ctx.facts.toolCalls.length <= max ? 1 : 0,
-      evidence: describeCalls(ctx.facts.toolCalls),
-    }),
+    evaluate: (ctx) => {
+      const n = ctx.facts.toolCalls.length;
+      // 上限断言:实测已超限是确凿失败;未超限时,partial 通道可能漏采,不能按不完整计数放行。
+      if (n > max) return { score: 0, expected: `≤ ${max} tool calls`, received: describeCalls(ctx.facts.toolCalls) };
+      const gap = coverageGap(ctx, "actions");
+      if (gap) return gap;
+      return 1;
+    },
   };
 }
 
@@ -199,9 +286,15 @@ export function loadedSkill(skill: string): Spec {
     evaluate: (ctx) => {
       const loaded = ctx.events.filter((e): e is Extract<StreamEvent, { type: "skill.loaded" }> => e.type === "skill.loaded");
       const matched = loaded.filter((e) => e.skill === skill);
+      if (matched.length) return { score: 1, received: matched.map((e) => e.skill).join(", ") };
+      const gap = coverageGap(ctx, "events");
+      if (gap) return gap;
       // 没命中时把实际加载过的 skill 列出来(常见失败是名字对不上,而不是一个都没加载)。
-      const shown = matched.length ? matched : loaded;
-      return { score: matched.length ? 1 : 0, evidence: shown.length ? shown.map((e) => e.skill).join(", ") : undefined };
+      return {
+        score: 0,
+        expected: `skill ${JSON.stringify(skill)} loaded`,
+        received: loaded.length ? loaded.map((e) => e.skill).join(", ") : "(no skills loaded)",
+      };
     },
   };
 }
@@ -213,8 +306,13 @@ export function noFailedActions(): Spec {
     evaluate: (ctx) => {
       const failedTools = ctx.facts.toolCalls.filter((tc) => tc.status === "failed");
       const failedSubs = ctx.facts.subagentCalls.filter((s) => s.status === "failed");
-      const evidence = [describeCalls(failedTools), describeSubagents(failedSubs)].filter(Boolean).join("\n") || undefined;
-      return { score: failedTools.length || failedSubs.length ? 0 : 1, evidence };
+      if (failedTools.length || failedSubs.length) {
+        const received = [describeCalls(failedTools), describeSubagents(failedSubs)].filter(Boolean).join("\n") || undefined;
+        return { score: 0, received };
+      }
+      const gap = coverageGap(ctx, "actions");
+      if (gap) return gap;
+      return 1;
     },
   };
 }
@@ -226,8 +324,14 @@ export function calledSubagent(name: string, match?: SubagentMatch): Spec {
     evaluate: (ctx) => {
       const matched = ctx.facts.subagentCalls.filter((call) => subagentMatches(call, name, match));
       const n = matched.length;
-      const score = match?.count !== undefined ? (n === match.count ? 1 : 0) : n >= 1 ? 1 : 0;
-      return { score, evidence: describeSubagents(matched.length ? matched : ctx.facts.subagentCalls) };
+      const ok = match?.count !== undefined ? n === match.count : n >= 1;
+      if (ok) return { score: 1, received: describeSubagents(matched) };
+      const definitiveOvershoot = match?.count !== undefined && n > match.count;
+      if (!definitiveOvershoot) {
+        const gap = coverageGap(ctx, "actions");
+        if (gap) return gap;
+      }
+      return { score: 0, received: describeSubagents(matched.length ? matched : ctx.facts.subagentCalls) };
     },
   };
 }
@@ -238,8 +342,18 @@ export function eventOfType(type: string, opts?: { count?: number }): Spec {
     severity: "gate",
     evaluate: (ctx) => {
       const n = ctx.events.filter((e) => e.type === type).length;
-      if (opts?.count !== undefined) return n === opts.count ? 1 : 0;
-      return n >= 1 ? 1 : 0;
+      const ok = opts?.count !== undefined ? n === opts.count : n >= 1;
+      if (ok) return 1;
+      const definitiveOvershoot = opts?.count !== undefined && n > opts.count;
+      if (!definitiveOvershoot) {
+        const gap = coverageGap(ctx, "events");
+        if (gap) return gap;
+      }
+      return {
+        score: 0,
+        expected: opts?.count !== undefined ? `exactly ${opts.count} × ${type}` : `≥1 × ${type}`,
+        received: `${n} × ${type}`,
+      };
     },
   };
 }
@@ -248,7 +362,13 @@ export function notEventOfType(type: string): Spec {
   return {
     name: `notEvent(${type})`,
     severity: "gate",
-    evaluate: (ctx) => (ctx.events.some((e) => e.type === type) ? 0 : 1),
+    evaluate: (ctx) => {
+      const hits = ctx.events.filter((e) => e.type === type);
+      if (hits.length > 0) return { score: 0, received: `${hits.length} × ${type}` };
+      const gap = coverageGap(ctx, "events");
+      if (gap) return gap;
+      return 1;
+    },
   };
 }
 
@@ -261,39 +381,61 @@ export function eventOrder(types: StreamEvent["type"][]): Spec {
       for (const ev of ctx.events) {
         if (i < types.length && ev.type === types[i]) i++;
       }
-      return i === types.length ? 1 : 0;
+      if (i === types.length) return 1;
+      const gap = coverageGap(ctx, "events");
+      if (gap) return gap;
+      return {
+        score: 0,
+        expected: types.join(" → "),
+        received: `missing ${types[i]} (matched ${i}/${types.length})`,
+      };
     },
   };
 }
 
+/** label 是失败时的全部解释(谓词不透明),必填、进断言标题。 */
 export function eventsSatisfy(
+  label: string,
   predicate: (events: readonly StreamEvent[]) => boolean,
-  label = "predicate",
 ): Spec {
+  if (typeof label !== "string" || label.trim().length === 0 || typeof predicate !== "function") {
+    throw new TypeError(
+      "eventsSatisfy(label, predicate) requires a non-empty string label followed by a predicate function; " +
+        `received (${typeof label}, ${typeof predicate}). The former (predicate, label) order is not supported.`,
+    );
+  }
   return {
-    name: `eventsSatisfy(${label})`,
+    name: label,
     severity: "gate",
-    evaluate: (ctx) => (predicate(ctx.events) ? 1 : 0),
+    evaluate: (ctx) => {
+      if (predicate(ctx.events)) return 1;
+      const gap = coverageGap(ctx, "events");
+      if (gap) return gap;
+      return { score: 0, received: `${ctx.events.length} events in scope` };
+    },
   };
 }
 
-// ── 工作区 / 沙箱 ──
-
-function diffMatchesRe(diff: DiffData, re: RegExp): boolean {
-  for (const [path, content] of Object.entries(diff.generatedFiles)) {
-    if (re.test(path) || re.test(content)) return true;
-  }
-  for (const path of diff.deletedFiles) {
-    if (re.test(path)) return true;
-  }
-  return false;
-}
+// ── 工作区 / 沙箱(断的是 agent 归因增量,见 docs/feature/sandbox/architecture.md)──
 
 export function fileChanged(path: string): Spec {
   return {
     name: `fileChanged(${path})`,
     severity: "gate",
-    evaluate: (ctx) => (ctx.diff.generatedFiles[path] !== undefined ? 1 : 0),
+    // 断「任一 send 窗口触及」(行为证据):净效果为 none(改完又改回)也算发生过。
+    evaluate: (ctx) => {
+      const summary = ctx.diff.files[path];
+      if (summary !== undefined && summary.net !== "deleted") return 1;
+      const windows = ctx.diff.windows.length;
+      return {
+        score: 0,
+        expected: "changed by agent in some send window",
+        received:
+          summary !== undefined
+            ? `net effect: ${summary.net} (touched in ${summary.windows.join(", ")})`
+            : `not changed in any of ${windows} send window${windows === 1 ? "" : "s"}`,
+      };
+    },
   };
 }
 
@@ -301,7 +443,10 @@ export function fileDeleted(path: string): Spec {
   return {
     name: `fileDeleted(${path})`,
     severity: "gate",
-    evaluate: (ctx) => (ctx.diff.deletedFiles.includes(path) ? 1 : 0),
+    evaluate: (ctx) =>
+      ctx.diff.files[path]?.net === "deleted"
+        ? 1
+        : { score: 0, expected: "deleted by agent", received: ctx.diff.files[path] ? `net effect: ${ctx.diff.files[path]!.net}` : "not touched by agent" },
   };
 }
 
@@ -309,7 +454,19 @@ export function notInDiff(re: RegExp): Spec {
   return {
     name: `notInDiff(${re})`,
     severity: "gate",
-    evaluate: (ctx) => (diffMatchesRe(ctx.diff, re) ? 0 : 1),
+    evaluate: (ctx) => {
+      for (const path of Object.keys(ctx.diff.files)) {
+        if (re.test(path)) return { score: 0, received: `matched path ${path}` };
+      }
+      for (const window of ctx.diff.windows) {
+        for (const [path, change] of Object.entries(window.changes)) {
+          if (change.after !== undefined && re.test(change.after)) {
+            return { score: 0, received: `matched in ${path} (window ${window.window})` };
+          }
+        }
+      }
+      return 1;
+    },
   };
 }
 
@@ -319,7 +476,10 @@ export function noFailedShellCommands(): Spec {
     severity: "gate",
     evaluate: (ctx) => {
       const failed = ctx.facts.toolCalls.filter((tc) => tc.name === "shell" && tc.status === "failed");
-      return { score: failed.length ? 0 : 1, evidence: describeCalls(failed) };
+      if (failed.length) return { score: 0, received: describeCalls(failed) };
+      const gap = coverageGap(ctx, "actions");
+      if (gap) return gap;
+      return 1;
     },
   };
 }
@@ -332,7 +492,11 @@ export function maxTokens(max: number): Spec {
     severity: "gate",
     evaluate: (ctx) => {
       const total = ctx.usage.inputTokens + ctx.usage.outputTokens;
-      return total <= max ? 1 : 0;
+      // 上限断言:实测已超限是确凿失败;未超限时缺 usage 不能按零聚合。
+      if (total > max) return { score: 0, expected: `≤ ${max} tokens`, received: `${total} tokens` };
+      const gap = coverageGap(ctx, "usage");
+      if (gap) return gap;
+      return 1;
     },
   };
 }
@@ -343,7 +507,10 @@ export function maxCost(usd: number): Spec {
     severity: "gate",
     evaluate: (ctx) => {
       const cost = ctx.usage.costUSD ?? 0;
-      return cost <= usd ? 1 : 0;
+      if (cost > usd) return { score: 0, expected: `≤ $${usd}`, received: `$${cost.toFixed(4)}` };
+      const gap = coverageGap(ctx, "usage");
+      if (gap) return gap;
+      return 1;
     },
   };
 }