niceeval 0.1.1 → 0.1.2

package/src/sandbox/vercel.ts

@@ -1,8 +1,6 @@
 // Vercel Sandbox 后端:用 @vercel/sandbox SDK 把 Vercel microVM 当隔离工作区跑 eval。
 // 契约对齐 ../types.ts 的 Sandbox 接口,与 DockerSandbox 可互换。
 
-import { readdir, readFile, stat } from "node:fs/promises";
-import { join, relative, sep } from "node:path";
 import { Sandbox as VSandbox } from "@vercel/sandbox";
 import type {
   Sandbox,
@@ -12,13 +10,11 @@ import type {
   SourceFiles,
   ReadSourceFilesOptions,
 } from "../types.ts";
-import { makeSourceFiles } from "./source-files.ts";
+import { readSourceFilesByList } from "./source-files.ts";
+import { collectLocalFiles } from "./local-files.ts";
+import { resolveSandboxPath } from "./paths.ts";
 import { t } from "../i18n/index.ts";
 
-const DEFAULT_SOURCE_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
-const DEFAULT_IGNORE_DIRS = [".git", ".next", "node_modules", "dist", "build", "coverage"];
-const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
-
 // Vercel Sandbox 的默认工作区路径(SDK writeFiles 默认落这里)。
 const VERCEL_WORKDIR = "/vercel/sandbox";
 
@@ -27,8 +23,26 @@ const DEFAULT_COMMAND_TIMEOUT_MS = 600_000;
 // Rotate session when it has been alive >270s to stay under the plan cap (~360-390s).
 const ROTATE_THRESHOLD_MS = 270_000;
 const SESSION_TIMEOUT_MS = 1_200_000;
+// rotate 时停掉旧 session 的等待上限:stop 挂起时不无限拖住当前命令。
+const STOP_OLD_SESSION_TIMEOUT_MS = 15_000;
+
+/** 给 promise 套超时;到点 reject,并清掉计时器避免拖住事件循环。 */
+async function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      promise,
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
 
 export class VercelSandbox implements Sandbox {
+  readonly workdir = VERCEL_WORKDIR;
   readonly otlpHost = null;
   private vsb: InstanceType<typeof VSandbox>;
   private commandTimeoutMs: number;
@@ -89,8 +103,18 @@ export class VercelSandbox implements Sandbox {
         source: { type: "snapshot", snapshotId },
         ...credParams,
       } as Parameters<typeof VSandbox.create>[0]);
+      const oldVsb = this.vsb;
       this.vsb = newVsb;
       this.sessionCreatedAt = Date.now();
+      // 旧 session 的 microVM 不随快照 / 新 session 创建自动回收,必须显式 stop,否则每次
+      // rotate 都泄漏一台在计费的 microVM。stop 与新 session 无数据依赖,不 await ——
+      // 挂起的 stop(最长 15s)不该拖住触发 rotate 的那条命令,还烧新 session 的时长。
+      // 失败只警告不静默(旧的到 session timeout 也会被平台回收)。
+      void withTimeout(oldVsb.stop(), STOP_OLD_SESSION_TIMEOUT_MS).catch((stopErr) => {
+        console.error(
+          `[VercelSandbox] warning: failed to stop rotated-out session, microVM may leak until session timeout: ${String(stopErr)}`,
+        );
+      });
       console.error(t("vercel.rotated", {
         seconds: Math.round(elapsed / 1000),
         sessionId: newVsb.currentSession().sessionId,
@@ -108,7 +132,7 @@ export class VercelSandbox implements Sandbox {
     const finished = await this.vsb.runCommand({
       cmd,
       args,
-      cwd: opts.cwd ?? VERCEL_WORKDIR,
+      cwd: resolveSandboxPath(this.workdir, opts.cwd),
       env: opts.env,
       sudo: opts.root ?? false,
       // 显式设 per-command timeout 防止长跑命令(npm build/install)被流截断。
@@ -126,73 +150,55 @@ export class VercelSandbox implements Sandbox {
   }
 
   async readFile(path: string): Promise<string> {
-    const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
+    const absPath = resolveSandboxPath(this.workdir, path);
     const buf = await this.vsb.readFileToBuffer({ path: absPath });
     if (!buf) throw new Error(t("vercel.fileNotFound", { path: absPath }));
     return buf.toString("utf8");
   }
 
   async fileExists(path: string): Promise<boolean> {
-    const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
+    const absPath = resolveSandboxPath(this.workdir, path);
     const buf = await this.vsb.readFileToBuffer({ path: absPath });
     return buf !== null;
   }
 
   async readSourceFiles(opts: ReadSourceFilesOptions = {}): Promise<SourceFiles> {
-    const extensions = opts.extensions ?? DEFAULT_SOURCE_EXTENSIONS;
-    const ignoreDirs = opts.ignoreDirs ?? DEFAULT_IGNORE_DIRS;
-    const ignoreFiles = new Set(opts.ignoreFiles ?? DEFAULT_IGNORE_FILES);
-
-    // 两阶段读取:Phase 1 只做 find(列路径,NDJSON 流短命令快速结束);
-    // Phase 2 逐文件用 readFileToBuffer(独立 HTTP GET,不依赖 NDJSON 流)。
-    // 这样即使 session 快到 plan 上限,后半段读取也不会被截断。
-    const dirPrune = ignoreDirs.map((d) => `-name '${d}'`).join(" -o ");
-    const nameTests = extensions.map((e) => `-name '*.${e}'`).join(" -o ");
-    const listScript = `find . \\( -type d \\( ${dirPrune} \\) \\) -prune -o -type f \\( ${nameTests} \\) -print`;
-    const result = await this.runShell(listScript);
-
-    const paths = result.stdout
-      .trim()
-      .split("\n")
-      .map((p) => p.trim().replace(/^\.\//, ""))
-      .filter((p) => p && !ignoreFiles.has(p.split("/").at(-1) ?? ""));
-
-    const files: { path: string; content: string }[] = [];
-    await Promise.all(
-      paths.map(async (path) => {
-        const absPath = `${VERCEL_WORKDIR}/${path}`;
-        try {
-          const buf = await this.vsb.readFileToBuffer({ path: absPath });
-          if (buf) files.push({ path, content: buf.toString("utf8") });
-        } catch {
-          // skip unreadable files (binary, permissions, etc.)
-        }
-      }),
-    );
-    return makeSourceFiles(files);
+    // 两阶段读取(共享模板):Phase 2 逐文件用 readFileToBuffer(独立 HTTP GET,
+    // 不依赖 NDJSON 流),即使 session 快到 plan 上限,后半段读取也不会被截断。
+    return readSourceFilesByList({
+      options: opts,
+      runShell: (script) => this.runShell(script),
+      readOne: async (path) => {
+        const buf = await this.vsb.readFileToBuffer({ path: `${VERCEL_WORKDIR}/${path}` });
+        return buf ? buf.toString("utf8") : null;
+      },
+    });
   }
 
-  async writeFiles(files: Record<string, string>, targetDir = VERCEL_WORKDIR): Promise<void> {
+  // targetDir 已由 paths.ts 的 normalizeSandboxPaths 解析成绝对路径;这里再解析一次
+  // 只是对直接使用后端实例(未包 normalize)的幂等防御,提到 map 外只算一次。
+  async writeFiles(files: Record<string, string>, targetDir?: string): Promise<void> {
+    const base = resolveSandboxPath(this.workdir, targetDir);
     const entries = Object.entries(files).map(([p, content]) => ({
-      // 相对路径 → Vercel SDK 自动 prepend /vercel/sandbox;绝对路径原样传。
-      path: p.startsWith("/") ? p : `${targetDir.replace(/\/$/, "")}/${p}`,
+      path: resolveSandboxPath(base, p),
       content,
     }));
     if (entries.length === 0) return;
     await this.vsb.writeFiles(entries);
   }
 
-  async uploadFiles(files: SandboxFile[], targetDir = VERCEL_WORKDIR): Promise<void> {
+  async uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void> {
     if (files.length === 0) return;
+    const base = resolveSandboxPath(this.workdir, targetDir);
     await this.vsb.writeFiles(
       files.map((f) => ({
-        path: f.path.startsWith("/") ? f.path : `${targetDir.replace(/\/$/, "")}/${f.path}`,
+        path: resolveSandboxPath(base, f.path),
         content: f.content,
       })),
     );
   }
 
-  async uploadDirectory(localDir: string, targetDir: string, opts: { ignore?: string[] } = {}): Promise<void> {
+  async uploadDirectory(localDir: string, targetDir?: string, opts: { ignore?: string[] } = {}): Promise<void> {
     await this.uploadFiles(await collectLocalFiles(localDir, opts.ignore), targetDir);
   }
 
@@ -201,36 +207,14 @@ export class VercelSandbox implements Sandbox {
   }
 
   async downloadFile(path: string): Promise<Buffer> {
-    const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
+    const absPath = resolveSandboxPath(this.workdir, path);
     const buf = await this.vsb.readFileToBuffer({ path: absPath });
     if (!buf) throw new Error(t("vercel.fileNotFound", { path: absPath }));
     return buf;
   }
 
   async uploadFile(path: string, content: Buffer): Promise<void> {
-    const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
+    const absPath = resolveSandboxPath(this.workdir, path);
     await this.vsb.writeFiles([{ path: absPath, content }]);
   }
 }
-
-async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
-  const ignored = new Set(ignore);
-  const out: SandboxFile[] = [];
-  async function walk(dir: string): Promise<void> {
-    for (const entry of await readdir(dir)) {
-      if (ignored.has(entry)) continue;
-      const abs = join(dir, entry);
-      const st = await stat(abs);
-      if (st.isDirectory()) {
-        await walk(abs);
-      } else if (st.isFile()) {
-        out.push({
-          path: relative(localDir, abs).split(sep).join("/"),
-          content: await readFile(abs),
-        });
-      }
-    }
-  }
-  await walk(localDir);
-  return out;
-}

package/src/scoring/collector.ts

@@ -4,11 +4,12 @@
 import type { AssertionResult, ScoringContext, Severity, SourceLoc } from "../types.ts";
 import { captureLoc } from "../source-loc.ts";
 import { t } from "../i18n/index.ts";
+import { formatThrown } from "../util.ts";
 
 export interface EvalScore {
   score: number;
   detail?: string;
-  /** 这条分数是看着什么材料算出来的(judge 收到的输入);供 view 展开排查「为什么是这个分」。 */
+  /** 这条分数是看着什么材料算出来的(judge 收到的输入,或 t.check 失败时实际被检查的值);供 view 展开排查「为什么是这个分」。 */
   evidence?: string;
 }
 
@@ -88,7 +89,7 @@ export class AssertionCollector {
       } catch (e) {
         score = 0;
         detail = `${detail ? detail + "; " : ""}${t("scoring.evalError", {
-          error: e instanceof Error ? e.message : String(e),
+          error: formatThrown(e),
         })}`;
       }
       out.push({

package/src/scoring/judge.ts

@@ -12,13 +12,14 @@ import { getEnv } from "../util.ts";
 import { t } from "../i18n/index.ts";
 
 interface ResolvedJudge {
-  model: string;
+  /** 未配置时为 undefined —— judge 没有内置默认模型,必须显式指定(config / eval / NICEEVAL_JUDGE_MODEL)。 */
+  model: string | undefined;
   baseUrl: string;
   apiKey: string | undefined;
 }
 
 function resolveJudge(judge: JudgeConfig | undefined): ResolvedJudge {
-  const model = judge?.model ?? "gpt-5.4-mini";
+  const model = judge?.model ?? getEnv("NICEEVAL_JUDGE_MODEL");
   const baseUrl =
     judge?.baseUrl ??
     getEnv("NICEEVAL_JUDGE_BASE") ??
@@ -33,84 +34,6 @@ function resolveJudge(judge: JudgeConfig | undefined): ResolvedJudge {
   return { model, baseUrl, apiKey };
 }
 
-/** 调评判模型,返回分数和原始推理文本。失败抛(collector 会兜成 0 分)。 */
-async function callJudge(
-  judge: ResolvedJudge,
-  system: string,
-  user: string,
-  signal?: AbortSignal,
-): Promise<EvalScore> {
-  if (!judge.apiKey) throw new Error(t("judge.apiKeyMissing"));
-  const url = `${judge.baseUrl.replace(/\/$/, "")}/chat/completions`;
-  const res = await fetch(url, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${judge.apiKey}`,
-    },
-    body: JSON.stringify({
-      model: judge.model,
-      messages: [
-        { role: "system", content: system },
-        { role: "user", content: user },
-      ],
-      temperature: 0,
-    }),
-    signal,
-  });
-  if (!res.ok) {
-    const body = await res.text().catch(() => "");
-    throw new Error(t("judge.httpError", { status: res.status, body: body.slice(0, 300) }));
-  }
-  const data = (await res.json()) as {
-    choices?: { message?: { content?: string } }[];
-  };
-  const content = data.choices?.[0]?.message?.content ?? "";
-  // evidence = 实际发给裁判的用户内容(材料 + 问题/标准)。view 展开就能看到「裁判到底读了什么」,
-  // 一眼分辨 0 分是回答真不行,还是喂错了材料(例如对话 eval 误喂 diff)。
-  return { ...parseJudgeReply(content), evidence: user };
-}
-
-/**
- * 解析评判回复:优先 JSON {reasoning, score}(detail = 理由,view 展开看 CoT);
- * 退化到纯数字 / 自由文本时 detail 落原文。
- */
-function parseJudgeReply(text: string): EvalScore {
-  const json = text.match(/\{[\s\S]*\}/);
-  if (json) {
-    try {
-      const obj = JSON.parse(json[0]) as { score?: unknown; reasoning?: unknown };
-      const reasoning = typeof obj.reasoning === "string" ? obj.reasoning.trim() : undefined;
-      const score = typeof obj.score === "number" ? clamp01(obj.score) : parseScore(text);
-      return { score, detail: reasoning || text || undefined };
-    } catch {
-      // 落到下面的纯文本解析
-    }
-  }
-  return { score: parseScore(text), detail: text || undefined };
-}
-
-/** 从模型回复里抠出 [0,1] 分。优先 JSON {score},否则取第一个数字。 */
-function parseScore(text: string): number {
-  const jsonMatch = text.match(/\{[^}]*"score"[^}]*\}/);
-  if (jsonMatch) {
-    try {
-      const obj = JSON.parse(jsonMatch[0]) as { score?: unknown };
-      if (typeof obj.score === "number") return clamp01(obj.score);
-    } catch {
-      // 落到下面的数字提取
-    }
-  }
-  const num = text.match(/(\d+(?:\.\d+)?)/);
-  if (num) {
-    let n = Number(num[1]);
-    if (n > 1 && n <= 100) n = n / 100; // 容忍 0–100 / 0–10
-    else if (n > 1 && n <= 10) n = n / 10;
-    return clamp01(n);
-  }
-  return 0;
-}
-
 function clamp01(n: number): number {
   if (!Number.isFinite(n)) return 0;
   return Math.max(0, Math.min(1, n));
@@ -140,16 +63,36 @@ function noOpJudge(): JudgeNamespace {
   return { autoevals: noOpAutoevals };
 }
 
-/** 预检显式配置的 judge:验证 API key 存在并发最小请求确认端点可达。
+/** 预检显式配置的 judge:验证 model + API key 存在,并发最小请求确认端点可达。
  *  返回错误描述字符串,可达则返回 undefined。*/
 export async function probeJudge(judge: JudgeConfig, signal?: AbortSignal): Promise<string | undefined> {
   const resolved = resolveJudge(judge);
+  if (!resolved.model) return t("judge.modelMissing");
   if (!resolved.apiKey) {
     const envHint = judge.apiKeyEnv ?? "NICEEVAL_JUDGE_KEY / OPENAI_API_KEY";
     return t("judge.probeMissingKey", { model: resolved.model, envHint });
   }
   try {
-    await callJudge(resolved, "Reply with the number 1 only.", "1", signal);
+    // 只确认可达 + 鉴权通过,不关心回复内容(真实评分走 autoevals)。
+    // 不带 max_tokens 等采样参数:新款模型(o 系 / gpt-5.x)会 400 拒掉 max_tokens,
+    // probe 的职责只是「端点通、key 对、model 认识」,参数越少越不误伤。
+    const url = `${resolved.baseUrl.replace(/\/$/, "")}/chat/completions`;
+    const res = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${resolved.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: resolved.model,
+        messages: [{ role: "user", content: "Reply with the single word: ok" }],
+      }),
+      signal,
+    });
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      throw new Error(t("judge.httpError", { status: res.status, body: body.slice(0, 300) }));
+    }
   } catch (e) {
     return t("judge.probeFailed", { model: resolved.model, error: e instanceof Error ? e.message : String(e) });
   }
@@ -164,73 +107,56 @@ export function buildJudge(deps: JudgeDeps): JudgeNamespace {
 
   const materialFor = async (ctx: ScoringContext, on?: string): Promise<string> => {
     if (on) {
-      // on 既可能是沙箱里的文件路径,也可能是一段字面文本
-      const fromFile = await ctx.readFile(on).catch(() => undefined);
-      if (fromFile !== undefined) return `----- ${on} -----\n${fromFile}`;
+      // on 既可能是沙箱里的文件路径,也可能是一段字面文本(如 t.sandbox.diff.get(...) 的内容)。
+      // 只有「长得像路径」(单行且不长)才尝试按文件读,避免对几 KB 的 diff 文本做无谓 IO,
+      // 也避免字面文本恰好命中某个存在的文件时被错读。
+      const looksLikePath = !on.includes("\n") && on.length <= 512;
+      if (looksLikePath) {
+        const fromFile = await ctx.readFile(on).catch(() => undefined);
+        if (fromFile !== undefined) return `----- ${on} -----\n${fromFile}`;
+      }
       return on;
     }
     return deps.getOutput();
   };
 
-  /** autoevals 公共参数:模型走 judge config,baseUrl + apiKey 透给 autoevals 内部建的 OpenAI client。 */
-  const autoevalsBase = {
-    model: resolved.model,
-    openAiBaseUrl: resolved.baseUrl,
-    openAiApiKey: resolved.apiKey,
-  } as const;
-
-  const closedQA = (criteria: string, opts?: { on?: string; model?: string }) =>
-    deps.record({
-      name: "judge:autoevals:closedQA",
-      severity: "soft",
-      evaluate: async (ctx) => {
-        const output = await materialFor(ctx, opts?.on);
-        const result = await ClosedQA({
-          input: deps.getInput(),
-          output,
-          criteria,
-          ...autoevalsBase,
-          ...(opts?.model ? { model: opts.model } : {}),
-        });
-        return { score: clamp01(result.score ?? 0), detail: (result as { rationale?: string }).rationale || undefined, evidence: output };
-      },
-    });
-
-  const factuality = (expected: string, opts?: { on?: string; model?: string }) =>
-    deps.record({
-      name: "judge:autoevals:factuality",
-      severity: "soft",
-      evaluate: async (ctx) => {
-        const output = await materialFor(ctx, opts?.on);
-        const result = await Factuality({
-          input: deps.getInput(),
-          output,
-          expected,
-          ...autoevalsBase,
-          ...(opts?.model ? { model: opts.model } : {}),
-        });
-        return { score: clamp01(result.score ?? 0), detail: (result as { rationale?: string }).rationale || undefined, evidence: output };
-      },
-    });
-
-  const summarizes = (expected: string, opts?: { on?: string; model?: string }) =>
-    deps.record({
-      name: "judge:autoevals:summarizes",
-      severity: "soft",
-      evaluate: async (ctx) => {
-        const output = await materialFor(ctx, opts?.on);
-        const result = await Summary({
-          input: deps.getInput(),
-          output,
-          expected,
-          ...autoevalsBase,
-          ...(opts?.model ? { model: opts.model } : {}),
-        });
-        return { score: clamp01(result.score ?? 0), detail: (result as { rationale?: string }).rationale || undefined, evidence: output };
-      },
-    });
-
-  const autoevalsNs: AutoevalsNamespace = { closedQA, factuality, summarizes };
-
-  return { autoevals: autoevalsNs };
+  type Scorer = (args: Record<string, unknown>) => Promise<{ score?: number | null }>;
+
+  // 三个 autoevals 方法只差评分器和材料字段名,共享行为(record spec / 材料构造 /
+  // 分数归一 / evidence)单一出处。model 解析:单次 { model } → judge config →
+  // NICEEVAL_JUDGE_MODEL;都没有是配置错误,调用点即报(不静默跳过,会藏住误配)。
+  const makeAutoeval =
+    (kind: "closedQA" | "factuality" | "summarizes", scorer: Scorer, payloadKey: "criteria" | "expected") =>
+    (reference: string, opts?: { on?: string; model?: string }) => {
+      const model = opts?.model ?? resolved.model;
+      if (!model) throw new Error(t("judge.modelMissing"));
+      return deps.record({
+        name: `judge:autoevals:${kind}`,
+        severity: "soft",
+        evaluate: async (ctx) => {
+          const output = await materialFor(ctx, opts?.on);
+          const result = await scorer({
+            input: deps.getInput(),
+            output,
+            [payloadKey]: reference,
+            model,
+            openAiBaseUrl: resolved.baseUrl,
+            openAiApiKey: resolved.apiKey,
+          });
+          return {
+            score: clamp01(result.score ?? 0),
+            detail: (result as { rationale?: string }).rationale || undefined,
+            evidence: output,
+          };
+        },
+      });
+    };
+
+  return {
+    autoevals: {
+      closedQA: makeAutoeval("closedQA", ClosedQA as unknown as Scorer, "criteria"),
+      factuality: makeAutoeval("factuality", Factuality as unknown as Scorer, "expected"),
+      summarizes: makeAutoeval("summarizes", Summary as unknown as Scorer, "expected"),
+    },
+  };
 }

package/src/scoring/match.ts

@@ -0,0 +1,79 @@
+// 值匹配共享工具:deepEqual 与 schema 校验的唯一实现。
+// expect 匹配器(equals / matches)和 turn.outputEquals / outputMatches 共用同一套,
+// 保证「同样两个值在不同断言入口下判定一致」——此前两处各写一份,NaN/Date 行为不同。
+
+/** 小而全的深比较:处理基本值、NaN、数组、Date、纯对象。 */
+export function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+
+  // NaN === NaN
+  if (typeof a === "number" && typeof b === "number") {
+    return Number.isNaN(a) && Number.isNaN(b);
+  }
+
+  if (a === null || b === null || typeof a !== "object" || typeof b !== "object") {
+    return false;
+  }
+
+  // Date 按时间戳比
+  if (a instanceof Date || b instanceof Date) {
+    return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
+  }
+
+  const aIsArr = Array.isArray(a);
+  const bIsArr = Array.isArray(b);
+  if (aIsArr !== bIsArr) return false;
+
+  if (aIsArr && bIsArr) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) return false;
+    }
+    return true;
+  }
+
+  const ao = a as Record<string, unknown>;
+  const bo = b as Record<string, unknown>;
+  const ak = Object.keys(ao);
+  const bk = Object.keys(bo);
+  if (ak.length !== bk.length) return false;
+  for (const k of ak) {
+    if (!Object.prototype.hasOwnProperty.call(bo, k)) return false;
+    if (!deepEqual(ao[k], bo[k])) return false;
+  }
+  return true;
+}
+
+/**
+ * 用 schema 校验 value。优先 Standard Schema(schema['~standard'].validate),
+ * 否则退化到 zod 风格的 .safeParse / .parse。校验通过 true,否则 false;任何异常 → false。
+ */
+export async function validateSchema(value: unknown, schema: unknown): Promise<boolean> {
+  try {
+    const std = (schema as { ["~standard"]?: { validate?: (v: unknown) => unknown } } | null)?.[
+      "~standard"
+    ];
+    if (std && typeof std.validate === "function") {
+      // validate 可能同步也可能返回 Promise;成功结果不带 issues。
+      const result = (await std.validate(value)) as { issues?: unknown } | null | undefined;
+      return result != null && result.issues == null;
+    }
+
+    const zodish = schema as {
+      safeParse?: (v: unknown) => { success?: boolean };
+      parse?: (v: unknown) => unknown;
+    } | null;
+
+    if (zodish && typeof zodish.safeParse === "function") {
+      const result = zodish.safeParse(value);
+      return Boolean(result && result.success);
+    }
+    if (zodish && typeof zodish.parse === "function") {
+      zodish.parse(value);
+      return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}

package/src/scoring/types.ts

@@ -0,0 +1,76 @@
+// scoring 域类型:值级断言(expect 匹配器)、断言记录与结果、评分上下文、judge 配置。
+
+import type { Severity, SourceLoc } from "../shared/types.ts";
+import type { DerivedFacts, StreamEvent, Usage } from "../o11y/types.ts";
+
+/** 值级断言(expect 匹配器)。纯函数 score + 可链式改严重级 / 阈值。 */
+export interface ValueAssertion {
+  readonly name: string;
+  readonly severity: Severity;
+  readonly threshold?: number;
+  score(value: unknown): number | Promise<number>;
+  gate(threshold?: number): ValueAssertion;
+  atLeast(threshold: number): ValueAssertion;
+}
+
+/** 收集到 collector 里的一条断言记录(评估前)。 */
+export interface AssertionSpec {
+  name: string;
+  severity: Severity;
+  threshold?: number;
+  /** 延迟评估:final 时拿到完整运行结果再算分。 */
+  evaluate(ctx: ScoringContext): Promise<number> | number;
+}
+
+/** 断言评估完的结果(进判决 / 报告)。 */
+export interface AssertionResult {
+  name: string;
+  severity: Severity;
+  threshold?: number;
+  score: number;
+  passed: boolean;
+  detail?: string;
+  /** 这条分数是看着什么材料算出来的(judge 收到的输入,或 t.check 失败时实际被检查的值)。view 展开排查「为什么是这个分」,默认不展示。 */
+  evidence?: string;
+  /** 所属分组(t.group 标题)。纯报告用,不影响 passed/score。 */
+  group?: string;
+  /** 断言在 eval 源码里的调用点(栈回溯抠出);view 把判决叠回这一行。 */
+  loc?: SourceLoc;
+}
+
+/** eval 作者拿到的可链式句柄(t.judge.autoevals.closedQA(...).atLeast(0.7))。 */
+export interface AssertionHandle {
+  atLeast(threshold: number): AssertionHandle;
+  gate(threshold?: number): AssertionHandle;
+}
+
+/** scoped / judge 断言在 final 评估时拿到的运行结果。 */
+export interface ScoringContext {
+  readonly events: readonly StreamEvent[];
+  readonly facts: DerivedFacts;
+  readonly diff: DiffData;
+  readonly scripts: Record<string, ScriptResult>;
+  readonly usage: Usage;
+  readonly status: "completed" | "failed" | "waiting";
+  /** 读沙箱里某文件的最终内容(judge / file 断言用)。 */
+  readFile(path: string): Promise<string | undefined>;
+}
+
+export interface ScriptResult {
+  success: boolean;
+  output: string;
+}
+
+export interface DiffData {
+  generatedFiles: Record<string, string>;
+  deletedFiles: string[];
+}
+
+export type ResultOutcome = "passed" | "failed" | "errored" | "skipped";
+
+export interface JudgeConfig {
+  model: string;
+  /** OpenAI 兼容 base url + key 来源;省略则从 env 探测(见 scoring/judge.ts)。 */
+  baseUrl?: string;
+  apiKeyEnv?: string;
+}