niceeval 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +8 -8
  2. package/README.zh.md +13 -13
  3. package/docs/README.md +119 -0
  4. package/docs/adapters/README.md +60 -0
  5. package/docs/adapters/authoring.md +179 -0
  6. package/docs/adapters/coding-agent-skills-plugins.md +324 -0
  7. package/docs/adapters/collection.md +128 -0
  8. package/docs/adapters/contract.md +263 -0
  9. package/docs/adapters/reference/agent-eval.md +215 -0
  10. package/docs/adapters/reference/agent-loop-apis.md +69 -0
  11. package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
  12. package/docs/adapters/reference/eve-protocol.md +127 -0
  13. package/docs/adapters/reference/otel-genai.md +107 -0
  14. package/docs/adapters/reference/otel-instrumentation.md +65 -0
  15. package/docs/adapters/targets.md +99 -0
  16. package/docs/architecture.md +140 -0
  17. package/docs/assertions.md +388 -0
  18. package/docs/capabilities-by-construction.md +48 -0
  19. package/docs/cli.md +171 -0
  20. package/docs/concepts.md +106 -0
  21. package/docs/e2e-ci.md +295 -0
  22. package/docs/eval-authoring.md +319 -0
  23. package/docs/experiments.md +154 -0
  24. package/docs/getting-started.md +224 -0
  25. package/docs/multi-agent.md +145 -0
  26. package/docs/observability.md +333 -0
  27. package/docs/origin-integration.md +195 -0
  28. package/docs/references.md +35 -0
  29. package/docs/results-format.md +228 -0
  30. package/docs/runner.md +104 -0
  31. package/docs/sandbox.md +276 -0
  32. package/docs/scoring.md +119 -0
  33. package/docs/source-map.md +106 -0
  34. package/docs/tier-sync.md +177 -0
  35. package/docs/view.md +157 -0
  36. package/docs/vision.md +88 -0
  37. package/package.json +39 -5
  38. package/src/agents/ai-sdk-otel.test.ts +22 -0
  39. package/src/agents/ai-sdk-otel.ts +62 -0
  40. package/src/agents/ai-sdk.test.ts +462 -0
  41. package/src/agents/ai-sdk.ts +575 -0
  42. package/src/agents/bub.ts +30 -37
  43. package/src/agents/claude-code.ts +7 -4
  44. package/src/agents/codex.ts +15 -10
  45. package/src/agents/index.ts +48 -1
  46. package/src/agents/sdk-streams.test.ts +145 -0
  47. package/src/agents/sdk-streams.ts +457 -0
  48. package/src/agents/shared.ts +77 -39
  49. package/src/agents/streaming.test.ts +152 -0
  50. package/src/agents/streaming.ts +195 -0
  51. package/src/agents/types.ts +218 -0
  52. package/src/agents/ui-message-stream.test.ts +241 -0
  53. package/src/agents/ui-message-stream.ts +368 -0
  54. package/src/cli.ts +124 -77
  55. package/src/context/context.test.ts +203 -7
  56. package/src/context/context.ts +158 -49
  57. package/src/context/session.test.ts +96 -0
  58. package/src/context/session.ts +119 -9
  59. package/src/context/types.ts +205 -0
  60. package/src/define.ts +4 -14
  61. package/src/expect/index.ts +8 -71
  62. package/src/i18n/core.ts +28 -0
  63. package/src/i18n/en.ts +47 -5
  64. package/src/i18n/index.ts +5 -17
  65. package/src/i18n/zh-CN.ts +47 -5
  66. package/src/index.ts +12 -0
  67. package/src/loaders/index.ts +8 -39
  68. package/src/o11y/otlp/canonical.ts +7 -6
  69. package/src/o11y/otlp/mappers/codex.ts +10 -8
  70. package/src/o11y/otlp/mappers/index.ts +9 -21
  71. package/src/o11y/otlp/receiver.ts +25 -7
  72. package/src/o11y/otlp/sandbox-receiver.ts +57 -21
  73. package/src/o11y/otlp/turn-otel.test.ts +110 -0
  74. package/src/o11y/otlp/turn-otel.ts +125 -0
  75. package/src/o11y/parsers/bub.ts +13 -11
  76. package/src/o11y/parsers/claude-code.ts +27 -51
  77. package/src/o11y/parsers/codex.ts +29 -46
  78. package/src/o11y/parsers/index.ts +5 -14
  79. package/src/o11y/tool-names.test.ts +61 -0
  80. package/src/o11y/tool-names.ts +74 -0
  81. package/src/o11y/types.ts +135 -0
  82. package/src/runner/attempt.ts +456 -0
  83. package/src/runner/fingerprint.ts +59 -0
  84. package/src/runner/remote-sandbox.ts +53 -0
  85. package/src/runner/report.ts +53 -0
  86. package/src/runner/reporters/artifacts.ts +25 -4
  87. package/src/runner/reporters/console.ts +2 -8
  88. package/src/runner/reporters/live.ts +2 -8
  89. package/src/runner/reporters/shared.ts +15 -0
  90. package/src/runner/reporters/table.ts +10 -62
  91. package/src/runner/run.ts +114 -619
  92. package/src/runner/types.ts +237 -0
  93. package/src/sandbox/checkpoint.ts +15 -13
  94. package/src/sandbox/docker-stream.ts +87 -0
  95. package/src/sandbox/docker.ts +42 -120
  96. package/src/sandbox/e2b.ts +24 -75
  97. package/src/sandbox/local-files.ts +33 -0
  98. package/src/sandbox/paths.test.ts +85 -0
  99. package/src/sandbox/paths.ts +45 -0
  100. package/src/sandbox/resolve.ts +10 -34
  101. package/src/sandbox/shell.ts +17 -0
  102. package/src/sandbox/source-files.ts +42 -2
  103. package/src/sandbox/types.ts +158 -0
  104. package/src/sandbox/vercel.ts +55 -71
  105. package/src/scoring/collector.ts +3 -2
  106. package/src/scoring/judge.ts +72 -146
  107. package/src/scoring/match.ts +79 -0
  108. package/src/scoring/scoped.ts +66 -18
  109. package/src/scoring/types.ts +76 -0
  110. package/src/shared/aggregate.ts +48 -0
  111. package/src/shared/format.ts +20 -0
  112. package/src/shared/outcome.ts +48 -0
  113. package/src/shared/types.ts +37 -0
  114. package/src/types.ts +20 -911
  115. package/src/view/aggregate.ts +103 -0
  116. package/src/view/app/App.tsx +26 -5
  117. package/src/view/app/components/AttemptModal.tsx +12 -3
  118. package/src/view/app/components/CodeView.tsx +16 -19
  119. package/src/view/app/components/CopyControls.tsx +4 -4
  120. package/src/view/app/components/ExperimentTable.tsx +5 -5
  121. package/src/view/app/i18n.ts +31 -7
  122. package/src/view/app/lib/format.ts +17 -20
  123. package/src/view/app/lib/outcome.ts +4 -16
  124. package/src/view/app/lib/transcript-data.tsx +20 -4
  125. package/src/view/app/main.tsx +3 -5
  126. package/src/view/app/pages/RunsPage.tsx +2 -2
  127. package/src/view/app/pages/TracesPage.tsx +2 -2
  128. package/src/view/app/shared.ts +2 -1
  129. package/src/view/app/types.ts +9 -44
  130. package/src/view/client-dist/app.css +1 -1
  131. package/src/view/client-dist/app.js +18 -18
  132. package/src/view/index.ts +24 -401
  133. package/src/view/loader.ts +234 -0
  134. package/src/view/server.ts +136 -0
  135. package/src/view/shared/types.ts +64 -0
  136. package/src/view/styles.css +60 -9
@@ -1,8 +1,6 @@
1
1
  // Vercel Sandbox 后端:用 @vercel/sandbox SDK 把 Vercel microVM 当隔离工作区跑 eval。
2
2
  // 契约对齐 ../types.ts 的 Sandbox 接口,与 DockerSandbox 可互换。
3
3
 
4
- import { readdir, readFile, stat } from "node:fs/promises";
5
- import { join, relative, sep } from "node:path";
6
4
  import { Sandbox as VSandbox } from "@vercel/sandbox";
7
5
  import type {
8
6
  Sandbox,
@@ -12,13 +10,11 @@ import type {
12
10
  SourceFiles,
13
11
  ReadSourceFilesOptions,
14
12
  } from "../types.ts";
15
- import { makeSourceFiles } from "./source-files.ts";
13
+ import { readSourceFilesByList } from "./source-files.ts";
14
+ import { collectLocalFiles } from "./local-files.ts";
15
+ import { resolveSandboxPath } from "./paths.ts";
16
16
  import { t } from "../i18n/index.ts";
17
17
 
18
- const DEFAULT_SOURCE_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
19
- const DEFAULT_IGNORE_DIRS = [".git", ".next", "node_modules", "dist", "build", "coverage"];
20
- const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
21
-
22
18
  // Vercel Sandbox 的默认工作区路径(SDK writeFiles 默认落这里)。
23
19
  const VERCEL_WORKDIR = "/vercel/sandbox";
24
20
 
@@ -27,8 +23,26 @@ const DEFAULT_COMMAND_TIMEOUT_MS = 600_000;
27
23
  // Rotate session when it has been alive >270s to stay under the plan cap (~360-390s).
28
24
  const ROTATE_THRESHOLD_MS = 270_000;
29
25
  const SESSION_TIMEOUT_MS = 1_200_000;
26
+ // rotate 时停掉旧 session 的等待上限:stop 挂起时不无限拖住当前命令。
27
+ const STOP_OLD_SESSION_TIMEOUT_MS = 15_000;
28
+
29
+ /** 给 promise 套超时;到点 reject,并清掉计时器避免拖住事件循环。 */
30
+ async function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
31
+ let timer: ReturnType<typeof setTimeout> | undefined;
32
+ try {
33
+ return await Promise.race([
34
+ promise,
35
+ new Promise<never>((_, reject) => {
36
+ timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
37
+ }),
38
+ ]);
39
+ } finally {
40
+ clearTimeout(timer);
41
+ }
42
+ }
30
43
 
31
44
  export class VercelSandbox implements Sandbox {
45
+ readonly workdir = VERCEL_WORKDIR;
32
46
  readonly otlpHost = null;
33
47
  private vsb: InstanceType<typeof VSandbox>;
34
48
  private commandTimeoutMs: number;
@@ -89,8 +103,18 @@ export class VercelSandbox implements Sandbox {
89
103
  source: { type: "snapshot", snapshotId },
90
104
  ...credParams,
91
105
  } as Parameters<typeof VSandbox.create>[0]);
106
+ const oldVsb = this.vsb;
92
107
  this.vsb = newVsb;
93
108
  this.sessionCreatedAt = Date.now();
109
+ // 旧 session 的 microVM 不随快照 / 新 session 创建自动回收,必须显式 stop,否则每次
110
+ // rotate 都泄漏一台在计费的 microVM。stop 与新 session 无数据依赖,不 await ——
111
+ // 挂起的 stop(最长 15s)不该拖住触发 rotate 的那条命令,还烧新 session 的时长。
112
+ // 失败只警告不静默(旧的到 session timeout 也会被平台回收)。
113
+ void withTimeout(oldVsb.stop(), STOP_OLD_SESSION_TIMEOUT_MS).catch((stopErr) => {
114
+ console.error(
115
+ `[VercelSandbox] warning: failed to stop rotated-out session, microVM may leak until session timeout: ${String(stopErr)}`,
116
+ );
117
+ });
94
118
  console.error(t("vercel.rotated", {
95
119
  seconds: Math.round(elapsed / 1000),
96
120
  sessionId: newVsb.currentSession().sessionId,
@@ -108,7 +132,7 @@ export class VercelSandbox implements Sandbox {
108
132
  const finished = await this.vsb.runCommand({
109
133
  cmd,
110
134
  args,
111
- cwd: opts.cwd ?? VERCEL_WORKDIR,
135
+ cwd: resolveSandboxPath(this.workdir, opts.cwd),
112
136
  env: opts.env,
113
137
  sudo: opts.root ?? false,
114
138
  // 显式设 per-command timeout 防止长跑命令(npm build/install)被流截断。
@@ -126,73 +150,55 @@ export class VercelSandbox implements Sandbox {
126
150
  }
127
151
 
128
152
  async readFile(path: string): Promise<string> {
129
- const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
153
+ const absPath = resolveSandboxPath(this.workdir, path);
130
154
  const buf = await this.vsb.readFileToBuffer({ path: absPath });
131
155
  if (!buf) throw new Error(t("vercel.fileNotFound", { path: absPath }));
132
156
  return buf.toString("utf8");
133
157
  }
134
158
 
135
159
  async fileExists(path: string): Promise<boolean> {
136
- const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
160
+ const absPath = resolveSandboxPath(this.workdir, path);
137
161
  const buf = await this.vsb.readFileToBuffer({ path: absPath });
138
162
  return buf !== null;
139
163
  }
140
164
 
141
165
  async readSourceFiles(opts: ReadSourceFilesOptions = {}): Promise<SourceFiles> {
142
- const extensions = opts.extensions ?? DEFAULT_SOURCE_EXTENSIONS;
143
- const ignoreDirs = opts.ignoreDirs ?? DEFAULT_IGNORE_DIRS;
144
- const ignoreFiles = new Set(opts.ignoreFiles ?? DEFAULT_IGNORE_FILES);
145
-
146
- // 两阶段读取:Phase 1 只做 find(列路径,NDJSON 流短命令快速结束);
147
- // Phase 2 逐文件用 readFileToBuffer(独立 HTTP GET,不依赖 NDJSON 流)。
148
- // 这样即使 session 快到 plan 上限,后半段读取也不会被截断。
149
- const dirPrune = ignoreDirs.map((d) => `-name '${d}'`).join(" -o ");
150
- const nameTests = extensions.map((e) => `-name '*.${e}'`).join(" -o ");
151
- const listScript = `find . \\( -type d \\( ${dirPrune} \\) \\) -prune -o -type f \\( ${nameTests} \\) -print`;
152
- const result = await this.runShell(listScript);
153
-
154
- const paths = result.stdout
155
- .trim()
156
- .split("\n")
157
- .map((p) => p.trim().replace(/^\.\//, ""))
158
- .filter((p) => p && !ignoreFiles.has(p.split("/").at(-1) ?? ""));
159
-
160
- const files: { path: string; content: string }[] = [];
161
- await Promise.all(
162
- paths.map(async (path) => {
163
- const absPath = `${VERCEL_WORKDIR}/${path}`;
164
- try {
165
- const buf = await this.vsb.readFileToBuffer({ path: absPath });
166
- if (buf) files.push({ path, content: buf.toString("utf8") });
167
- } catch {
168
- // skip unreadable files (binary, permissions, etc.)
169
- }
170
- }),
171
- );
172
- return makeSourceFiles(files);
166
+ // 两阶段读取(共享模板):Phase 2 逐文件用 readFileToBuffer(独立 HTTP GET,
167
+ // 不依赖 NDJSON 流),即使 session 快到 plan 上限,后半段读取也不会被截断。
168
+ return readSourceFilesByList({
169
+ options: opts,
170
+ runShell: (script) => this.runShell(script),
171
+ readOne: async (path) => {
172
+ const buf = await this.vsb.readFileToBuffer({ path: `${VERCEL_WORKDIR}/${path}` });
173
+ return buf ? buf.toString("utf8") : null;
174
+ },
175
+ });
173
176
  }
174
177
 
175
- async writeFiles(files: Record<string, string>, targetDir = VERCEL_WORKDIR): Promise<void> {
178
+ // targetDir 已由 paths.ts normalizeSandboxPaths 解析成绝对路径;这里再解析一次
179
+ // 只是对直接使用后端实例(未包 normalize)的幂等防御,提到 map 外只算一次。
180
+ async writeFiles(files: Record<string, string>, targetDir?: string): Promise<void> {
181
+ const base = resolveSandboxPath(this.workdir, targetDir);
176
182
  const entries = Object.entries(files).map(([p, content]) => ({
177
- // 相对路径 → Vercel SDK 自动 prepend /vercel/sandbox;绝对路径原样传。
178
- path: p.startsWith("/") ? p : `${targetDir.replace(/\/$/, "")}/${p}`,
183
+ path: resolveSandboxPath(base, p),
179
184
  content,
180
185
  }));
181
186
  if (entries.length === 0) return;
182
187
  await this.vsb.writeFiles(entries);
183
188
  }
184
189
 
185
- async uploadFiles(files: SandboxFile[], targetDir = VERCEL_WORKDIR): Promise<void> {
190
+ async uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void> {
186
191
  if (files.length === 0) return;
192
+ const base = resolveSandboxPath(this.workdir, targetDir);
187
193
  await this.vsb.writeFiles(
188
194
  files.map((f) => ({
189
- path: f.path.startsWith("/") ? f.path : `${targetDir.replace(/\/$/, "")}/${f.path}`,
195
+ path: resolveSandboxPath(base, f.path),
190
196
  content: f.content,
191
197
  })),
192
198
  );
193
199
  }
194
200
 
195
- async uploadDirectory(localDir: string, targetDir: string, opts: { ignore?: string[] } = {}): Promise<void> {
201
+ async uploadDirectory(localDir: string, targetDir?: string, opts: { ignore?: string[] } = {}): Promise<void> {
196
202
  await this.uploadFiles(await collectLocalFiles(localDir, opts.ignore), targetDir);
197
203
  }
198
204
 
@@ -201,36 +207,14 @@ export class VercelSandbox implements Sandbox {
201
207
  }
202
208
 
203
209
  async downloadFile(path: string): Promise<Buffer> {
204
- const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
210
+ const absPath = resolveSandboxPath(this.workdir, path);
205
211
  const buf = await this.vsb.readFileToBuffer({ path: absPath });
206
212
  if (!buf) throw new Error(t("vercel.fileNotFound", { path: absPath }));
207
213
  return buf;
208
214
  }
209
215
 
210
216
  async uploadFile(path: string, content: Buffer): Promise<void> {
211
- const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
217
+ const absPath = resolveSandboxPath(this.workdir, path);
212
218
  await this.vsb.writeFiles([{ path: absPath, content }]);
213
219
  }
214
220
  }
215
-
216
- async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
217
- const ignored = new Set(ignore);
218
- const out: SandboxFile[] = [];
219
- async function walk(dir: string): Promise<void> {
220
- for (const entry of await readdir(dir)) {
221
- if (ignored.has(entry)) continue;
222
- const abs = join(dir, entry);
223
- const st = await stat(abs);
224
- if (st.isDirectory()) {
225
- await walk(abs);
226
- } else if (st.isFile()) {
227
- out.push({
228
- path: relative(localDir, abs).split(sep).join("/"),
229
- content: await readFile(abs),
230
- });
231
- }
232
- }
233
- }
234
- await walk(localDir);
235
- return out;
236
- }
@@ -4,11 +4,12 @@
4
4
  import type { AssertionResult, ScoringContext, Severity, SourceLoc } from "../types.ts";
5
5
  import { captureLoc } from "../source-loc.ts";
6
6
  import { t } from "../i18n/index.ts";
7
+ import { formatThrown } from "../util.ts";
7
8
 
8
9
  export interface EvalScore {
9
10
  score: number;
10
11
  detail?: string;
11
- /** 这条分数是看着什么材料算出来的(judge 收到的输入);供 view 展开排查「为什么是这个分」。 */
12
+ /** 这条分数是看着什么材料算出来的(judge 收到的输入,或 t.check 失败时实际被检查的值);供 view 展开排查「为什么是这个分」。 */
12
13
  evidence?: string;
13
14
  }
14
15
 
@@ -88,7 +89,7 @@ export class AssertionCollector {
88
89
  } catch (e) {
89
90
  score = 0;
90
91
  detail = `${detail ? detail + "; " : ""}${t("scoring.evalError", {
91
- error: e instanceof Error ? e.message : String(e),
92
+ error: formatThrown(e),
92
93
  })}`;
93
94
  }
94
95
  out.push({
@@ -12,13 +12,14 @@ import { getEnv } from "../util.ts";
12
12
  import { t } from "../i18n/index.ts";
13
13
 
14
14
  interface ResolvedJudge {
15
- model: string;
15
+ /** 未配置时为 undefined —— judge 没有内置默认模型,必须显式指定(config / eval / NICEEVAL_JUDGE_MODEL)。 */
16
+ model: string | undefined;
16
17
  baseUrl: string;
17
18
  apiKey: string | undefined;
18
19
  }
19
20
 
20
21
  function resolveJudge(judge: JudgeConfig | undefined): ResolvedJudge {
21
- const model = judge?.model ?? "gpt-5.4-mini";
22
+ const model = judge?.model ?? getEnv("NICEEVAL_JUDGE_MODEL");
22
23
  const baseUrl =
23
24
  judge?.baseUrl ??
24
25
  getEnv("NICEEVAL_JUDGE_BASE") ??
@@ -33,84 +34,6 @@ function resolveJudge(judge: JudgeConfig | undefined): ResolvedJudge {
33
34
  return { model, baseUrl, apiKey };
34
35
  }
35
36
 
36
- /** 调评判模型,返回分数和原始推理文本。失败抛(collector 会兜成 0 分)。 */
37
- async function callJudge(
38
- judge: ResolvedJudge,
39
- system: string,
40
- user: string,
41
- signal?: AbortSignal,
42
- ): Promise<EvalScore> {
43
- if (!judge.apiKey) throw new Error(t("judge.apiKeyMissing"));
44
- const url = `${judge.baseUrl.replace(/\/$/, "")}/chat/completions`;
45
- const res = await fetch(url, {
46
- method: "POST",
47
- headers: {
48
- "Content-Type": "application/json",
49
- Authorization: `Bearer ${judge.apiKey}`,
50
- },
51
- body: JSON.stringify({
52
- model: judge.model,
53
- messages: [
54
- { role: "system", content: system },
55
- { role: "user", content: user },
56
- ],
57
- temperature: 0,
58
- }),
59
- signal,
60
- });
61
- if (!res.ok) {
62
- const body = await res.text().catch(() => "");
63
- throw new Error(t("judge.httpError", { status: res.status, body: body.slice(0, 300) }));
64
- }
65
- const data = (await res.json()) as {
66
- choices?: { message?: { content?: string } }[];
67
- };
68
- const content = data.choices?.[0]?.message?.content ?? "";
69
- // evidence = 实际发给裁判的用户内容(材料 + 问题/标准)。view 展开就能看到「裁判到底读了什么」,
70
- // 一眼分辨 0 分是回答真不行,还是喂错了材料(例如对话 eval 误喂 diff)。
71
- return { ...parseJudgeReply(content), evidence: user };
72
- }
73
-
74
- /**
75
- * 解析评判回复:优先 JSON {reasoning, score}(detail = 理由,view 展开看 CoT);
76
- * 退化到纯数字 / 自由文本时 detail 落原文。
77
- */
78
- function parseJudgeReply(text: string): EvalScore {
79
- const json = text.match(/\{[\s\S]*\}/);
80
- if (json) {
81
- try {
82
- const obj = JSON.parse(json[0]) as { score?: unknown; reasoning?: unknown };
83
- const reasoning = typeof obj.reasoning === "string" ? obj.reasoning.trim() : undefined;
84
- const score = typeof obj.score === "number" ? clamp01(obj.score) : parseScore(text);
85
- return { score, detail: reasoning || text || undefined };
86
- } catch {
87
- // 落到下面的纯文本解析
88
- }
89
- }
90
- return { score: parseScore(text), detail: text || undefined };
91
- }
92
-
93
- /** 从模型回复里抠出 [0,1] 分。优先 JSON {score},否则取第一个数字。 */
94
- function parseScore(text: string): number {
95
- const jsonMatch = text.match(/\{[^}]*"score"[^}]*\}/);
96
- if (jsonMatch) {
97
- try {
98
- const obj = JSON.parse(jsonMatch[0]) as { score?: unknown };
99
- if (typeof obj.score === "number") return clamp01(obj.score);
100
- } catch {
101
- // 落到下面的数字提取
102
- }
103
- }
104
- const num = text.match(/(\d+(?:\.\d+)?)/);
105
- if (num) {
106
- let n = Number(num[1]);
107
- if (n > 1 && n <= 100) n = n / 100; // 容忍 0–100 / 0–10
108
- else if (n > 1 && n <= 10) n = n / 10;
109
- return clamp01(n);
110
- }
111
- return 0;
112
- }
113
-
114
37
  function clamp01(n: number): number {
115
38
  if (!Number.isFinite(n)) return 0;
116
39
  return Math.max(0, Math.min(1, n));
@@ -140,16 +63,36 @@ function noOpJudge(): JudgeNamespace {
140
63
  return { autoevals: noOpAutoevals };
141
64
  }
142
65
 
143
- /** 预检显式配置的 judge:验证 API key 存在并发最小请求确认端点可达。
66
+ /** 预检显式配置的 judge:验证 model + API key 存在,并发最小请求确认端点可达。
144
67
  * 返回错误描述字符串,可达则返回 undefined。*/
145
68
  export async function probeJudge(judge: JudgeConfig, signal?: AbortSignal): Promise<string | undefined> {
146
69
  const resolved = resolveJudge(judge);
70
+ if (!resolved.model) return t("judge.modelMissing");
147
71
  if (!resolved.apiKey) {
148
72
  const envHint = judge.apiKeyEnv ?? "NICEEVAL_JUDGE_KEY / OPENAI_API_KEY";
149
73
  return t("judge.probeMissingKey", { model: resolved.model, envHint });
150
74
  }
151
75
  try {
152
- await callJudge(resolved, "Reply with the number 1 only.", "1", signal);
76
+ // 只确认可达 + 鉴权通过,不关心回复内容(真实评分走 autoevals)
77
+ // 不带 max_tokens 等采样参数:新款模型(o 系 / gpt-5.x)会 400 拒掉 max_tokens,
78
+ // probe 的职责只是「端点通、key 对、model 认识」,参数越少越不误伤。
79
+ const url = `${resolved.baseUrl.replace(/\/$/, "")}/chat/completions`;
80
+ const res = await fetch(url, {
81
+ method: "POST",
82
+ headers: {
83
+ "Content-Type": "application/json",
84
+ Authorization: `Bearer ${resolved.apiKey}`,
85
+ },
86
+ body: JSON.stringify({
87
+ model: resolved.model,
88
+ messages: [{ role: "user", content: "Reply with the single word: ok" }],
89
+ }),
90
+ signal,
91
+ });
92
+ if (!res.ok) {
93
+ const body = await res.text().catch(() => "");
94
+ throw new Error(t("judge.httpError", { status: res.status, body: body.slice(0, 300) }));
95
+ }
153
96
  } catch (e) {
154
97
  return t("judge.probeFailed", { model: resolved.model, error: e instanceof Error ? e.message : String(e) });
155
98
  }
@@ -164,73 +107,56 @@ export function buildJudge(deps: JudgeDeps): JudgeNamespace {
164
107
 
165
108
  const materialFor = async (ctx: ScoringContext, on?: string): Promise<string> => {
166
109
  if (on) {
167
- // on 既可能是沙箱里的文件路径,也可能是一段字面文本
168
- const fromFile = await ctx.readFile(on).catch(() => undefined);
169
- if (fromFile !== undefined) return `----- ${on} -----\n${fromFile}`;
110
+ // on 既可能是沙箱里的文件路径,也可能是一段字面文本(如 t.sandbox.diff.get(...) 的内容)。
111
+ // 只有「长得像路径」(单行且不长)才尝试按文件读,避免对几 KB diff 文本做无谓 IO,
112
+ // 也避免字面文本恰好命中某个存在的文件时被错读。
113
+ const looksLikePath = !on.includes("\n") && on.length <= 512;
114
+ if (looksLikePath) {
115
+ const fromFile = await ctx.readFile(on).catch(() => undefined);
116
+ if (fromFile !== undefined) return `----- ${on} -----\n${fromFile}`;
117
+ }
170
118
  return on;
171
119
  }
172
120
  return deps.getOutput();
173
121
  };
174
122
 
175
- /** autoevals 公共参数:模型走 judge config,baseUrl + apiKey 透给 autoevals 内部建的 OpenAI client。 */
176
- const autoevalsBase = {
177
- model: resolved.model,
178
- openAiBaseUrl: resolved.baseUrl,
179
- openAiApiKey: resolved.apiKey,
180
- } as const;
181
-
182
- const closedQA = (criteria: string, opts?: { on?: string; model?: string }) =>
183
- deps.record({
184
- name: "judge:autoevals:closedQA",
185
- severity: "soft",
186
- evaluate: async (ctx) => {
187
- const output = await materialFor(ctx, opts?.on);
188
- const result = await ClosedQA({
189
- input: deps.getInput(),
190
- output,
191
- criteria,
192
- ...autoevalsBase,
193
- ...(opts?.model ? { model: opts.model } : {}),
194
- });
195
- return { score: clamp01(result.score ?? 0), detail: (result as { rationale?: string }).rationale || undefined, evidence: output };
196
- },
197
- });
198
-
199
- const factuality = (expected: string, opts?: { on?: string; model?: string }) =>
200
- deps.record({
201
- name: "judge:autoevals:factuality",
202
- severity: "soft",
203
- evaluate: async (ctx) => {
204
- const output = await materialFor(ctx, opts?.on);
205
- const result = await Factuality({
206
- input: deps.getInput(),
207
- output,
208
- expected,
209
- ...autoevalsBase,
210
- ...(opts?.model ? { model: opts.model } : {}),
211
- });
212
- return { score: clamp01(result.score ?? 0), detail: (result as { rationale?: string }).rationale || undefined, evidence: output };
213
- },
214
- });
215
-
216
- const summarizes = (expected: string, opts?: { on?: string; model?: string }) =>
217
- deps.record({
218
- name: "judge:autoevals:summarizes",
219
- severity: "soft",
220
- evaluate: async (ctx) => {
221
- const output = await materialFor(ctx, opts?.on);
222
- const result = await Summary({
223
- input: deps.getInput(),
224
- output,
225
- expected,
226
- ...autoevalsBase,
227
- ...(opts?.model ? { model: opts.model } : {}),
228
- });
229
- return { score: clamp01(result.score ?? 0), detail: (result as { rationale?: string }).rationale || undefined, evidence: output };
230
- },
231
- });
232
-
233
- const autoevalsNs: AutoevalsNamespace = { closedQA, factuality, summarizes };
234
-
235
- return { autoevals: autoevalsNs };
123
+ type Scorer = (args: Record<string, unknown>) => Promise<{ score?: number | null }>;
124
+
125
+ // 三个 autoevals 方法只差评分器和材料字段名,共享行为(record spec / 材料构造 /
126
+ // 分数归一 / evidence)单一出处。model 解析:单次 { model } → judge config →
127
+ // NICEEVAL_JUDGE_MODEL;都没有是配置错误,调用点即报(不静默跳过,会藏住误配)。
128
+ const makeAutoeval =
129
+ (kind: "closedQA" | "factuality" | "summarizes", scorer: Scorer, payloadKey: "criteria" | "expected") =>
130
+ (reference: string, opts?: { on?: string; model?: string }) => {
131
+ const model = opts?.model ?? resolved.model;
132
+ if (!model) throw new Error(t("judge.modelMissing"));
133
+ return deps.record({
134
+ name: `judge:autoevals:${kind}`,
135
+ severity: "soft",
136
+ evaluate: async (ctx) => {
137
+ const output = await materialFor(ctx, opts?.on);
138
+ const result = await scorer({
139
+ input: deps.getInput(),
140
+ output,
141
+ [payloadKey]: reference,
142
+ model,
143
+ openAiBaseUrl: resolved.baseUrl,
144
+ openAiApiKey: resolved.apiKey,
145
+ });
146
+ return {
147
+ score: clamp01(result.score ?? 0),
148
+ detail: (result as { rationale?: string }).rationale || undefined,
149
+ evidence: output,
150
+ };
151
+ },
152
+ });
153
+ };
154
+
155
+ return {
156
+ autoevals: {
157
+ closedQA: makeAutoeval("closedQA", ClosedQA as unknown as Scorer, "criteria"),
158
+ factuality: makeAutoeval("factuality", Factuality as unknown as Scorer, "expected"),
159
+ summarizes: makeAutoeval("summarizes", Summary as unknown as Scorer, "expected"),
160
+ },
161
+ };
236
162
  }
@@ -0,0 +1,79 @@
1
+ // 值匹配共享工具:deepEqual 与 schema 校验的唯一实现。
2
+ // expect 匹配器(equals / matches)和 turn.outputEquals / outputMatches 共用同一套,
3
+ // 保证「同样两个值在不同断言入口下判定一致」——此前两处各写一份,NaN/Date 行为不同。
4
+
5
+ /** 小而全的深比较:处理基本值、NaN、数组、Date、纯对象。 */
6
+ export function deepEqual(a: unknown, b: unknown): boolean {
7
+ if (a === b) return true;
8
+
9
+ // NaN === NaN
10
+ if (typeof a === "number" && typeof b === "number") {
11
+ return Number.isNaN(a) && Number.isNaN(b);
12
+ }
13
+
14
+ if (a === null || b === null || typeof a !== "object" || typeof b !== "object") {
15
+ return false;
16
+ }
17
+
18
+ // Date 按时间戳比
19
+ if (a instanceof Date || b instanceof Date) {
20
+ return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
21
+ }
22
+
23
+ const aIsArr = Array.isArray(a);
24
+ const bIsArr = Array.isArray(b);
25
+ if (aIsArr !== bIsArr) return false;
26
+
27
+ if (aIsArr && bIsArr) {
28
+ if (a.length !== b.length) return false;
29
+ for (let i = 0; i < a.length; i++) {
30
+ if (!deepEqual(a[i], b[i])) return false;
31
+ }
32
+ return true;
33
+ }
34
+
35
+ const ao = a as Record<string, unknown>;
36
+ const bo = b as Record<string, unknown>;
37
+ const ak = Object.keys(ao);
38
+ const bk = Object.keys(bo);
39
+ if (ak.length !== bk.length) return false;
40
+ for (const k of ak) {
41
+ if (!Object.prototype.hasOwnProperty.call(bo, k)) return false;
42
+ if (!deepEqual(ao[k], bo[k])) return false;
43
+ }
44
+ return true;
45
+ }
46
+
47
+ /**
48
+ * 用 schema 校验 value。优先 Standard Schema(schema['~standard'].validate),
49
+ * 否则退化到 zod 风格的 .safeParse / .parse。校验通过 true,否则 false;任何异常 → false。
50
+ */
51
+ export async function validateSchema(value: unknown, schema: unknown): Promise<boolean> {
52
+ try {
53
+ const std = (schema as { ["~standard"]?: { validate?: (v: unknown) => unknown } } | null)?.[
54
+ "~standard"
55
+ ];
56
+ if (std && typeof std.validate === "function") {
57
+ // validate 可能同步也可能返回 Promise;成功结果不带 issues。
58
+ const result = (await std.validate(value)) as { issues?: unknown } | null | undefined;
59
+ return result != null && result.issues == null;
60
+ }
61
+
62
+ const zodish = schema as {
63
+ safeParse?: (v: unknown) => { success?: boolean };
64
+ parse?: (v: unknown) => unknown;
65
+ } | null;
66
+
67
+ if (zodish && typeof zodish.safeParse === "function") {
68
+ const result = zodish.safeParse(value);
69
+ return Boolean(result && result.success);
70
+ }
71
+ if (zodish && typeof zodish.parse === "function") {
72
+ zodish.parse(value);
73
+ return true;
74
+ }
75
+ return false;
76
+ } catch {
77
+ return false;
78
+ }
79
+ }