niceeval 0.1.1 → 0.1.2

package/src/context/types.ts

@@ -0,0 +1,205 @@
+// context 域类型:eval 作者拿到的 `t`(TestContext)及其子句柄(turn / session / sandbox 视图)。
+// `t` 的形状按 Agent 能力组装(见 docs/architecture.md「能力决定形状」)。
+
+import type { InputRequest, StreamEvent, ToolCall, Usage } from "../o11y/types.ts";
+import type { AssertionHandle, ValueAssertion } from "../scoring/types.ts";
+import type {
+  CommandOptions,
+  CommandResult,
+  ReadSourceFilesOptions,
+  SandboxFile,
+  SourceFiles,
+} from "../sandbox/types.ts";
+
+/** t.send() 返回的句柄:从事件流派生便利字段 + expectOk。 */
+export interface TurnHandle {
+  readonly events: StreamEvent[];
+  readonly toolCalls: readonly ToolCall[];
+  readonly status: "completed" | "failed" | "waiting";
+  readonly message: string;
+  readonly data?: unknown;
+  readonly usage?: Usage;
+  /** 上一轮若 failed 则抛(中止后续)。 */
+  expectOk(): TurnHandle;
+  outputEquals(value: unknown): AssertionHandle;
+  outputMatches(schema: unknown): AssertionHandle;
+  /** 断言本轮助手回复包含 token(仅限本轮事件流,不跨轮)。 */
+  messageIncludes(token: string | RegExp): AssertionHandle;
+  succeeded(): AssertionHandle;
+  calledTool(name: string, match?: ToolMatch): AssertionHandle;
+  notCalledTool(name: string, match?: ToolMatch): AssertionHandle;
+  toolOrder(names: string[]): AssertionHandle;
+  usedNoTools(): AssertionHandle;
+  maxToolCalls(max: number): AssertionHandle;
+  event(type: StreamEvent["type"], opts?: { count?: number }): AssertionHandle;
+  notEvent(type: StreamEvent["type"]): AssertionHandle;
+  calledSubagent(name: string, match?: SubagentMatch): AssertionHandle;
+  eventOrder(types: StreamEvent["type"][]): AssertionHandle;
+  eventsSatisfy(predicate: (events: readonly StreamEvent[]) => boolean, label?: string): AssertionHandle;
+  readonly judge: JudgeNamespace;
+}
+
+/** autoevals 子命名空间:结构化的参考材料对照评估(closedQA / factuality / summarizes)。 */
+export interface AutoevalsNamespace {
+  closedQA(question: string, opts?: { on?: string; model?: string }): AssertionHandle;
+  factuality(expected: string, opts?: { on?: string; model?: string }): AssertionHandle;
+  summarizes(source: string, opts?: { on?: string; model?: string }): AssertionHandle;
+}
+
+export interface JudgeNamespace {
+  /** 结构化对照评估的子命名空间(t.judge.autoevals.closedQA / .factuality / .summarizes)。 */
+  autoevals: AutoevalsNamespace;
+}
+
+export interface DiffView {
+  get(path: string): string | undefined;
+  isEmpty(): boolean;
+  matches(re: RegExp): boolean;
+}
+
+/** 工具匹配小语言。 */
+export interface ToolMatch {
+  input?: Record<string, unknown>;
+  count?: number;
+  status?: "completed" | "failed" | "rejected";
+}
+
+export interface SubagentMatch {
+  count?: number;
+  status?: "completed" | "failed";
+  remoteUrl?: string | RegExp;
+}
+
+export interface InputRequestFilter {
+  id?: string | RegExp;
+  prompt?: string | RegExp;
+  display?: string | RegExp;
+  action?: string | RegExp;
+  input?: Record<string, unknown>;
+  optionIds?: readonly string[];
+}
+
+/**
+ * `t.respond(...)` 的对象形式:显式指名回答的是哪条请求(多个请求并停时用它消歧,
+ * 字符串形式做不到)。`optionId` 与 `text` 二选一——`optionId` 必须存在于
+ * `request.options` 里,写错直接抛;`text` 是自由文本,不做校验。
+ */
+export interface RespondAnswer {
+  readonly request: InputRequest;
+  readonly optionId?: string;
+  readonly text?: string;
+}
+
+/** eval 作者可见的受限沙箱视图:能执行命令 / 文件 IO / 读最终 diff,但不能 stop。 */
+export interface SandboxHandle {
+  readonly workdir: string;
+  runCommand(cmd: string, args?: string[], opts?: CommandOptions): Promise<CommandResult>;
+  runShell(script: string, opts?: CommandOptions): Promise<CommandResult>;
+  readFile(path: string): Promise<string>;
+  fileExists(path: string): Promise<boolean>;
+  readSourceFiles(opts?: ReadSourceFilesOptions): Promise<SourceFiles>;
+  writeFiles(files: Record<string, string>, targetDir?: string): Promise<void>;
+  uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void>;
+  uploadDirectory(localDir: string, targetDir?: string, opts?: { ignore?: string[] }): Promise<void>;
+  downloadFile(path: string): Promise<Buffer>;
+  uploadFile(path: string, content: Buffer): Promise<void>;
+  readonly sandboxId: string;
+  readonly diff: DiffView;
+}
+
+export interface SessionHandle {
+  send(text: string): Promise<TurnHandle>;
+  sendFile(path: string, text?: string): Promise<TurnHandle>;
+  requireInputRequest(filter?: InputRequestFilter): InputRequest;
+  respond(...responses: (string | RespondAnswer)[]): Promise<TurnHandle>;
+  respondAll(optionId: string): Promise<TurnHandle>;
+  readonly reply: string;
+  readonly sessionId: string | undefined;
+  readonly events: readonly StreamEvent[];
+  succeeded(): AssertionHandle;
+  parked(): AssertionHandle;
+  messageIncludes(token: string | RegExp): AssertionHandle;
+  calledTool(name: string, match?: ToolMatch): AssertionHandle;
+  notCalledTool(name: string, match?: ToolMatch): AssertionHandle;
+  toolOrder(names: string[]): AssertionHandle;
+  usedNoTools(): AssertionHandle;
+  maxToolCalls(max: number): AssertionHandle;
+  loadedSkill(skill: string): AssertionHandle;
+  noFailedActions(): AssertionHandle;
+  event(type: StreamEvent["type"], opts?: { count?: number }): AssertionHandle;
+  notEvent(type: StreamEvent["type"]): AssertionHandle;
+  calledSubagent(name: string, match?: SubagentMatch): AssertionHandle;
+  eventOrder(types: StreamEvent["type"][]): AssertionHandle;
+  eventsSatisfy(predicate: (events: readonly StreamEvent[]) => boolean, label?: string): AssertionHandle;
+  maxTokens(max: number): AssertionHandle;
+  maxCost(usd: number): AssertionHandle;
+  readonly usage: Usage;
+  readonly judge: JudgeNamespace;
+}
+
+/**
+ * eval 作者拿到的高层上下文。运行器按 agent 能力组装;tsx 不做类型检查,所以这里
+ * 用一个宽接口承载全部动作(运行时按 capability 守卫)。
+ */
+export interface TestContext {
+  // 会话
+  send(text: string): Promise<TurnHandle>;
+  /** 发一条带文件(图片等多模态输入)的消息。`path` 相对项目根;读出后 base64 随 TurnInput.files 交给 adapter。 */
+  sendFile(path: string, text?: string): Promise<TurnHandle>;
+  requireInputRequest(filter?: InputRequestFilter): InputRequest;
+  respond(...responses: (string | RespondAnswer)[]): Promise<TurnHandle>;
+  respondAll(optionId: string): Promise<TurnHandle>;
+  readonly reply: string;
+  readonly sessionId: string | undefined;
+  readonly events: readonly StreamEvent[];
+  newSession(): SessionHandle;
+
+  // 运行上下文
+  readonly signal: AbortSignal;
+  readonly model?: string;
+  readonly flags: Readonly<Record<string, unknown>>;
+  log(msg: string): void;
+  skip(reason: string): never;
+
+  // 值级断言
+  check(value: unknown, assertion: ValueAssertion): AssertionHandle;
+  require(value: unknown, assertion: ValueAssertion): Promise<unknown>;
+  /**
+   * 把一组断言归到一个有标题的分组下(对照 vitest 的 test('title', ...))。纯组织/报告用,
+   * 不改打分:组里每条断言仍独立计分。可嵌套(标题用 › 连接)。
+   */
+  group<T>(title: string, fn: () => Promise<T> | T): Promise<T>;
+
+  // 作用域断言(工具 / 会话)
+  succeeded(): AssertionHandle;
+  parked(): AssertionHandle;
+  messageIncludes(token: string | RegExp): AssertionHandle;
+  calledTool(name: string, match?: ToolMatch): AssertionHandle;
+  notCalledTool(name: string, match?: ToolMatch): AssertionHandle;
+  toolOrder(names: string[]): AssertionHandle;
+  usedNoTools(): AssertionHandle;
+  maxToolCalls(max: number): AssertionHandle;
+  loadedSkill(skill: string): AssertionHandle;
+  noFailedActions(): AssertionHandle;
+  event(type: StreamEvent["type"], opts?: { count?: number }): AssertionHandle;
+  notEvent(type: StreamEvent["type"]): AssertionHandle;
+  calledSubagent(name: string, match?: SubagentMatch): AssertionHandle;
+  eventOrder(types: StreamEvent["type"][]): AssertionHandle;
+  eventsSatisfy(predicate: (events: readonly StreamEvent[]) => boolean, label?: string): AssertionHandle;
+
+  // 工作区 / 沙箱
+  readonly sandbox: SandboxHandle;
+  file(path: string): string;
+  fileChanged(path: string): AssertionHandle;
+  fileDeleted(path: string): AssertionHandle;
+  notInDiff(re: RegExp): AssertionHandle;
+  noFailedShellCommands(): AssertionHandle;
+
+  // 效率 / 成本
+  readonly usage: Usage;
+  maxTokens(max: number): AssertionHandle;
+  maxCost(usd: number): AssertionHandle;
+
+  // judge
+  readonly judge: JudgeNamespace;
+}

package/src/define.ts

@@ -15,26 +15,15 @@ import type {
 } from "./types.ts";
 import { t } from "./i18n/index.ts";
 
-const SANDBOX_DEFAULT_CAPS = {
-  conversation: true,
-  toolObservability: true,
-  workspace: true,
-  sandbox: true,
-} as const;
-
-const REMOTE_DEFAULT_CAPS = {
-  conversation: true,
-  toolObservability: true,
-} as const;
-
 /** 沙箱型 agent:在沙箱里 spawn 一个 coding agent 的 CLI,跑完读回 transcript。 */
 export function defineSandboxAgent(def: SandboxAgentDef): Agent {
   if (!def.name) throw new Error(t("define.sandboxAgentNameRequired"));
   return {
     name: def.name,
-    capabilities: { ...SANDBOX_DEFAULT_CAPS, ...(def.capabilities ?? {}) },
+    kind: "sandbox",
     setup: def.setup,
     tracing: def.tracing,
+    spanMapper: def.spanMapper,
     send: def.send,
     teardown: def.teardown,
   };
@@ -45,9 +34,10 @@ export function defineAgent(def: RemoteAgentDef): Agent {
   if (!def.name) throw new Error(t("define.agentNameRequired"));
   return {
     name: def.name,
-    capabilities: { ...REMOTE_DEFAULT_CAPS, ...(def.capabilities ?? {}) },
+    kind: "remote",
     setup: def.setup,
     tracing: def.tracing,
+    spanMapper: def.spanMapper,
     send: def.send,
     teardown: def.teardown,
   };

package/src/expect/index.ts

@@ -3,6 +3,11 @@
 
 import type { Severity, ValueAssertion } from "../types.ts";
 import { stripComments } from "../util.ts";
+import { deepEqual, validateSchema } from "../scoring/match.ts";
+
+// 自定义匹配器作者用的公共类型(docs/scoring.md 的 `Assertion` 即它)。
+export type { Severity, ValueAssertion } from "../types.ts";
+export type { ValueAssertion as Assertion } from "../types.ts";
 
 /** includes / excludes 的可选项。stripComments:先剥注释再匹配(只对真实代码生效)。 */
 export interface MatchOptions {
@@ -45,48 +50,6 @@ function safeLabel(v: unknown): string {
   }
 }
 
-/** 小而全的深比较:处理基本值、NaN、数组、Date、纯对象。 */
-function deepEqual(a: unknown, b: unknown): boolean {
-  if (a === b) return true;
-
-  // NaN === NaN
-  if (typeof a === "number" && typeof b === "number") {
-    return Number.isNaN(a) && Number.isNaN(b);
-  }
-
-  if (a === null || b === null || typeof a !== "object" || typeof b !== "object") {
-    return false;
-  }
-
-  // Date 按时间戳比
-  if (a instanceof Date || b instanceof Date) {
-    return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
-  }
-
-  const aIsArr = Array.isArray(a);
-  const bIsArr = Array.isArray(b);
-  if (aIsArr !== bIsArr) return false;
-
-  if (aIsArr && bIsArr) {
-    if (a.length !== b.length) return false;
-    for (let i = 0; i < a.length; i++) {
-      if (!deepEqual(a[i], b[i])) return false;
-    }
-    return true;
-  }
-
-  const ao = a as Record<string, unknown>;
-  const bo = b as Record<string, unknown>;
-  const ak = Object.keys(ao);
-  const bk = Object.keys(bo);
-  if (ak.length !== bk.length) return false;
-  for (const k of ak) {
-    if (!Object.prototype.hasOwnProperty.call(bo, k)) return false;
-    if (!deepEqual(ao[k], bo[k])) return false;
-  }
-  return true;
-}
-
 /** 经典 DP 编辑距离(滚动两行)。 */
 function levenshtein(a: string, b: string): number {
   const m = a.length;
@@ -153,35 +116,9 @@ export function equals(expected: unknown): ValueAssertion {
  * 否则退化到 zod 风格的 .safeParse / .parse。校验通过 1,否则 0;任何异常 → 0。
  */
 export function matches(schema: unknown): ValueAssertion {
-  return createAssertion("matches(schema)", "gate", async (value) => {
-    try {
-      const std = (schema as { ["~standard"]?: { validate?: (v: unknown) => unknown } } | null)?.[
-        "~standard"
-      ];
-      if (std && typeof std.validate === "function") {
-        // validate 可能同步也可能返回 Promise;成功结果不带 issues。
-        const result = (await std.validate(value)) as { issues?: unknown } | null | undefined;
-        return result != null && result.issues == null ? 1 : 0;
-      }
-
-      const zodish = schema as {
-        safeParse?: (v: unknown) => { success?: boolean };
-        parse?: (v: unknown) => unknown;
-      } | null;
-
-      if (zodish && typeof zodish.safeParse === "function") {
-        const result = zodish.safeParse(value);
-        return result && result.success ? 1 : 0;
-      }
-      if (zodish && typeof zodish.parse === "function") {
-        zodish.parse(value);
-        return 1;
-      }
-      return 0;
-    } catch {
-      return 0;
-    }
-  });
+  return createAssertion("matches(schema)", "gate", async (value) =>
+    (await validateSchema(value, schema)) ? 1 : 0,
+  );
 }
 
 /** 归一化 Levenshtein 相似度 [0,1]。默认软分,阈值 0.6。 */

package/src/i18n/core.ts

@@ -0,0 +1,28 @@
+// i18n 内核:CLI(src/i18n/index.ts)与 view 前端(src/view/app/i18n.ts)共用。
+// 必须保持环境无关(不 import node/browser API),vite 前端和 node CLI 都直接打包它。
+// 两侧只在这里之外各自注入 locale 来源(env vs navigator+localStorage)与默认值(zh-CN vs en)。
+
+export type Locale = "zh-CN" | "en";
+
+export type Vars = Record<string, string | number | boolean | undefined>;
+
+/** 一份字典:消息 key → 文案(可含 {{var}} 占位)。 */
+export type Dictionary<K extends string = string> = Record<K, string>;
+
+/** 把 {{var}} 占位替换成 vars 里的值;缺失的变量替换成空串而不是保留占位。 */
+export function interpolate(message: string, vars: Vars = {}): string {
+  return message.replace(/\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g, (_match, name: string) =>
+    vars[name] === undefined ? "" : String(vars[name]),
+  );
+}
+
+/**
+ * 把 "zh_CN"、"en-US"、navigator.language 这类原始值归一成受支持的 Locale。
+ * C / POSIX 视为「未指定」返回 undefined(交给下一个候选);非中文一律落 en。
+ */
+export function normalizeLocale(raw: string | undefined): Locale | undefined {
+  if (!raw) return undefined;
+  const value = raw.trim().toLowerCase().replace("_", "-");
+  if (!value || value === "c" || value === "posix") return undefined;
+  return value.startsWith("zh") ? "zh-CN" : "en";
+}

package/src/i18n/en.ts

@@ -3,12 +3,14 @@ import type { Messages } from "./zh-CN.ts";
 export const en = {
   "agent.installFailed": "Install failed: {{key}}\n{{tail}}",
   "agent.unknown": "Unknown agent \"{{name}}\". Registered agents: {{known}}.",
-  "bub.diagnose.exitCode": "bub run exited with code {{code}}",
-  "bub.diagnose.lastError": "last error: {{message}}",
-  "bub.diagnose.noTape": "tape was not generated",
-  "bub.diagnose.outputTail": "output tail: {{tail}}",
-  "bub.diagnose.zeroEvents": "tape exists but contains 0 events",
+  "agent.diagnose.exitCode": "agent run exited with code {{code}}",
+  "agent.diagnose.lastError": "last error: {{message}}",
+  "agent.diagnose.noTranscript": "transcript was not generated",
+  "agent.diagnose.outputTail": "output tail: {{tail}}",
+  "agent.diagnose.zeroEvents": "transcript exists but contains 0 events",
+  "bub.homeDetectFailed": "Failed to detect sandbox $HOME (empty output from `printf $HOME`). Refusing to fall back to a backend-specific path; check the sandbox backend.",
   "bub.installFailed": "bub install failed after {{attempts}} attempts:\n{{tail}}",
+  "bub.setupNotRun": "bub adapter setup() has not run in this sandbox (missing home/workspace info). The runner must call setup before send.",
   "checkpoint.emptyTar": "checkpoint: tar is empty (paths: {{paths}})",
   "cli.all": "(all)",
   "cli.browserOpenFailed": "Could not open the browser automatically. Open manually: {{url}}\n",
@@ -19,6 +21,30 @@ export const en = {
   "cli.dry.noMatches": "(no matches)",
   "cli.dry.row": "  {{who}}{{experiment}}: {{evals}}  ×{{runs}}\n",
   "cli.error": "niceeval error: {{error}}\n",
+  "cli.flag.invalidNumber": "Flag --{{flag}} expects a number, got \"{{value}}\".\n",
+  "runner.budgetUnenforceable":
+    "budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
+  "judge.modelMissing":
+    "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).",
+  "loaders.yamlMissing":
+    "loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
+  "cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
+  "cli.envInvalidNumber": "Environment variable {{name}} is not a number: \"{{value}}\".\n",
+  "cli.help":
+    "niceeval — agent-native evals\n\n" +
+    "Usage:\n" +
+    "  niceeval exp [group|experiment] [eval-id-prefix…]   run experiments\n" +
+    "  niceeval list                                       list discovered evals\n" +
+    "  niceeval view [summary.json|dir] [--out d] [--port n] [--no-open]\n" +
+    "  niceeval clean                                      delete .niceeval/ artifacts\n" +
+    "  niceeval init                                       scaffold config + evals/\n\n" +
+    "Flags:\n" +
+    "  --runs n  --max-concurrency n  --timeout ms  --budget usd  --tag t\n" +
+    "  --early-exit / --no-early-exit  --strict  --force  --dry  --quiet\n" +
+    "  --junit path  --out dir  --port n  --open / --no-open  -h, --help\n\n" +
+    "Positional args only select which evals to run (id prefixes); which agent and\n" +
+    "how to run come from experiments/ + flags. Env overrides (flag > env > config):\n" +
+    "  NICEEVAL_RUNS  NICEEVAL_MAX_CONCURRENCY  NICEEVAL_TIMEOUT  NICEEVAL_BUDGET\n",
   "cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
   "cli.eval.noMatchHintExperiment": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
   "cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
@@ -36,9 +62,13 @@ export const en = {
   "cli.run.experimentRequired": "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n",
   "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
+  "cli.sandboxFlagRemoved": "`--sandbox` is not a CLI flag. Set `sandbox` in the experiment (or `niceeval.config.ts` as a project-wide fallback) to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n",
   "cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
   "cli.view.exported": "Exported eval report page: {{out}}\n",
+  "cli.view.incompatible": "{{dir}}: written by niceeval {{producer}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nRun `{{command}}` to view it.\n",
   "cli.view.url": "niceeval view: {{url}}\n",
+  "context.capabilityMissing":
+    "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.",
   "context.skipEmpty": "skip() requires a non-empty reason.",
   "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
   "context.turnFailedDefault": "This send returned failed (turn status = failed)",
@@ -56,6 +86,13 @@ export const en = {
   "docker.imagePullStart": "Pulling Docker image: {{image}}...",
   "docker.readFileFailed": "Failed to read file {{path}}: {{stderr}}",
   "docker.unsupportedRuntime": "Unsupported runtime: {{runtime}}",
+  "hitl.answerNeedsOptionOrText": "The object form of t.respond needs either optionId or text (neither was given).",
+  "hitl.invalidOption": "Answer \"{{optionId}}\" is not an option of request {{requestId}} ({{options}}).",
+  "hitl.noOptions": "this request has no options",
+  "hitl.requestMissingId": "This input.requested request has no stable id, so a response cannot be built — the adapter must give every pending request a stable id.",
+  "hitl.respondAllEmpty": "There is no pending input.requested request; respond() / respondAll() cannot work. Confirm the turn parked with t.parked(), then answer via t.requireInputRequest() or t.respond().",
+  "hitl.respondEmpty": "t.respond(...) requires at least one answer.",
+  "hitl.stringAmbiguous": "There are {{count}} pending input requests; a plain-string answer cannot be matched to one. Use the { request, optionId } or { request, text } object form to name it explicitly.",
   "judge.apiKeyMissing": "judge is missing an API key (CODEX_API_KEY / OPENAI_API_KEY).",
   "judge.httpError": "judge HTTP {{status}}: {{body}}",
   "judge.probeFailed": "judge precheck failed ({{model}}): {{error}}",
@@ -93,6 +130,9 @@ export const en = {
   "report.table.status": "Status",
   "report.table.successRate": "Success Rate",
   "report.table.tokens": "Tokens",
+  "otel.noSpans": "otel: 0 spans this turn — endpoint not wired? (env not injected / service not restarted / no flush)",
+  "otel.portInUse": "OTLP receiver port {{port}} is already in use (another process is bound to it). Pick a free port in defineConfig({ telemetry: { port } }), or stop whatever is using {{port}} and retry.",
+  "otel.windowAttribution": "otel: spans missing our traceparent, attributing by time window (turns for this agent serialized; concurrency resumes once W3C propagation is confirmed)",
   "runner.diffProgress": "captured diff: {{changed}} changed / {{deleted}} deleted",
   "runner.driveAgent": "driving agent...",
   "runner.evalSetup": "eval setup (installing dependencies)...",
@@ -101,6 +141,7 @@ export const en = {
   "runner.otlpInSandbox": "OTLP in-sandbox collector -> {{endpoint}}{{proto}}",
   "runner.otlpOverride": "OTLP receiver (host override) -> {{endpoint}}",
   "runner.otlpReceiver": "OTLP receiver -> {{endpoint}}{{proto}}",
+  "runner.otlpShared": "OTLP shared receiver (run-scoped) -> {{endpoint}}",
   "runner.remoteSandboxUnavailable": "remote agents do not have sandbox.{{method}}; use a sandbox agent or remove workspace assertions.",
   "runner.reporterDiagnostic": "  · [diagnostic] {{stage}} failed (ignored): {{message}}\n",
   "runner.scoreJudge": "scoring / judge...",
@@ -113,6 +154,7 @@ export const en = {
   "runner.resumeCarry": "  · reusing {{carried}} passing results from last run, re-running {{retry}} evals\n",
   "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
+  "sandbox.missingSpec": "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",

package/src/i18n/index.ts

@@ -1,25 +1,16 @@
+// CLI 侧 i18n:内核(插值/归一)在 core.ts;这里只注入 env 来源与 zh-CN 默认值。
+
 import { en } from "./en.ts";
 import { zhCN, type MessageKey, type Messages } from "./zh-CN.ts";
+import { interpolate, normalizeLocale, type Locale, type Vars } from "./core.ts";
 
-export type Locale = "zh-CN" | "en";
-
-type Vars = Record<string, string | number | boolean | undefined>;
+export type { Locale, Vars } from "./core.ts";
 
 const dictionaries: Record<Locale, Messages> = {
   "zh-CN": zhCN,
   en,
 };
 
-function normalizeLocale(raw: string | undefined): Locale | undefined {
-  if (!raw) return undefined;
-  const value = raw.trim().toLowerCase().replace("_", "-");
-  if (!value) return undefined;
-  if (value === "c" || value === "posix") return undefined;
-  if (value.startsWith("zh")) return "zh-CN";
-  if (value.startsWith("en")) return "en";
-  return "en";
-}
-
 export function detectLocale(env: NodeJS.ProcessEnv = process.env): Locale {
   return (
     normalizeLocale(env.NICEEVAL_LANG) ??
@@ -32,8 +23,5 @@ export function detectLocale(env: NodeJS.ProcessEnv = process.env): Locale {
 }
 
 export function t(key: MessageKey, vars: Vars = {}): string {
-  const message = dictionaries[detectLocale()][key];
-  return message.replace(/\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g, (_match, name: string) =>
-    vars[name] === undefined ? "" : String(vars[name]),
-  );
+  return interpolate(dictionaries[detectLocale()][key], vars);
 }

package/src/i18n/zh-CN.ts

@@ -1,12 +1,14 @@
 export const zhCN = {
   "agent.installFailed": "安装失败:{{key}}\n{{tail}}",
   "agent.unknown": "未知 agent \"{{name}}\"。已注册:{{known}}。",
-  "bub.diagnose.exitCode": "bub run 退出码 {{code}}",
-  "bub.diagnose.lastError": "最后错误:{{message}}",
-  "bub.diagnose.noTape": "tape 未生成",
-  "bub.diagnose.outputTail": "输出末尾:{{tail}}",
-  "bub.diagnose.zeroEvents": "tape 存在但 0 事件",
+  "agent.diagnose.exitCode": "agent 运行退出码 {{code}}",
+  "agent.diagnose.lastError": "最后错误:{{message}}",
+  "agent.diagnose.noTranscript": "transcript 未生成",
+  "agent.diagnose.outputTail": "输出末尾:{{tail}}",
+  "agent.diagnose.zeroEvents": "transcript 存在但 0 事件",
+  "bub.homeDetectFailed": "无法探测沙箱 $HOME(printf $HOME 输出为空)。不兜底到后端专属固定路径,请检查沙箱后端。",
   "bub.installFailed": "bub 安装失败(重试 {{attempts}} 次):\n{{tail}}",
+  "bub.setupNotRun": "bub adapter 的 setup() 尚未在该沙箱运行(缺 home/workspace 信息);运行器应先调 setup 再 send。",
   "checkpoint.emptyTar": "checkpoint: tar 为空(paths: {{paths}})",
   "cli.all": "(全部)",
   "cli.browserOpenFailed": "无法自动打开浏览器,请手动访问:{{url}}\n",
@@ -17,6 +19,30 @@ export const zhCN = {
   "cli.dry.noMatches": "(无匹配)",
   "cli.dry.row": "  {{who}}{{experiment}}: {{evals}}  ×{{runs}}\n",
   "cli.error": "niceeval 出错:{{error}}\n",
+  "cli.flag.invalidNumber": "标志 --{{flag}} 需要数字,收到 \"{{value}}\"。\n",
+  "runner.budgetUnenforceable":
+    "{{budgetKey}} 的 budget:连续多个 attempt 完成后都拿不到成本数据(agent 不上报用量且模型不在价格表)——该 agent 的 budget 无法执行,取消护栏继续跑。\n",
+  "judge.modelMissing":
+    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定评判模型(没有内置默认模型)。",
+  "loaders.yamlMissing":
+    "loadYaml(\"{{path}}\") 需要 YAML 解析器:请先 `pnpm add yaml`(或改用 loadJson + JSON 数据集)。",
+  "cli.flag.parseError": "{{message}}\n运行 `niceeval --help` 查看用法。\n",
+  "cli.envInvalidNumber": "环境变量 {{name}} 不是数字:\"{{value}}\"。\n",
+  "cli.help":
+    "niceeval — agent-native evals\n\n" +
+    "用法:\n" +
+    "  niceeval exp [组|实验] [eval-id 前缀…]   跑实验\n" +
+    "  niceeval list                            列出发现到的 eval\n" +
+    "  niceeval view [summary.json|目录] [--out 目录] [--port n] [--no-open]\n" +
+    "  niceeval clean                           删除 .niceeval/ 历史工件\n" +
+    "  niceeval init                            脚手架 config + evals/\n\n" +
+    "标志:\n" +
+    "  --runs n  --max-concurrency n  --timeout ms  --budget usd  --tag t\n" +
+    "  --early-exit / --no-early-exit  --strict  --force  --dry  --quiet\n" +
+    "  --junit path  --out dir  --port n  --open / --no-open  -h, --help\n\n" +
+    "位置参数只选「跑哪些 eval」(id 前缀);对着哪个 agent、怎么跑来自 experiments/ 与\n" +
+    "标志。环境变量覆盖(标志 > 环境变量 > config):\n" +
+    "  NICEEVAL_RUNS  NICEEVAL_MAX_CONCURRENCY  NICEEVAL_TIMEOUT  NICEEVAL_BUDGET\n",
   "cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
   "cli.eval.noMatchHintExperiment": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
   "cli.eval.noMatchKnown": "已发现 {{count}} 个 eval:{{evals}}\n",
@@ -34,9 +60,13 @@ export const zhCN = {
   "cli.run.experimentRequired": "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n",
   "cli.run.experimentRequiredHint": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "已发现实验:{{experiments}}\n",
+  "cli.sandboxFlagRemoved": "`--sandbox` 不是 CLI flag。请在 experiment(或 niceeval.config.ts 做全项目兜底)里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n",
   "cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
   "cli.view.exported": "已导出实验查看页:{{out}}\n",
+  "cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
   "cli.view.url": "niceeval view: {{url}}\n",
+  "context.capabilityMissing":
+    "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。",
   "context.skipEmpty": "skip() 需要一个非空理由。",
   "context.turnFailed": "本轮 send 返回 failed(turn status = failed):{{message}}",
   "context.turnFailedDefault": "本轮 send 返回 failed(turn status = failed)",
@@ -54,6 +84,13 @@ export const zhCN = {
   "docker.imagePullStart": "Pulling Docker image: {{image}}...",
   "docker.readFileFailed": "Failed to read file {{path}}: {{stderr}}",
   "docker.unsupportedRuntime": "Unsupported runtime: {{runtime}}",
+  "hitl.answerNeedsOptionOrText": "t.respond 的对象形式需要 optionId 或 text 二选一(两者都没给)。",
+  "hitl.invalidOption": "回答 \"{{optionId}}\" 不是请求 {{requestId}} 的可选项({{options}})。",
+  "hitl.noOptions": "该请求没有可选项",
+  "hitl.requestMissingId": "该 input.requested 请求没有稳定的 id,无法生成 responses——adapter 侧要给每条待回答请求一个稳定 id。",
+  "hitl.respondAllEmpty": "没有待回答的 input.requested 请求,respond() / respondAll() 无法工作;先用 t.parked() 确认停轮,再用 t.requireInputRequest() 或 t.respond() 回答。",
+  "hitl.respondEmpty": "t.respond(...) 至少需要一个回答。",
+  "hitl.stringAmbiguous": "有 {{count}} 条待回答请求,字符串回答无法对位,请用 { request, optionId } 或 { request, text } 对象形式显式指名。",
   "judge.apiKeyMissing": "judge 缺少 API key(CODEX_API_KEY / OPENAI_API_KEY)。",
   "judge.httpError": "judge HTTP {{status}}: {{body}}",
   "judge.probeFailed": "judge 预检失败({{model}}): {{error}}",
@@ -91,6 +128,9 @@ export const zhCN = {
   "report.table.status": "状态",
   "report.table.successRate": "成功率",
   "report.table.tokens": "Tokens",
+  "otel.noSpans": "otel:本轮 0 span —— 端点没接上?(env 没注入 / 服务没重启 / 没 flush)",
+  "otel.portInUse": "OTLP 接收端口 {{port}} 已被占用(另一个进程占着这个端口)。在 defineConfig({ telemetry: { port } }) 里换一个空闲端口,或者停掉占用 {{port}} 的进程后重试。",
+  "otel.windowAttribution": "otel:span 未带本轮 traceparent,按时间窗口归属(该 agent 的轮次已串行;应用支持 W3C 传播后自动并发)",
   "runner.diffProgress": "采 diff:{{changed}} 改 / {{deleted}} 删",
   "runner.driveAgent": "驱动 agent…",
   "runner.evalSetup": "eval setup(装依赖)…",
@@ -99,6 +139,7 @@ export const zhCN = {
   "runner.otlpInSandbox": "OTLP in-sandbox collector → {{endpoint}}{{proto}}",
   "runner.otlpOverride": "OTLP 接收器(覆盖 host) → {{endpoint}}",
   "runner.otlpReceiver": "OTLP 接收器 → {{endpoint}}{{proto}}",
+  "runner.otlpShared": "OTLP 共享接收器(run 级) → {{endpoint}}",
   "runner.remoteSandboxUnavailable": "remote agent 没有 sandbox.{{method}};请改用 sandbox agent 或移除 workspace 断言。",
   "runner.reporterDiagnostic": "  · [diagnostic] {{stage}} 失败(已忽略):{{message}}\n",
   "runner.scoreJudge": "评分 / judge…",
@@ -111,6 +152,7 @@ export const zhCN = {
   "runner.resumeCarry": "  · 复用上次 {{carried}} 个通过的结果,重跑 {{retry}} 个 eval\n",
   "runner.useRemoteAgent": "使用 remote agent(不创建沙箱)…",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend not implemented; use docker, vercel, or e2b",
+  "sandbox.missingSpec": "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",

package/src/index.ts

@@ -14,6 +14,8 @@ export type {
   Turn,
   TurnInput,
   InputFile,
+  InputResponse,
+  RespondAnswer,
   TurnHandle,
   SessionHandle,
   TestContext,
@@ -30,6 +32,16 @@ export type {
   ReporterEvent,
   EvalResult,
   RunSummary,
+  RunShape,
+  AssertionResult,
+  ExperimentRunInfo,
+  DiffData,
+  DiffView,
+  ScriptResult,
+  SandboxHandle,
+  CommandResult,
+  InputRequest,
+  InputRequestFilter,
   O11ySummary,
   TraceSpan,
   SpanKind,