niceeval 0.1.1 → 0.1.2

package/src/cli.ts

@@ -8,6 +8,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { pathToFileURL } from "node:url";
+import { parseArgs as nodeParseArgs } from "node:util";
 import { discoverEvals, discoverExperiments, makeFilter } from "./runner/discover.ts";
 import { runEvals, type AgentRun } from "./runner/run.ts";
 import { stopAllSandboxes, liveSandboxCount } from "./sandbox/registry.ts";
@@ -16,13 +17,22 @@ import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
 import { JUnit } from "./runner/reporters/json.ts";
 import { Live as LiveReporter, type LiveRow } from "./runner/reporters/live.ts";
 import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
-import { buildView, startViewServer, loadMostRecentResults } from "./view/index.ts";
+import { buildView, startViewServer, loadMostRecentResults, IncompatibleResultsError } from "./view/index.ts";
 import { t } from "./i18n/index.ts";
+import { formatThrown } from "./util.ts";
 import type { Config, DiscoveredExperiment, Reporter } from "./types.ts";
 
+/** `niceeval view <summary.json>` 指向版本不同的报告时:打印 npx 提示后退出,不抛堆栈。 */
+function exitOnIncompatibleResults(e: unknown): never {
+  if (e instanceof IncompatibleResultsError) {
+    process.stderr.write(e.message);
+    process.exit(1);
+  }
+  throw e;
+}
+
 interface Flags {
   agent?: string;
-  sandbox?: string;
   model?: string;
   runs?: number;
   maxConcurrency?: number;
@@ -38,85 +48,111 @@ interface Flags {
   open?: boolean;
   out?: string;
   port?: number;
+  help: boolean;
 }
 
-const BOOL_FLAGS = new Set([
-  "dry",
-  "quiet",
-  "force",
-  "strict",
-  "early-exit",
-  "no-early-exit",
-  "open",
-  "no-open",
-  "force",
-  "watch",
-  "json",
-]);
+// 表驱动的 flag 定义(node:util parseArgs)。--no-x 显式声明,不依赖 allowNegative(需 Node 20.14+,
+// engines 是 >=18)。未知 flag 由 strict 模式报清晰错误,不再静默吞掉后面的位置参数。
+const FLAG_OPTIONS = {
+  agent: { type: "string" },
+  model: { type: "string" },
+  runs: { type: "string" },
+  "max-concurrency": { type: "string" },
+  timeout: { type: "string" },
+  budget: { type: "string" },
+  tag: { type: "string" },
+  junit: { type: "string" },
+  out: { type: "string" },
+  port: { type: "string" },
+  // --sandbox 已移除(sandbox 归 config/experiment);留着解析是为了给出迁移提示而非「未知 flag」。
+  sandbox: { type: "string" },
+  // --watch / --json:文档曾接受、尚未实现的 flag。容忍(no-op)而不是硬报「未知 flag」,
+  // 免得照旧文档写的脚本直接崩;真正实现前不接任何行为。
+  watch: { type: "boolean" },
+  json: { type: "boolean" },
+  dry: { type: "boolean" },
+  quiet: { type: "boolean" },
+  force: { type: "boolean" },
+  strict: { type: "boolean" },
+  "early-exit": { type: "boolean" },
+  "no-early-exit": { type: "boolean" },
+  open: { type: "boolean" },
+  "no-open": { type: "boolean" },
+  help: { type: "boolean", short: "h" },
+} as const;
+
+function numberFlag(name: string, raw: string | undefined): number | undefined {
+  if (raw === undefined) return undefined;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) {
+    process.stderr.write(t("cli.flag.invalidNumber", { flag: name, value: raw }));
+    process.exit(1);
+  }
+  return n;
+}
 
 function parseArgs(argv: string[]): { command: string; positionals: string[]; flags: Flags } {
   if (argv[0] === "--") argv = argv.slice(1);
-  const positionals: string[] = [];
-  const flags: Flags = { dry: false, quiet: false, force: false, strict: false };
-  let command = "run";
-  let i = 0;
 
-  // 第一个非 flag token 若是已知命令,则为命令
-  const commands = new Set(["exp", "list", "view", "clean", "init", "watch", "run"]);
-  if (argv[0] && !argv[0].startsWith("-") && commands.has(argv[0])) {
-    command = argv[0];
-    i = 1;
+  let values: Record<string, string | boolean | undefined>;
+  let rawPositionals: string[];
+  try {
+    const parsed = nodeParseArgs({ args: argv, options: FLAG_OPTIONS, allowPositionals: true, strict: true });
+    values = parsed.values as Record<string, string | boolean | undefined>;
+    rawPositionals = parsed.positionals;
+  } catch (e) {
+    process.stderr.write(t("cli.flag.parseError", { message: e instanceof Error ? e.message : String(e) }));
+    process.exit(1);
   }
 
-  for (; i < argv.length; i++) {
-    const tok = argv[i]!;
-    if (tok.startsWith("--")) {
-      const name = tok.slice(2);
-      if (name === "no-early-exit") {
-        flags.earlyExit = false;
-        continue;
-      }
-      if (name === "early-exit") {
-        flags.earlyExit = true;
-        continue;
-      }
-      if (name === "no-open") {
-        flags.open = false;
-        continue;
-      }
-      if (name === "open") {
-        flags.open = true;
-        continue;
-      }
-      if (BOOL_FLAGS.has(name)) {
-        if (name === "dry") flags.dry = true;
-        else if (name === "quiet") flags.quiet = true;
-        else if (name === "force") flags.force = true;
-        else if (name === "strict") flags.strict = true;
-        continue;
-      }
-      const value = argv[++i];
-      switch (name) {
-        case "agent": flags.agent = value; break;
-        case "sandbox": flags.sandbox = value; break;
-        case "model": flags.model = value; break;
-        case "runs": flags.runs = Number(value); break;
-        case "max-concurrency": flags.maxConcurrency = Number(value); break;
-case "timeout": flags.timeout = Number(value); break;
-        case "budget": flags.budget = Number(value); break;
-        case "tag": flags.tag = value; break;
-        case "junit": flags.junit = value; break;
-        case "out": flags.out = value; break;
-        case "port": flags.port = Number(value); break;
-        default: break; // 未知 flag 忽略
-      }
-    } else {
-      positionals.push(tok);
-    }
+  if (values.sandbox !== undefined) {
+    process.stderr.write(t("cli.sandboxFlagRemoved"));
+    process.exit(1);
+  }
+
+  // 第一个位置参数若是已知命令,则为命令;其余是 eval id 前缀 / view 输入。
+  const commands = new Set(["exp", "list", "view", "clean", "init", "watch", "run"]);
+  let command = "run";
+  let positionals = rawPositionals;
+  if (rawPositionals[0] && commands.has(rawPositionals[0])) {
+    command = rawPositionals[0];
+    positionals = rawPositionals.slice(1);
   }
+
+  const flags: Flags = {
+    agent: values.agent as string | undefined,
+    model: values.model as string | undefined,
+    runs: numberFlag("runs", values.runs as string | undefined),
+    maxConcurrency: numberFlag("max-concurrency", values["max-concurrency"] as string | undefined),
+    timeout: numberFlag("timeout", values.timeout as string | undefined),
+    budget: numberFlag("budget", values.budget as string | undefined),
+    tag: values.tag as string | undefined,
+    junit: values.junit as string | undefined,
+    out: values.out as string | undefined,
+    port: numberFlag("port", values.port as string | undefined),
+    dry: values.dry === true,
+    quiet: values.quiet === true,
+    force: values.force === true,
+    strict: values.strict === true,
+    earlyExit: values["no-early-exit"] === true ? false : values["early-exit"] === true ? true : undefined,
+    open: values["no-open"] === true ? false : values.open === true ? true : undefined,
+    help: values.help === true,
+  };
   return { command, positionals, flags };
 }
 
+/** 调度项的环境变量层(标志 > 环境变量 > config > 默认,见 docs/cli.md「配置优先级」)。 */
+function envNumber(name: string): number | undefined {
+  const raw = process.env[name]?.trim();
+  if (!raw) return undefined;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) {
+    process.stderr.write(t("cli.envInvalidNumber", { name, value: raw }));
+    process.exit(1);
+  }
+  return n;
+}
+
 /** 加载 cwd/.env(不覆盖已有环境变量)。 */
 async function loadDotenv(cwd: string): Promise<void> {
   const path = join(cwd, ".env");
@@ -203,13 +239,19 @@ async function main(): Promise<void> {
   await loadDotenv(cwd);
   const { command, positionals, flags } = parseArgs(process.argv.slice(2));
 
+  // --help 不需要 config,先于一切命令处理。
+  if (flags.help) {
+    process.stdout.write(t("cli.help"));
+    process.exit(0);
+  }
+
   if (command === "view") {
     if (flags.out) {
-      const out = await buildView({ input: positionals[0], out: flags.out });
+      const out = await buildView({ input: positionals[0], out: flags.out }).catch(exitOnIncompatibleResults);
       process.stdout.write(t("cli.view.exported", { out }));
       process.exit(0);
     }
-    const server = await startViewServer({ input: positionals[0], port: flags.port });
+    const server = await startViewServer({ input: positionals[0], port: flags.port }).catch(exitOnIncompatibleResults);
     process.stdout.write(t("cli.view.url", { url: server.url }));
     if (flags.open !== false) {
       const opened = await openBrowser(server.url);
@@ -273,11 +315,11 @@ async function main(): Promise<void> {
         agent: exp.agent,
         model: exp.model,
         flags: exp.flags ?? {},
-        runs: flags.runs ?? exp.runs ?? 1,
+        runs: flags.runs ?? envNumber("NICEEVAL_RUNS") ?? exp.runs ?? 1,
         earlyExit: flags.earlyExit ?? exp.earlyExit ?? true,
-        sandbox: flags.sandbox ?? exp.sandbox ?? config.sandbox,
-        timeoutMs: flags.timeout ?? exp.timeoutMs ?? config.timeoutMs,
-        budget: flags.budget ?? exp.budget,
+        sandbox: exp.sandbox ?? config.sandbox,
+        timeoutMs: flags.timeout ?? envNumber("NICEEVAL_TIMEOUT") ?? exp.timeoutMs ?? config.timeoutMs,
+        budget: flags.budget ?? envNumber("NICEEVAL_BUDGET") ?? exp.budget,
         evalFilter: evalsFilterFromExperiment(exp.evals, extraPatterns),
         experimentId: exp.id,
         strict: flags.strict,
@@ -399,7 +441,12 @@ async function main(): Promise<void> {
     evals,
     agentRuns,
     reporters,
-    maxConcurrency: flags.maxConcurrency ?? expMaxConcurrency ?? config.maxConcurrency ?? sandboxDefaultConcurrency,
+    maxConcurrency:
+      flags.maxConcurrency ??
+      envNumber("NICEEVAL_MAX_CONCURRENCY") ??
+      expMaxConcurrency ??
+      config.maxConcurrency ??
+      sandboxDefaultConcurrency,
     signal: ctrl.signal,
     onProgress,
     priorResults,
@@ -414,7 +461,7 @@ async function main(): Promise<void> {
 }
 
 main().catch(async (e) => {
-  process.stderr.write(t("cli.error", { error: e instanceof Error ? e.stack ?? e.message : String(e) }));
+  process.stderr.write(t("cli.error", { error: formatThrown(e) }));
   // 真·崩溃路径也别留孤儿:强清还活着的沙箱(带超时),再退。
   await stopAllSandboxes();
   process.exit(2);

package/src/context/context.test.ts

@@ -1,16 +1,17 @@
 import { describe, expect, it } from "vitest";
 import { createEvalContext } from "./context.ts";
 import { includes } from "../expect/index.ts";
-import type { Agent, AgentContext, Sandbox, StreamEvent, Turn, TurnInput } from "../types.ts";
+import type { Agent, AgentContext, InputRequest, Sandbox, StreamEvent, Turn, TurnInput } from "../types.ts";
 
 // 计算工具 + 最终回复"1 + 1 = **2** 哦!😊"——复现截图里的场景:助手回复明明包含 "2",
 // 但 t.check(t.reply, includes("2")) 却失败。
 function calculatorAgent(): Agent {
   return {
     name: "calculator",
-    capabilities: { conversation: true, toolObservability: true },
+    // 测试注入了真实的 fake sandbox,kind: "sandbox" 让 t.sandbox 过沙箱能力守卫。
+    kind: "sandbox",
     async send(_input: TurnInput, ctx: AgentContext): Promise<Turn> {
-      ctx.session.id = "sess-1";
+      ctx.session.capture("sess-1");
       const events: StreamEvent[] = [
         { type: "action.called", callId: "c1", name: "calculate", input: { expr: "1+1" }, tool: undefined },
         { type: "action.result", callId: "c1", output: { result: 2 }, status: "completed" },
@@ -21,8 +22,12 @@ function calculatorAgent(): Agent {
   };
 }
 
-function fakeSandbox(): Sandbox {
+type FakeSandbox = Sandbox & { calls: { uploadDirectory: [string, string | undefined][] } };
+
+function fakeSandbox(): FakeSandbox {
+  const calls: { uploadDirectory: [string, string | undefined][] } = { uploadDirectory: [] };
   return {
+    workdir: "/sandbox/work",
     runCommand: async () => { throw new Error("not implemented"); },
     runShell: async () => { throw new Error("not implemented"); },
     readFile: async () => "",
@@ -36,23 +41,27 @@ function fakeSandbox(): Sandbox {
     }),
     writeFiles: async () => {},
     uploadFiles: async () => {},
-    uploadDirectory: async () => {},
+    uploadDirectory: async (localDir, targetDir) => {
+      calls.uploadDirectory.push([localDir, targetDir]);
+    },
     stop: async () => {},
     sandboxId: "fake",
     otlpHost: null,
     downloadFile: async () => Buffer.from(""),
     uploadFile: async () => {},
+    calls,
   };
 }
 
-function makeContext(agent: Agent) {
+function makeContext(agent: Agent, sandbox = fakeSandbox(), evalBaseDir?: string) {
   return createEvalContext({
     agent,
-    sandbox: fakeSandbox(),
+    sandbox,
     flags: {},
     signal: new AbortController().signal,
     log: () => {},
     judge: undefined,
+    evalBaseDir,
   });
 }
 
@@ -79,6 +88,25 @@ describe("createEvalContext / TestContext live state", () => {
     });
     expect(result.passed).toBe(true);
     expect(result.score).toBe(1);
+    expect(result.evidence).toBeUndefined();
+  });
+
+  it("t.check(...) attaches the actually-checked value as evidence when it fails", async () => {
+    const { context, state } = makeContext(calculatorAgent());
+    await context.send("1+1=?");
+    context.check(context.reply, includes("banana"));
+
+    const [result] = await state.collector.finalize({
+      events: [],
+      facts: { toolCalls: [], subagentCalls: [], inputRequests: [], parked: false, messageCount: 0, compactions: 0 },
+      diff: state.late.diff,
+      scripts: state.late.scripts,
+      usage: { inputTokens: 0, outputTokens: 0 },
+      status: "completed",
+      readFile: async () => undefined,
+    });
+    expect(result.passed).toBe(false);
+    expect(result.evidence).toBe("1 + 1 = **2** 哦!😊");
   });
 
   it("t.events reflects the turn's events after send(), not an empty snapshot", async () => {
@@ -93,4 +121,172 @@ describe("createEvalContext / TestContext live state", () => {
     await context.send("1+1=?");
     expect(context.sessionId).toBe("sess-1");
   });
+
+  it("exposes sandbox workdir to eval authors", () => {
+    const { context } = makeContext(calculatorAgent());
+    expect(context.sandbox.workdir).toBe("/sandbox/work");
+  });
+
+  it("resolves uploadDirectory local paths relative to the eval file directory", async () => {
+    const sandbox = fakeSandbox();
+    const { context } = makeContext(calculatorAgent(), sandbox, "/repo/evals/nested");
+
+    await context.sandbox.uploadDirectory("../fixtures/app", "src");
+
+    expect(sandbox.calls.uploadDirectory).toEqual([
+      ["/repo/evals/fixtures/app", "src"],
+    ]);
+  });
+});
+
+// agent 按调用次数依次吐出预设的 Turn 序列,同时记下每次收到的 TurnInput——
+// 用来断言 t.respond()/t.respondAll() 到 adapter 的 input.responses 长什么样。
+function scriptedAgent(turns: Turn[]): Agent & { received: TurnInput[] } {
+  const received: TurnInput[] = [];
+  let i = 0;
+  const agent: Agent = {
+    name: "scripted",
+    kind: "remote",
+    async send(input: TurnInput) {
+      received.push(input);
+      const turn = turns[Math.min(i, turns.length - 1)] as Turn;
+      i++;
+      return turn;
+    },
+  };
+  return Object.assign(agent, { received });
+}
+
+function waitingTurn(...requests: InputRequest[]): Turn {
+  return {
+    status: "waiting",
+    events: requests.map((request) => ({ type: "input.requested" as const, request })),
+  };
+}
+
+function completedTurn(text = "ok"): Turn {
+  return { status: "completed", events: [{ type: "message", role: "assistant", text }] };
+}
+
+describe("t.respond() / t.respondAll(): structured InputResponse", () => {
+  it("string arg hitting a pending request's option becomes { requestId, optionId }", async () => {
+    const agent = scriptedAgent([
+      waitingTurn({ id: "req_1", action: "send_email", options: [{ id: "approve" }, { id: "deny" }] }),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("draft an email");
+    await context.respond("approve");
+
+    expect(agent.received[1]?.text).toBe("approve");
+    expect(agent.received[1]?.responses).toEqual([{ requestId: "req_1", optionId: "approve" }]);
+  });
+
+  it("string arg that matches no option becomes free-text { requestId, text }", async () => {
+    const agent = scriptedAgent([
+      waitingTurn({ id: "req_1", action: "send_email", options: [{ id: "approve" }, { id: "deny" }] }),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("draft an email");
+    await context.respond("change the recipient to ceo@corp.com");
+
+    expect(agent.received[1]?.responses).toEqual([
+      { requestId: "req_1", text: "change the recipient to ceo@corp.com" },
+    ]);
+  });
+
+  it("string arg throws a clear error when multiple requests are pending (needs object form to disambiguate)", async () => {
+    const agent = scriptedAgent([
+      waitingTurn(
+        { id: "req_1", action: "edit_a", options: [{ id: "approve" }, { id: "deny" }] },
+        { id: "req_2", action: "edit_b", options: [{ id: "approve" }, { id: "deny" }] },
+      ),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("apply two edits");
+
+    await expect(context.respond("approve")).rejects.toThrow(/字符串回答无法对位|cannot be matched/);
+  });
+
+  it("object form { request, optionId } disambiguates when multiple requests are pending", async () => {
+    const agent = scriptedAgent([
+      waitingTurn(
+        { id: "req_1", action: "edit_a", options: [{ id: "approve" }, { id: "deny" }] },
+        { id: "req_2", action: "edit_b", options: [{ id: "approve" }, { id: "deny" }] },
+      ),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("apply two edits");
+    const req2 = context.requireInputRequest({ action: "edit_b" });
+    await context.respond({ request: req2, optionId: "deny" });
+
+    expect(agent.received[1]?.text).toBe("deny");
+    expect(agent.received[1]?.responses).toEqual([{ requestId: "req_2", optionId: "deny" }]);
+  });
+
+  it("object form with an optionId not present in the request's options throws instead of silently forwarding", async () => {
+    const agent = scriptedAgent([
+      waitingTurn({ id: "req_1", action: "send_email", options: [{ id: "approve" }, { id: "deny" }] }),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("draft an email");
+    const req = context.requireInputRequest();
+
+    await expect(context.respond({ request: req, optionId: "yolo" })).rejects.toThrow(/req_1/);
+    // 校验先于发送:没有第二次 send 发生。
+    expect(agent.received.length).toBe(1);
+  });
+
+  it("respondAll(optionId) answers every pending request and joins input.text with \\n", async () => {
+    const agent = scriptedAgent([
+      waitingTurn(
+        { id: "req_1", action: "edit_a", options: [{ id: "approve" }, { id: "deny" }] },
+        { id: "req_2", action: "edit_b", options: [{ id: "approve" }, { id: "deny" }] },
+      ),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("apply two edits");
+    await context.respondAll("approve");
+
+    expect(agent.received[1]?.text).toBe("approve\napprove");
+    expect(agent.received[1]?.responses).toEqual([
+      { requestId: "req_1", optionId: "approve" },
+      { requestId: "req_2", optionId: "approve" },
+    ]);
+  });
+
+  it("respondAll(optionId) validates the option against every pending request before sending anything", async () => {
+    const agent = scriptedAgent([
+      waitingTurn(
+        { id: "req_1", action: "edit_a", options: [{ id: "approve" }, { id: "deny" }] },
+        { id: "req_2", action: "edit_b", options: [{ id: "yes" }, { id: "no" }] },
+      ),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("apply two edits");
+
+    await expect(context.respondAll("approve")).rejects.toThrow(/req_2/);
+    expect(agent.received.length).toBe(1);
+  });
+
+  it("TurnInput.responses reaches the adapter unchanged (not derived/guessed on the adapter side)", async () => {
+    const agent = scriptedAgent([
+      waitingTurn({ id: "req_1", action: "send_email", options: [{ id: "approve" }, { id: "deny" }] }),
+      completedTurn(),
+    ]);
+    const { context } = makeContext(agent);
+    await context.send("draft an email");
+    const req = context.requireInputRequest();
+    await context.respond({ request: req, text: "wait, add a subject line first" });
+
+    expect(agent.received[1]?.responses).toEqual([
+      { requestId: "req_1", text: "wait, add a subject line first" },
+    ]);
+  });
 });