niceeval 0.1.0

package/src/context/context.ts

@@ -0,0 +1,427 @@
+// 构造 eval 作者拿到的高层上下文 t。这里把会话驱动(SessionManager)、
+// 断言收集(AssertionCollector)、作用域断言、judge 命名空间接到一起。
+
+import { readFile } from "node:fs/promises";
+import { basename, extname } from "node:path";
+import { SessionManager, RunSession, lastAssistantText } from "./session.ts";
+import { AssertionCollector } from "../scoring/collector.ts";
+import type { Spec } from "../scoring/collector.ts";
+import * as Scoped from "../scoring/scoped.ts";
+import { buildJudge } from "../scoring/judge.ts";
+import { EvalSkipped, EvalRequirementFailed, TurnFailed } from "./control-flow.ts";
+import { deriveRunFacts } from "../o11y/derive.ts";
+import { t } from "../i18n/index.ts";
+import type {
+  Agent,
+  DiffData,
+  DiffView,
+  InputFile,
+  InputRequest,
+  InputRequestFilter,
+  JudgeConfig,
+  Sandbox,
+  SandboxHandle,
+  ScoringContext,
+  ScriptResult,
+  SessionHandle,
+  StreamEvent,
+  Telemetry,
+  TestContext,
+  Turn,
+  TurnHandle,
+  Usage,
+  ValueAssertion,
+} from "../types.ts";
+
+/** t.file(path) 返回它,延迟到 finalize 再读沙箱文件;t.check 识别并解析它。 */
+export class FileRef {
+  constructor(public readonly path: string) {}
+}
+
+/** 运行器在 test 跑完后填进来的「迟到结果」(diff / 脚本),供 finalize 用。 */
+export interface LateResult {
+  diff: DiffData;
+  scripts: Record<string, ScriptResult>;
+}
+
+export interface ContextState {
+  readonly collector: AssertionCollector;
+  readonly manager: SessionManager;
+  skipReason?: string;
+  readonly late: LateResult;
+}
+
+export interface ContextDeps {
+  agent: Agent;
+  sandbox: Sandbox;
+  model?: string;
+  flags: Record<string, unknown>;
+  signal: AbortSignal;
+  log(msg: string): void;
+  judge: JudgeConfig | undefined;
+  /** tracing agent 的 OTLP 端点(运行器起接收器后注入);经 send ctx 透给 adapter。 */
+  telemetry?: Telemetry;
+}
+
+export function createEvalContext(deps: ContextDeps): { context: TestContext; state: ContextState } {
+  const manager = new SessionManager({
+    agent: deps.agent,
+    sandbox: deps.sandbox,
+    model: deps.model,
+    flags: deps.flags,
+    signal: deps.signal,
+    log: deps.log,
+    telemetry: deps.telemetry,
+  });
+  const collector = new AssertionCollector();
+  const state: ContextState = {
+    collector,
+    manager,
+    late: { diff: { generatedFiles: {}, deletedFiles: [] }, scripts: {} },
+  };
+
+  async function resolveValue(value: unknown, sc: ScoringContext): Promise<unknown> {
+    if (value instanceof FileRef) return (await sc.readFile(value.path)) ?? "";
+    return value;
+  }
+
+  const diffView: DiffView = {
+    get: (path) => state.late.diff.generatedFiles[path],
+    isEmpty: () =>
+      Object.keys(state.late.diff.generatedFiles).length === 0 &&
+      state.late.diff.deletedFiles.length === 0,
+    matches: (re) =>
+      Object.entries(state.late.diff.generatedFiles).some(([p, c]) => re.test(p) || re.test(c)),
+  };
+
+  const sandboxHandle: SandboxHandle = {
+    get sandboxId() {
+      return deps.sandbox.sandboxId;
+    },
+    get diff() {
+      return diffView;
+    },
+    runCommand: (cmd, args, opts) => deps.sandbox.runCommand(cmd, args, opts),
+    runShell: (script, opts) => deps.sandbox.runShell(script, opts),
+    readFile: (path) => deps.sandbox.readFile(path),
+    fileExists: (path) => deps.sandbox.fileExists(path),
+    readSourceFiles: (opts) => deps.sandbox.readSourceFiles(opts),
+    writeFiles: (files, targetDir) => deps.sandbox.writeFiles(files, targetDir),
+    uploadFiles: (files, targetDir) => deps.sandbox.uploadFiles(files, targetDir),
+    uploadDirectory: (localDir, targetDir, opts) => deps.sandbox.uploadDirectory(localDir, targetDir, opts),
+    downloadFile: (path) => deps.sandbox.downloadFile(path),
+    uploadFile: (path, content) => deps.sandbox.uploadFile(path, content),
+  };
+
+  function recordScoped(
+    spec: Spec,
+    getEvents: () => readonly StreamEvent[],
+    getStatus: () => "completed" | "failed" | "waiting",
+    getUsage: () => Usage,
+  ) {
+    return collector.record({
+      ...spec,
+      evaluate: (ctx) => {
+        const events = getEvents();
+        return spec.evaluate({
+          ...ctx,
+          events,
+          facts: deriveRunFacts(events),
+          status: getStatus(),
+          usage: getUsage(),
+        });
+      },
+    });
+  }
+
+  function makeJudge(session: RunSession) {
+    return buildJudge({
+      record: (spec) => collector.record(spec),
+      judge: deps.judge,
+      getOutput: () => conversationText(session.events),
+      getInput: () => session.lastInput,
+      signal: deps.signal,
+    });
+  }
+
+  function makeSessionHandle(session: RunSession): SessionHandle {
+    const scoped = (spec: Spec) =>
+      recordScoped(spec, () => session.events, () => session.lastStatus, () => session.usage);
+
+    const handle: SessionHandle = {
+      send: async (text) => makeTurnHandle(await manager.send(session, text), collector, deps),
+      sendFile: async (path, text) =>
+        makeTurnHandle(await manager.send(session, text ?? "", [await readInputFile(path)]), collector, deps),
+      requireInputRequest: (filter) => requireInputRequest(session, filter),
+      respond: async (...responses) => {
+        if (responses.length === 0) throw new Error("respond() requires at least one response");
+        session.pendingInputRequests.length = 0;
+        return makeTurnHandle(await manager.send(session, responses.join("\n")), collector, deps);
+      },
+      respondAll: async (optionId) => {
+        if (session.pendingInputRequests.length === 0) {
+          throw new Error("respondAll() requires at least one pending input request");
+        }
+        const responses = session.pendingInputRequests.map(() => optionId);
+        session.pendingInputRequests.length = 0;
+        return makeTurnHandle(await manager.send(session, responses.join("\n")), collector, deps);
+      },
+      get reply() {
+        return session.lastMessage;
+      },
+      get sessionId() {
+        return session.id;
+      },
+      get events() {
+        return session.events.slice();
+      },
+      succeeded: () => scoped(Scoped.succeeded()),
+      parked: () => scoped(Scoped.parked()),
+      messageIncludes: (token) => scoped(Scoped.messageIncludes(token)),
+      calledTool: (name, match) => scoped(Scoped.calledTool(name, match)),
+      notCalledTool: (name, match) => scoped(Scoped.notCalledTool(name, match)),
+      toolOrder: (names) => scoped(Scoped.toolOrder(names)),
+      usedNoTools: () => scoped(Scoped.usedNoTools()),
+      maxToolCalls: (max) => scoped(Scoped.maxToolCalls(max)),
+      loadedSkill: (skill) => scoped(Scoped.loadedSkill(skill)),
+      noFailedActions: () => scoped(Scoped.noFailedActions()),
+      event: (type, opts) => scoped(Scoped.eventOfType(type, opts)),
+      notEvent: (type) => scoped(Scoped.notEventOfType(type)),
+      calledSubagent: (name, match) => scoped(Scoped.calledSubagent(name, match)),
+      eventOrder: (types) => scoped(Scoped.eventOrder(types)),
+      eventsSatisfy: (predicate, label) => scoped(Scoped.eventsSatisfy(predicate, label)),
+      maxTokens: (max) => scoped(Scoped.maxTokens(max)),
+      maxCost: (usd) => scoped(Scoped.maxCost(usd)),
+      get usage() {
+        return session.usage;
+      },
+      get judge() {
+        return makeJudge(session);
+      },
+    };
+    return handle;
+  }
+
+  const primary = makeSessionHandle(manager.primary);
+  // primary.reply/sessionId/events/usage/judge 是 getter,读的是 manager.primary 的实时状态。
+  // 不能 `{ ...primary, ... }` 展开——对象展开会在展开的那一刻把每个 getter 求值成静态值,
+  // 之后 t.reply 就永远冻结在「还没 send 过」的初始状态(空字符串)。改用
+  // Object.getOwnPropertyDescriptors 搬运属性描述符,getter 保持 getter,照常读到最新状态。
+  const extra = {
+    newSession: () => makeSessionHandle(manager.newSession()),
+    signal: deps.signal,
+    model: deps.model,
+    flags: deps.flags,
+    log: deps.log,
+    skip: (reason: string) => {
+      if (reason.trim().length === 0) throw new Error(t("context.skipEmpty"));
+      state.skipReason = reason;
+      throw new EvalSkipped(reason);
+    },
+
+    check: (value: unknown, assertion: ValueAssertion) =>
+      collector.record({
+        name: assertion.name,
+        severity: assertion.severity,
+        threshold: assertion.threshold,
+        evaluate: async (sc) => assertion.score(await resolveValue(value, sc)),
+      }),
+    group: <T,>(title: string, fn: () => Promise<T> | T) => collector.withGroup(title, fn),
+    require: async (value: unknown, assertion: ValueAssertion) => {
+      const v = value instanceof FileRef ? await deps.sandbox.readFile(value.path).catch(() => "") : value;
+      const score = await assertion.score(v);
+      const passed = assertion.threshold === undefined ? score > 0 : score >= assertion.threshold;
+      collector.record({
+        name: assertion.name,
+        severity: "gate",
+        threshold: assertion.threshold,
+        evaluate: () => score,
+      });
+      if (!passed) throw new EvalRequirementFailed(assertion.name);
+      return value;
+    },
+
+    sandbox: sandboxHandle,
+    file: (path: string) => new FileRef(path) as unknown as string,
+    fileChanged: (path: string) => collector.record(Scoped.fileChanged(path)),
+    fileDeleted: (path: string) => collector.record(Scoped.fileDeleted(path)),
+    notInDiff: (re: RegExp) => collector.record(Scoped.notInDiff(re)),
+    noFailedShellCommands: () => collector.record(Scoped.noFailedShellCommands()),
+  };
+  const context = Object.defineProperties(
+    {},
+    {
+      ...Object.getOwnPropertyDescriptors(primary),
+      ...Object.getOwnPropertyDescriptors(extra),
+    },
+  ) as TestContext;
+
+  return { context, state };
+}
+
+/** 读本地文件(相对项目根)成 InputFile:推断 MIME + base64 编码,供 t.sendFile。 */
+async function readInputFile(path: string): Promise<InputFile> {
+  const buf = await readFile(path);
+  return { filename: basename(path), mimeType: mimeTypeFor(path), dataBase64: buf.toString("base64") };
+}
+
+function mimeTypeFor(path: string): string {
+  switch (extname(path).toLowerCase()) {
+    case ".png":
+      return "image/png";
+    case ".jpg":
+    case ".jpeg":
+      return "image/jpeg";
+    case ".gif":
+      return "image/gif";
+    case ".webp":
+      return "image/webp";
+    default:
+      return "application/octet-stream";
+  }
+}
+
+function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: ContextDeps): TurnHandle {
+  const message = lastAssistantText(turn.events) ?? "";
+  const facts = deriveRunFacts(turn.events);
+  const usage = turn.usage ?? { inputTokens: 0, outputTokens: 0 };
+
+  const scoped = (spec: Spec) =>
+    collector.record({
+      ...spec,
+      evaluate: (ctx) =>
+        spec.evaluate({
+          ...ctx,
+          events: turn.events,
+          facts,
+          status: turn.status,
+          usage,
+        }),
+    });
+
+  const handle: TurnHandle = {
+    events: turn.events,
+    toolCalls: facts.toolCalls,
+    status: turn.status,
+    message,
+    data: turn.data,
+    usage: turn.usage,
+    expectOk() {
+      if (turn.status === "failed") {
+        const lastError = [...turn.events]
+          .reverse()
+          .find((e): e is Extract<StreamEvent, { type: "error" }> => e.type === "error");
+        throw new TurnFailed(
+          lastError ? t("context.turnFailed", { message: lastError.message }) : undefined,
+        );
+      }
+      return handle;
+    },
+    outputEquals: (value) =>
+      collector.record({
+        name: "outputEquals",
+        severity: "gate",
+        evaluate: () => (deepEqual(turn.data, value) ? 1 : 0),
+      }),
+    outputMatches: (schema) =>
+      collector.record({
+        name: "outputMatches",
+        severity: "gate",
+        evaluate: () => (validateSchema(turn.data, schema) ? 1 : 0),
+      }),
+    messageIncludes: (token) => scoped(Scoped.messageIncludes(token)),
+    succeeded: () => scoped(Scoped.succeeded()),
+    calledTool: (name, match) => scoped(Scoped.calledTool(name, match)),
+    notCalledTool: (name, match) => scoped(Scoped.notCalledTool(name, match)),
+    toolOrder: (names) => scoped(Scoped.toolOrder(names)),
+    usedNoTools: () => scoped(Scoped.usedNoTools()),
+    maxToolCalls: (max) => scoped(Scoped.maxToolCalls(max)),
+    event: (type, opts) => scoped(Scoped.eventOfType(type, opts)),
+    notEvent: (type) => scoped(Scoped.notEventOfType(type)),
+    calledSubagent: (name, match) => scoped(Scoped.calledSubagent(name, match)),
+    eventOrder: (types) => scoped(Scoped.eventOrder(types)),
+    eventsSatisfy: (predicate, label) => scoped(Scoped.eventsSatisfy(predicate, label)),
+    judge: buildJudge({
+      record: (spec) => collector.record(spec),
+      judge: deps.judge,
+      getOutput: () => message,
+      getInput: () => "",
+      signal: deps.signal,
+    }),
+  };
+  return handle;
+}
+
+function conversationText(events: readonly StreamEvent[]): string {
+  return events
+    .filter((e): e is Extract<StreamEvent, { type: "message" }> => e.type === "message")
+    .map((e) => `${e.role}: ${e.text}`)
+    .join("\n");
+}
+
+function requireInputRequest(session: RunSession, filter?: InputRequestFilter): InputRequest {
+  const matches = session.pendingInputRequests.filter((request) => inputRequestMatches(request, filter));
+  if (matches.length !== 1) {
+    throw new Error(`Expected exactly one pending input request, found ${matches.length}`);
+  }
+  return matches[0] as InputRequest;
+}
+
+function inputRequestMatches(request: InputRequest, filter?: InputRequestFilter): boolean {
+  if (!filter) return true;
+  if (filter.id !== undefined && !stringMatches(request.id ?? "", filter.id)) return false;
+  if (filter.prompt !== undefined && !stringMatches(request.prompt ?? "", filter.prompt)) return false;
+  if (filter.display !== undefined && !stringMatches(request.display ?? "", filter.display)) return false;
+  if (filter.action !== undefined && !stringMatches(request.action ?? "", filter.action)) return false;
+  if (filter.optionIds !== undefined) {
+    const optionIds = new Set((request.options ?? []).map((o) => o.id));
+    if (!filter.optionIds.every((id) => optionIds.has(id))) return false;
+  }
+  if (filter.input !== undefined && !partialObjectMatches(request.input, filter.input)) return false;
+  return true;
+}
+
+function stringMatches(actual: string, expected: string | RegExp): boolean {
+  return expected instanceof RegExp ? expected.test(actual) : actual === expected;
+}
+
+function partialObjectMatches(actual: unknown, expected: Record<string, unknown>): boolean {
+  if (actual === null || typeof actual !== "object") return false;
+  const obj = actual as Record<string, unknown>;
+  for (const [key, value] of Object.entries(expected)) {
+    if (!deepEqual(obj[key], value)) return false;
+  }
+  return true;
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b || a === null || b === null || typeof a !== "object") return false;
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  const ak = Object.keys(a as object);
+  const bk = Object.keys(b as object);
+  if (ak.length !== bk.length) return false;
+  for (const k of ak) {
+    if (!deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k])) return false;
+  }
+  return true;
+}
+
+function validateSchema(value: unknown, schema: unknown): boolean {
+  try {
+    const std = (schema as { ["~standard"]?: { validate(v: unknown): { issues?: unknown } } })["~standard"];
+    if (std && typeof std.validate === "function") {
+      const r = std.validate(value) as { issues?: unknown };
+      return !(r && r.issues);
+    }
+    const zodLike = schema as { safeParse?(v: unknown): { success: boolean }; parse?(v: unknown): unknown };
+    if (typeof zodLike.safeParse === "function") return zodLike.safeParse(value).success;
+    if (typeof zodLike.parse === "function") {
+      zodLike.parse(value);
+      return true;
+    }
+  } catch {
+    return false;
+  }
+  return false;
+}

package/src/context/control-flow.ts

@@ -0,0 +1,27 @@
+// test(t) 里的非错误控制流信号。运行器据类型分流:跳过不是失败,断言失败不是异常。
+
+import { t } from "../i18n/index.ts";
+
+/** t.skip(reason):该 eval 不构成有效测试,记 skipped(不计入,不算 agent 挂)。 */
+export class EvalSkipped extends Error {
+  constructor(public readonly reason: string) {
+    super(`eval skipped: ${reason}`);
+    this.name = "EvalSkipped";
+  }
+}
+
+/** t.require / turn.expectOk 不过:正常的断言失败,中止后续,但已记录的断言决定判决。 */
+export class EvalRequirementFailed extends Error {
+  constructor(public readonly assertionName: string) {
+    super(`requirement failed: ${assertionName}`);
+    this.name = "EvalRequirementFailed";
+  }
+}
+
+/** 本轮 send 返回 failed,作者调了 expectOk():视为执行错误(eval failed)。 */
+export class TurnFailed extends Error {
+  constructor(message = t("context.turnFailedDefault")) {
+    super(message);
+    this.name = "TurnFailed";
+  }
+}

package/src/context/session.ts

@@ -0,0 +1,124 @@
+// 会话驱动:把 t.send(text) 翻成 agent.send(input, ctx),在同一沙箱里多轮 resume /
+// newSession,并把每轮的标准事件流与用量累加进整次运行(供作用域断言 / o11y)。
+
+import type { Agent, AgentContext, InputFile, InputRequest, Sandbox, StreamEvent, Telemetry, Turn, Usage } from "../types.ts";
+import { captureLoc } from "../source-loc.ts";
+import { t } from "../i18n/index.ts";
+
+/** 一条会话线的可变状态。adapter 读 isNew 决定是否 --resume,写 id 供下轮续接。 */
+export class RunSession {
+  id: string | undefined = undefined;
+  isNew = true;
+  index = 1;
+  lastMessage = "";
+  lastInput = "";
+  lastStatus: "completed" | "failed" | "waiting" = "completed";
+  readonly events: StreamEvent[] = [];
+  readonly pendingInputRequests: InputRequest[] = [];
+  readonly usage: Usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, requests: 0 };
+}
+
+export interface SessionDeps {
+  agent: Agent;
+  sandbox: Sandbox;
+  model?: string;
+  flags: Record<string, unknown>;
+  signal: AbortSignal;
+  log(msg: string): void;
+  /** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
+  telemetry?: Telemetry;
+}
+
+export class SessionManager {
+  /** 整次运行(所有会话、所有轮)累计的标准事件流。 */
+  readonly allEvents: StreamEvent[] = [];
+  readonly usage: Usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, requests: 0 };
+  lastStatus: "completed" | "failed" | "waiting" = "completed";
+
+  readonly primary: RunSession;
+  private readonly sessions: RunSession[] = [];
+  private turnCount = 0;
+  private sessionCount = 0;
+
+  constructor(private readonly deps: SessionDeps) {
+    this.primary = this.newSession();
+  }
+
+  newSession(): RunSession {
+    const s = new RunSession();
+    s.index = ++this.sessionCount;
+    this.sessions.push(s);
+    return s;
+  }
+
+  async send(session: RunSession, text: string, files?: readonly InputFile[]): Promise<Turn> {
+    // 抓住作者调 t.send / t.sendFile 那一行(view 把回复叠回这一行)。
+    const loc = captureLoc();
+    const ctx: AgentContext = {
+      signal: this.deps.signal,
+      model: this.deps.model,
+      flags: this.deps.flags,
+      sandbox: this.deps.sandbox,
+      session: session as unknown as AgentContext["session"],
+      telemetry: this.deps.telemetry,
+      log: this.deps.log,
+    };
+
+    const n = ++this.turnCount;
+    const attach = files?.length ? ` 📎${files.length}` : "";
+    const preview = (text.replace(/\s+/g, " ").slice(0, 36) || (files?.[0]?.filename ?? t("session.fileFallback"))) + attach;
+    const turnLabel = session.index === 1
+      ? t("session.turn.primary", { turn: n })
+      : t("session.turn.secondary", { session: session.index, turn: n });
+    this.deps.log(`${turnLabel} → "${preview}…"`);
+    const t0 = Date.now();
+
+    session.lastInput = text;
+    const userEvent: StreamEvent = { type: "message", role: "user", text, loc };
+    this.allEvents.push(userEvent);
+    session.events.push(userEvent);
+    session.pendingInputRequests.length = 0;
+    const turn = await this.deps.agent.send({ text, files }, ctx);
+
+    this.allEvents.push(...turn.events);
+    session.events.push(...turn.events);
+    session.pendingInputRequests.push(
+      ...turn.events
+        .filter((e): e is Extract<StreamEvent, { type: "input.requested" }> => e.type === "input.requested")
+        .map((e) => e.request),
+    );
+    if (turn.usage) {
+      accumulateUsage(this.usage, turn.usage);
+      accumulateUsage(session.usage, turn.usage);
+    }
+    session.isNew = false;
+    session.lastStatus = turn.status;
+    this.lastStatus = turn.status;
+    const reply = lastAssistantText(turn.events);
+    if (reply !== undefined) session.lastMessage = reply;
+
+    const tok = (turn.usage?.inputTokens ?? 0) + (turn.usage?.outputTokens ?? 0);
+    const tools = turn.events.filter((e) => e.type === "action.called").length;
+    this.deps.log(
+      `${turnLabel} ← ${turn.status} · ${t("session.tools", { count: tools })} · ${tok} tok · ${Math.round((Date.now() - t0) / 1000)}s`,
+    );
+    return turn;
+  }
+}
+
+export function lastAssistantText(events: readonly StreamEvent[]): string | undefined {
+  for (let i = events.length - 1; i >= 0; i--) {
+    const e = events[i];
+    if (e.type === "message" && e.role === "assistant" && e.text.trim()) return e.text;
+  }
+  return undefined;
+}
+
+function accumulateUsage(acc: Usage, add: Usage): void {
+  acc.inputTokens += add.inputTokens ?? 0;
+  acc.outputTokens += add.outputTokens ?? 0;
+  acc.cacheReadTokens = (acc.cacheReadTokens ?? 0) + (add.cacheReadTokens ?? 0);
+  acc.cacheWriteTokens = (acc.cacheWriteTokens ?? 0) + (add.cacheWriteTokens ?? 0);
+  acc.requests = (acc.requests ?? 0) + (add.requests ?? 1);
+  if (add.costUSD !== undefined) acc.costUSD = (acc.costUSD ?? 0) + add.costUSD;
+}

package/src/define.ts

@@ -0,0 +1,112 @@
+// 定义入口:把用户对象规格化成核心认得的形状。路径即身份 —— 这里禁止手写 id,
+// 由发现阶段从文件路径推导(见 runner/discover.ts)。
+
+import type {
+  Agent,
+  Config,
+  CustomSandboxSpec,
+  DockerSandboxSpec,
+  E2BSandboxSpec,
+  EvalDef,
+  ExperimentDef,
+  RemoteAgentDef,
+  SandboxAgentDef,
+  VercelSandboxSpec,
+} from "./types.ts";
+import { t } from "./i18n/index.ts";
+
+const SANDBOX_DEFAULT_CAPS = {
+  conversation: true,
+  toolObservability: true,
+  workspace: true,
+  sandbox: true,
+} as const;
+
+const REMOTE_DEFAULT_CAPS = {
+  conversation: true,
+  toolObservability: true,
+} as const;
+
+/** 沙箱型 agent:在沙箱里 spawn 一个 coding agent 的 CLI,跑完读回 transcript。 */
+export function defineSandboxAgent(def: SandboxAgentDef): Agent {
+  if (!def.name) throw new Error(t("define.sandboxAgentNameRequired"));
+  return {
+    name: def.name,
+    capabilities: { ...SANDBOX_DEFAULT_CAPS, ...(def.capabilities ?? {}) },
+    setup: def.setup,
+    tracing: def.tracing,
+    send: def.send,
+    teardown: def.teardown,
+  };
+}
+
+/** 远程 / 进程内 agent:在 send 里直接驱动你的函数 / 服务。 */
+export function defineAgent(def: RemoteAgentDef): Agent {
+  if (!def.name) throw new Error(t("define.agentNameRequired"));
+  return {
+    name: def.name,
+    capabilities: { ...REMOTE_DEFAULT_CAPS, ...(def.capabilities ?? {}) },
+    setup: def.setup,
+    tracing: def.tracing,
+    send: def.send,
+    teardown: def.teardown,
+  };
+}
+
+/** 会话型 eval。禁止提供 id —— 从路径推导。 */
+export function defineEval(def: EvalDef): EvalDef {
+  if ((def as { id?: unknown }).id !== undefined) {
+    throw new Error(t("define.evalIdRejected"));
+  }
+  if (typeof def.test !== "function") {
+    throw new Error(t("define.evalTestRequired"));
+  }
+  return def;
+}
+
+/** 实验:可签入的运行配置(怎么跑这批 eval)。 */
+export function defineExperiment(def: ExperimentDef): ExperimentDef {
+  if ((def as { id?: unknown }).id !== undefined) {
+    throw new Error(t("define.experimentIdRejected"));
+  }
+  if (!def.agent) throw new Error(t("define.experimentAgentRequired"));
+  return def;
+}
+
+/** 项目级配置。 */
+export function defineConfig(config: Config): Config {
+  return config;
+}
+
+// ───────────────────────── Sandbox 工厂 ─────────────────────────
+// Sandbox 与 agent 一样用数据结构带参数(见 docs/sandbox.md)。这些工厂只是把
+// 后端 + 参数包成 spec 对象;真正的行为在 sandbox/<backend>.ts 里,由 resolve.ts 派发。
+
+/** Docker 沙箱:本地容器。`image` 可覆盖默认 `node:*-slim`(预制模板:烘焙好 agent CLI 的镜像)。 */
+export function dockerSandbox(opts: Omit<DockerSandboxSpec, "backend"> = {}): DockerSandboxSpec {
+  return { backend: "docker", ...opts };
+}
+
+/** Vercel Sandbox:microVM。`snapshotId` 从已有快照起(预制模板:烘焙好 agent CLI 的快照)。 */
+export function vercelSandbox(opts: Omit<VercelSandboxSpec, "backend"> = {}): VercelSandboxSpec {
+  return { backend: "vercel", ...opts };
+}
+
+/** E2B 沙箱。`template` 选 e2b 模板名/ID(预制模板:如 `"niceeval-agents"`);省略用 e2b 默认 `"base"`。 */
+export function e2bSandbox(opts: Omit<E2BSandboxSpec, "backend"> = {}): E2BSandboxSpec {
+  return { backend: "e2b", ...opts };
+}
+
+/**
+ * 自定义沙箱后端:`create` 直接返回一个实现 `Sandbox` 接口的实例,不需要 niceeval 内置支持
+ * 这个后端名字。用于接入 docker/vercel/e2b 之外的运行环境(自建 VM、Modal、Fly 等)。
+ */
+export function defineSandbox(def: {
+  name: string;
+  create: CustomSandboxSpec["create"];
+  recommendedConcurrency?: number;
+}): CustomSandboxSpec {
+  if (!def.name) throw new Error(t("define.sandboxNameRequired"));
+  if (typeof def.create !== "function") throw new Error(t("define.sandboxCreateRequired"));
+  return { backend: def.name, create: def.create, recommendedConcurrency: def.recommendedConcurrency };
+}