niceeval 0.3.0 → 0.4.1

package/src/results/types.ts

@@ -0,0 +1,98 @@
+// results 域类型:openResults 返回的类型化句柄(读取面契约,见 docs/results-lib.md)。
+//
+// 结果数据类型(EvalResult / RunSummary / StreamEvent / …)仍住在各自的域文件里,
+// 这里只 import,不搬家 —— 「类型的家」迁移是 results-lib 设计里单独的一步,不在本次范围。
+
+import type { EvalResult, RunSummary } from "../runner/types.ts";
+import type { O11ySummary, StreamEvent, TraceSpan } from "../o11y/types.ts";
+import type { DiffData } from "../scoring/types.ts";
+import type { SourceArtifact } from "../shared/types.ts";
+
+/** attempt 级工件的种类;文件名恒为 `<kind>.json`,布局见 docs/results-format.md。 */
+export const ARTIFACT_KINDS = ["events", "sources", "trace", "o11y", "diff"] as const;
+export type ArtifactKind = (typeof ARTIFACT_KINDS)[number];
+
+/**
+ * 回到证据的引用:run 目录名(相对结果根目录)+ summary.results[] 下标。
+ * Reports 的 MetricCell.refs 与 view 深链 `#/attempt/<run>/<result>` 用的是同一个身份。
+ */
+export interface AttemptRef {
+  run: string;
+  result: number;
+}
+
+/** 一个落盘 run 目录;忠实反映磁盘,不合并、不去重。 */
+export interface RunHandle {
+  /** run 目录的绝对路径(summary.json 所在目录)。 */
+  dir: string;
+  /** 与写入侧同一类型的 RunSummary(results[] 是瘦身条目)。 */
+  summary: RunSummary;
+  /** summary.results[] 逐条包一层懒加载句柄。 */
+  attempts: AttemptHandle[];
+}
+
+/**
+ * 单个 eval attempt:瘦身条目 + 重工件的懒加载方法。
+ * 懒加载即存在性判断:工件缺失返回 null,不抛错;同一 handle 内读过一次即缓存。
+ */
+export interface AttemptHandle {
+  /** 所属物理 run;去重「保留最新 run 的那份」靠它比较新旧。 */
+  run: RunHandle;
+  /** 本 attempt 的证据引用(run 目录名 + summary.results 下标);消费方不必自己反查下标。 */
+  ref: AttemptRef;
+  /** EvalResult 瘦身条目:判决、断言、用量、成本、experiment 元数据。 */
+  result: EvalResult;
+  events(): Promise<StreamEvent[] | null>;
+  trace(): Promise<TraceSpan[] | null>;
+  o11y(): Promise<O11ySummary | null>;
+  diff(): Promise<DiffData | null>;
+  sources(): Promise<SourceArtifact[] | null>;
+}
+
+/**
+ * 快照 = 一个 experiment 在一个 run 里的那部分 attempt(experiment × run 切片)。
+ * 一个 run 目录可以装多个 experiment(`niceeval exp compare`),所以「每个 experiment
+ * 最新一次」只能用快照粒度表达,不能用 run 粒度。
+ */
+export interface SnapshotHandle {
+  /** experiment id;结果里缺 experimentId 时以 "<agent>/<model>" 合成,并记入集合级 warnings。 */
+  experimentId: string;
+  /** 所属物理 run。 */
+  run: RunHandle;
+  /** = run.summary.startedAt;快照新旧按它比较。 */
+  startedAt: string;
+  /** 本快照自己的 agent —— 不是 run 顶层那个「第一个配置」。 */
+  agent: string;
+  model?: string;
+  attempts: AttemptHandle[];
+  /** 覆盖的 eval 集合(去重,保持首次出现顺序),供选择器做残缺检测。 */
+  evalIds: string[];
+  /** experimentId 是合成键(结果里没有 experimentId)时为 true;选择器据此调整提示文案。 */
+  synthetic?: boolean;
+}
+
+/** 目录扫描里读不了、但必须让调用方知道的 run;无关 JSON 不记(静默忽略)。 */
+export interface SkippedResultsRun {
+  /** run 目录的绝对路径。 */
+  dir: string;
+  /** summary.json 的绝对路径。 */
+  path: string;
+  reason: "incompatible-version" | "malformed";
+  /** incompatible-version:那份报告声明的 schemaVersion。 */
+  schemaVersion?: number;
+  /** incompatible-version:写报告的 niceeval 版本,拼 `npx niceeval@<version> view` 提示用。 */
+  producerVersion?: string;
+  /** malformed:一句英文诊断。 */
+  detail?: string;
+}
+
+/** openResults 的返回:runs 忠实磁盘;snapshots 只切片、不合并、不去重;skipped 不静默丢。 */
+export interface ResultsCollection {
+  /** 按 startedAt 降序(最新在前),同刻按目录名降序。 */
+  runs: RunHandle[];
+  /** 按 runs 顺序逐 run 切片;每个 experiment 在每个 run 里至多一个快照。 */
+  snapshots: SnapshotHandle[];
+  skipped: SkippedResultsRun[];
+  /** 集合级警告(如 experimentId 缺失时的合成键);渲染与否在消费方,缺口永远被算出来。 */
+  warnings: string[];
+}

package/src/runner/attempt.ts

@@ -226,6 +226,7 @@ async function runAttemptBody(
   const attemptCtx: AgentContext = {
     signal,
     model: run.model,
+    reasoningEffort: run.reasoningEffort,
     flags: run.flags,
     sandbox,
     session: createAgentSession(),
@@ -267,6 +268,7 @@ async function runAttemptBody(
       agent: run.agent,
       sandbox,
       model: run.model,
+      reasoningEffort: run.reasoningEffort,
       flags: run.flags,
       signal,
       log,
@@ -366,7 +368,7 @@ async function runAttemptBody(
     const durationMs = Date.now() - t0;
     const o11y = buildO11ySummary(events, usage, durationMs);
     // 实测成本(网关带回)优先,缺则按 model + 用量查价格表估算(见 o11y/cost.ts)。
-    const cost = usage.costUSD ?? estimateCost(run.model, usage);
+    const cost = usage.costUSD ?? estimateCost(run.model, usage, config.pricing);
     if (cost !== undefined) o11y.estimatedCostUSD = cost;
 
     // 收 test 引用到的 eval 源码(按 send / 断言的 loc 去重),供 view 渲染代码视图。

package/src/runner/report.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from "vitest";
+import { filterSummary, scopeReporter } from "./report.ts";
+import type { Agent, EvalResult, Reporter, RunShape, RunSummary } from "../types.ts";
+
+function result(id: string, overrides: Partial<EvalResult> = {}): EvalResult {
+  return {
+    id,
+    agent: "codex",
+    outcome: "passed",
+    attempt: 0,
+    durationMs: 1000,
+    assertions: [],
+    ...overrides,
+  };
+}
+
+function summary(results: EvalResult[]): RunSummary {
+  return {
+    agent: "codex",
+    startedAt: "2026-07-07T00:00:00.000Z",
+    completedAt: "2026-07-07T00:01:00.000Z",
+    passed: results.filter((r) => r.outcome === "passed").length,
+    failed: results.filter((r) => r.outcome === "failed").length,
+    skipped: 0,
+    errored: 0,
+    durationMs: 60_000,
+    results,
+    outputDir: ".niceeval/x",
+  };
+}
+
+describe("filterSummary", () => {
+  it("按 eval id 过滤结果并重新计数,保留原 completedAt / outputDir", () => {
+    const s = summary([
+      result("a/1", { usage: { inputTokens: 10, outputTokens: 5 }, estimatedCostUSD: 0.1 }),
+      result("a/1", { outcome: "failed", attempt: 1 }),
+      result("b/1", { usage: { inputTokens: 100, outputTokens: 50 }, estimatedCostUSD: 1 }),
+    ]);
+    const sub = filterSummary(s, new Set(["a/1"]));
+    expect(sub.results.map((r) => r.id)).toEqual(["a/1", "a/1"]);
+    expect(sub.passed).toBe(1);
+    expect(sub.failed).toBe(1);
+    expect(sub.usage).toEqual({ inputTokens: 10, outputTokens: 5 });
+    expect(sub.estimatedCostUSD).toBe(0.1);
+    expect(sub.completedAt).toBe("2026-07-07T00:01:00.000Z");
+    expect(sub.outputDir).toBe(".niceeval/x");
+  });
+});
+
+describe("scopeReporter", () => {
+  const agent = { name: "codex" } as Agent;
+  const scopedShape: RunShape = { evals: 1, configs: 1, totalRuns: 2, maxConcurrency: 4 };
+
+  function recordingReporter() {
+    const calls: { method: string; args: unknown[] }[] = [];
+    const reporter: Reporter = {
+      onRunStart: (...args) => void calls.push({ method: "onRunStart", args }),
+      onEvalComplete: (...args) => void calls.push({ method: "onEvalComplete", args }),
+      onRunComplete: (...args) => void calls.push({ method: "onRunComplete", args }),
+      onEvent: (...args) => void calls.push({ method: "onEvent", args }),
+    };
+    return { calls, reporter };
+  }
+
+  it("onRunStart 收到过滤后的 eval 列表和作用域 shape", async () => {
+    const { calls, reporter } = recordingReporter();
+    const scoped = scopeReporter(reporter, new Set(["a/1"]), scopedShape);
+    await scoped.onRunStart?.([{ id: "a/1" }, { id: "b/1" }], agent, { evals: 2, configs: 1, totalRuns: 4, maxConcurrency: 4 });
+    expect(calls[0]?.args).toEqual([[{ id: "a/1" }], agent, scopedShape]);
+  });
+
+  it("onEvalComplete 只转发被观测 eval 的结果", async () => {
+    const { calls, reporter } = recordingReporter();
+    const scoped = scopeReporter(reporter, new Set(["a/1"]));
+    await scoped.onEvalComplete?.(result("b/1"));
+    await scoped.onEvalComplete?.(result("a/1"));
+    expect(calls).toHaveLength(1);
+    expect((calls[0]?.args[0] as EvalResult).id).toBe("a/1");
+  });
+
+  it("onRunComplete 收到重新计数的子集汇总", async () => {
+    const { calls, reporter } = recordingReporter();
+    const scoped = scopeReporter(reporter, new Set(["a/1"]));
+    await scoped.onRunComplete?.(summary([result("a/1"), result("b/1", { outcome: "failed" })]));
+    const got = calls[0]?.args[0] as RunSummary;
+    expect(got.results.map((r) => r.id)).toEqual(["a/1"]);
+    expect(got.passed).toBe(1);
+    expect(got.failed).toBe(0);
+  });
+
+  it("onEvent 过滤 eval 级事件,重写汇总类事件", async () => {
+    const { calls, reporter } = recordingReporter();
+    const scoped = scopeReporter(reporter, new Set(["a/1"]), scopedShape);
+    await scoped.onEvent?.({ type: "eval:start", eval: { id: "b/1" }, agent, attempt: 0 });
+    await scoped.onEvent?.({ type: "eval:start", eval: { id: "a/1" }, agent, attempt: 0 });
+    await scoped.onEvent?.({ type: "run:summary", summary: summary([result("a/1"), result("b/1")]) });
+    expect(calls).toHaveLength(2);
+    expect(calls[0]?.args[0]).toMatchObject({ type: "eval:start", eval: { id: "a/1" } });
+    const summaryEvent = calls[1]?.args[0] as { type: string; summary: RunSummary };
+    expect(summaryEvent.type).toBe("run:summary");
+    expect(summaryEvent.summary.results.map((r) => r.id)).toEqual(["a/1"]);
+  });
+
+  it("底层 reporter 未实现的回调不会被包装出来", () => {
+    const scoped = scopeReporter({}, new Set(["a/1"]));
+    expect(scoped.onRunStart).toBeUndefined();
+    expect(scoped.onEvalComplete).toBeUndefined();
+    expect(scoped.onRunComplete).toBeUndefined();
+    expect(scoped.onEvent).toBeUndefined();
+  });
+});

package/src/runner/report.ts

@@ -1,7 +1,7 @@
 // reporter 编排与运行级汇总。reporter 是「结果消费方」:单个 reporter 抛错只记
 // diagnostic,不能让整次调度崩。
 
-import type { EvalResult, LocalizedText, Reporter, ReporterEvent, RunSummary } from "../types.ts";
+import type { EvalResult, LocalizedText, Reporter, ReporterEvent, RunShape, RunSummary } from "../types.ts";
 import { t } from "../i18n/index.ts";
 import { formatThrown } from "../util.ts";
 
@@ -18,6 +18,57 @@ export async function emitReporterEvent(reporters: readonly Reporter[], event: R
   await Promise.all(reporters.map((r) => runReporter(`event:${event.type}`, () => r.onEvent?.(event))));
 }
 
+/** 按 eval id 过滤 RunSummary 并重新计数 —— eval 级 reporter 只看它观测的那部分。 */
+export function filterSummary(summary: RunSummary, ids: ReadonlySet<string>): RunSummary {
+  const results = summary.results.filter((r) => ids.has(r.id));
+  const sub = summarize(results, summary.agent, summary.startedAt, summary.durationMs, summary.name);
+  // completedAt 用原值(summarize 会重新取 now);format / producer / outputDir 等元数据原样保留。
+  return { ...summary, ...sub, completedAt: summary.completedAt };
+}
+
+/**
+ * eval 级 reporter 的作用域包装:实例只观测「引用它的那些 eval」。
+ * 结果类回调按 eval id 过滤,汇总类回调收到重新计数的子集汇总;
+ * shape 由调用方按作用域预先算好(包装器自己看不到 attempts)。
+ */
+export function scopeReporter(r: Reporter, ids: ReadonlySet<string>, shape?: RunShape): Reporter {
+  const scoped: Reporter = {};
+  if (r.onRunStart) {
+    scoped.onRunStart = (evals, agent, fullShape) =>
+      r.onRunStart!(evals.filter((e) => ids.has(e.id)), agent, shape ?? fullShape);
+  }
+  if (r.onEvalComplete) {
+    scoped.onEvalComplete = (result) => (ids.has(result.id) ? r.onEvalComplete!(result) : undefined);
+  }
+  if (r.onRunComplete) {
+    scoped.onRunComplete = (summary) => r.onRunComplete!(filterSummary(summary, ids));
+  }
+  if (r.onEvent) {
+    scoped.onEvent = (event) => {
+      switch (event.type) {
+        case "run:start":
+          return r.onEvent!({
+            ...event,
+            evals: event.evals.filter((e) => ids.has(e.id)),
+            shape: shape ?? event.shape,
+          });
+        case "eval:start":
+          return ids.has(event.eval.id) ? r.onEvent!(event) : undefined;
+        case "eval:complete":
+          return ids.has(event.result.id) ? r.onEvent!(event) : undefined;
+        case "run:earlyExit":
+          return ids.has(event.evalId) ? r.onEvent!(event) : undefined;
+        case "run:summary":
+        case "run:saved":
+          return r.onEvent!({ ...event, summary: filterSummary(event.summary, ids) });
+        default:
+          return r.onEvent!(event);
+      }
+    };
+  }
+  return scoped;
+}
+
 /** 全局汇总:outcome 计数 + token / cost 折叠。按 attempt 计(eval 级折叠见 shared/outcome.ts)。 */
 export function summarize(
   results: EvalResult[],

package/src/runner/reporters/artifacts.ts

@@ -40,7 +40,7 @@ function slimResult(r: EvalResult): EvalResult {
   // 携带结果(跨实验复用上次 pass,见 run.ts 的 carriedResults):本轮没有任何新数据,
   // rest 上已经带着 artifactBase 指向旧 run 的产物目录,hasSources/hasEvents/hasTrace
   // 也是从旧 summary 带过来的真值——不能因为"这轮没数据"就重新推导成 false / 编出一个
-  // 这轮压根没写过文件的新 artifactsDir(会让 artifactBase 在下次 withArtifactBases 时被覆盖)。
+  // 这轮压根没写过文件的新 artifactsDir(会让 artifactBase 在下次 withViewRefs 时被覆盖)。
   if (rest.artifactBase) return rest;
   return {
     ...rest,

package/src/runner/reporters/braintrust.test.ts

@@ -0,0 +1,106 @@
+import { describe, expect, it } from "vitest";
+import { toBraintrustEvent } from "./braintrust.ts";
+import type { EvalResult } from "../../types.ts";
+
+function baseResult(overrides: Partial<EvalResult> = {}): EvalResult {
+  return {
+    id: "algebra/quadratic",
+    agent: "codex",
+    outcome: "passed",
+    attempt: 0,
+    durationMs: 42_000,
+    assertions: [],
+    ...overrides,
+  };
+}
+
+describe("toBraintrustEvent", () => {
+  it("soft 断言按名字记分,gate 断言带 gate: 前缀", () => {
+    const event = toBraintrustEvent(
+      baseResult({
+        assertions: [
+          { name: "compiles", severity: "gate", score: 1, passed: true },
+          { name: "closedQA", severity: "soft", score: 0.7, passed: true },
+        ],
+      }),
+    );
+    expect(event.scores).toEqual({ "gate:compiles": 1, closedQA: 0.7 });
+  });
+
+  it("重名断言追加 #n 消歧,分数被夹到 0..1", () => {
+    const event = toBraintrustEvent(
+      baseResult({
+        assertions: [
+          { name: "check", severity: "soft", score: 0.5, passed: true },
+          { name: "check", severity: "soft", score: 2, passed: true },
+          { name: "check", severity: "soft", score: -1, passed: false },
+        ],
+      }),
+    );
+    expect(event.scores).toEqual({ check: 0.5, "check#2": 1, "check#3": 0 });
+  });
+
+  it("metrics:start/end 由 startedAt+durationMs 推出,token/成本缺就不写", () => {
+    const event = toBraintrustEvent(
+      baseResult({
+        startedAt: "2026-07-07T00:00:00.000Z",
+        durationMs: 2_000,
+        usage: { inputTokens: 100, outputTokens: 50, cacheReadTokens: 10 },
+        estimatedCostUSD: 0.31,
+      }),
+    );
+    const start = Date.parse("2026-07-07T00:00:00.000Z") / 1000;
+    expect(event.metrics).toEqual({
+      start,
+      end: start + 2,
+      prompt_tokens: 100,
+      completion_tokens: 50,
+      tokens: 150,
+      cache_read_tokens: 10,
+      estimated_cost_usd: 0.31,
+    });
+
+    const bare = toBraintrustEvent(baseResult());
+    expect(bare.metrics).toEqual({});
+  });
+
+  it("output 取事件流里最后一条 assistant message", () => {
+    const event = toBraintrustEvent(
+      baseResult({
+        events: [
+          { type: "message", role: "user", text: "question" },
+          { type: "message", role: "assistant", text: "draft" },
+          { type: "message", role: "assistant", text: "final answer" },
+        ],
+      }),
+    );
+    expect(event.output).toBe("final answer");
+    expect(toBraintrustEvent(baseResult()).output).toBeUndefined();
+  });
+
+  it("metadata 带身份维度与失败断言明细;id 在 (experiment, eval, agent, model, attempt) 上唯一", () => {
+    const event = toBraintrustEvent(
+      baseResult({
+        experimentId: "compare/codex",
+        model: "gpt-5.2",
+        attempt: 1,
+        outcome: "failed",
+        experiment: { id: "compare/codex", flags: { tape: true } },
+        assertions: [{ name: "compiles", severity: "gate", score: 0, passed: false, detail: "tsc failed" }],
+      }),
+    );
+    expect(event.metadata).toEqual({
+      eval: "algebra/quadratic",
+      agent: "codex",
+      attempt: 1,
+      outcome: "failed",
+      model: "gpt-5.2",
+      experiment: "compare/codex",
+      flags: { tape: true },
+      failedAssertions: [{ name: "compiles", detail: "tsc failed" }],
+    });
+    expect(event.id).toBe("compare/codex|algebra/quadratic|codex|gpt-5.2|a1");
+    const other = toBraintrustEvent(baseResult({ experimentId: "compare/codex", model: "gpt-5.2", attempt: 2 }));
+    expect(other.id).not.toBe(event.id);
+  });
+});

package/src/runner/reporters/braintrust.ts

@@ -0,0 +1,197 @@
+// Braintrust 报告器:把一次 niceeval 运行作为一个 Braintrust experiment 上报,
+// 每个 attempt 一行(scores = 断言,metrics = 时长/token/成本),跨提交比较与团队共享。
+// `braintrust` 是可选依赖:动态 import,装了才用得上,没装在 onRunStart 报错
+// (reporter 错误按框架约定只记 diagnostic,不会让运行崩)。
+
+import type { EvalResult, Reporter } from "../../types.ts";
+
+export interface BraintrustConfig {
+  /** Braintrust 项目名;省略时用 "niceeval"。 */
+  project?: string;
+  /** Braintrust 项目 id;与 project 给一个即可。 */
+  projectId?: string;
+  /** 实验名;省略时由 Braintrust 自动命名。 */
+  experiment?: string;
+  /** 作为对比基线(diff base)的既有实验名。 */
+  baseExperiment?: string;
+  /** 作为对比基线(diff base)的既有实验 id。 */
+  baseExperimentId?: string;
+  /** true = 更新同名既有实验,而不是新建一个。 */
+  update?: boolean;
+  /** 实验级附加 metadata;与 niceeval 自动写入的字段合并,同名以这里为准。 */
+  metadata?: Record<string, unknown>;
+  /** API key;省略时 SDK 读 BRAINTRUST_API_KEY 环境变量。 */
+  apiKey?: string;
+}
+
+/**
+ * Braintrust SDK 的最小类型面。本地声明是为了不把 `braintrust` 变成编译期依赖;
+ * 动态 import 后按这个形状断言。
+ */
+interface BraintrustSdk {
+  // SDK 里 init 同步返回(登录与建实验都是惰性的);统一 await,同步/异步都兼容。
+  init(options: {
+    project?: string;
+    projectId?: string;
+    experiment?: string;
+    baseExperiment?: string;
+    baseExperimentId?: string;
+    update?: boolean;
+    apiKey?: string;
+    metadata?: Record<string, unknown>;
+    setCurrent?: boolean;
+  }): BraintrustExperiment | Promise<BraintrustExperiment>;
+  flush(): Promise<void>;
+}
+
+interface BraintrustExperiment {
+  log(event: BraintrustLogEvent): void;
+  summarize(): Promise<{ experimentUrl?: string }>;
+}
+
+export interface BraintrustLogEvent {
+  id?: string;
+  input?: unknown;
+  output?: unknown;
+  error?: unknown;
+  scores?: Record<string, number>;
+  metadata?: Record<string, unknown>;
+  metrics?: Record<string, number>;
+}
+
+/**
+ * 创建 Braintrust 报告器。挂在 `defineConfig({ reporters })` 上观测整次运行,
+ * 或挂在单个 eval 的 `reporters` 上只观测它(同一实例被多个 eval 引用时共享一个实验)。
+ */
+export function Braintrust(config: BraintrustConfig = {}): Reporter {
+  let sdk: BraintrustSdk | undefined;
+  let experiment: BraintrustExperiment | undefined;
+
+  return {
+    async onRunStart(evals, agent) {
+      sdk = await loadBraintrustSdk();
+      experiment = await sdk.init({
+        project: config.projectId ? undefined : (config.project ?? "niceeval"),
+        projectId: config.projectId,
+        experiment: config.experiment,
+        baseExperiment: config.baseExperiment,
+        baseExperimentId: config.baseExperimentId,
+        update: config.update,
+        apiKey: config.apiKey,
+        metadata: {
+          agent: agent?.name,
+          evals: evals.map((e) => e.id),
+          ...config.metadata,
+        },
+        // 不设为全局 current experiment,避免污染用户代码里的 braintrust 全局态。
+        setCurrent: false,
+      });
+    },
+
+    onEvalComplete(result) {
+      experiment?.log(toBraintrustEvent(result));
+    },
+
+    async onRunComplete() {
+      if (!experiment) return;
+      try {
+        await sdk?.flush();
+        const summary = await experiment.summarize();
+        if (summary.experimentUrl) {
+          process.stderr.write(`Braintrust experiment: ${summary.experimentUrl}\n`);
+        }
+      } finally {
+        experiment = undefined;
+        sdk = undefined;
+      }
+    },
+  };
+}
+
+/**
+ * EvalResult → Braintrust 一行。导出仅为单测;映射口径:
+ * - scores:soft 断言按名字记分,gate 断言记在 `gate:` 前缀下 —— 实验 diff 里
+ *   gate 回归和 soft 分数回归用同一套机制看。重名断言追加 `#n` 消歧,不静默覆盖。
+ * - metrics:start/end(Braintrust 由此算时长)+ token 用量 + 估算成本;缺就不写,不编 0。
+ * - metadata:身份维度(agent / model / experiment / attempt / flags)+ 失败断言明细。
+ */
+export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
+  const scores: Record<string, number> = {};
+  for (const a of result.assertions) {
+    const base = a.severity === "gate" ? `gate:${a.name}` : a.name;
+    let key = base;
+    for (let n = 2; key in scores; n++) key = `${base}#${n}`;
+    // Braintrust 要求 0..1
+    scores[key] = Math.min(1, Math.max(0, a.score));
+  }
+
+  const metrics: Record<string, number> = {};
+  if (result.startedAt) {
+    const start = Date.parse(result.startedAt) / 1000;
+    if (Number.isFinite(start)) {
+      metrics.start = start;
+      metrics.end = start + result.durationMs / 1000;
+    }
+  }
+  if (result.usage) {
+    metrics.prompt_tokens = result.usage.inputTokens;
+    metrics.completion_tokens = result.usage.outputTokens;
+    metrics.tokens = result.usage.inputTokens + result.usage.outputTokens;
+    if (result.usage.cacheReadTokens !== undefined) metrics.cache_read_tokens = result.usage.cacheReadTokens;
+    if (result.usage.cacheWriteTokens !== undefined) metrics.cache_write_tokens = result.usage.cacheWriteTokens;
+  }
+  if (result.estimatedCostUSD !== undefined) metrics.estimated_cost_usd = result.estimatedCostUSD;
+
+  const metadata: Record<string, unknown> = {
+    eval: result.id,
+    agent: result.agent,
+    attempt: result.attempt,
+    outcome: result.outcome,
+  };
+  if (result.model !== undefined) metadata.model = result.model;
+  if (result.experimentId !== undefined) metadata.experiment = result.experimentId;
+  if (result.experiment?.flags && Object.keys(result.experiment.flags).length > 0) {
+    metadata.flags = result.experiment.flags;
+  }
+  if (result.skipReason !== undefined) metadata.skipReason = result.skipReason;
+  const failed = result.assertions
+    .filter((a) => !a.passed)
+    .map((a) => ({ name: a.name, detail: a.detail }));
+  if (failed.length > 0) metadata.failedAssertions = failed;
+
+  // 一次运行内 (experiment, eval, agent, model, attempt) 唯一;Braintrust 按 id 合并重复行。
+  const id = [result.experimentId ?? "", result.id, result.agent, result.model ?? "", `a${result.attempt}`].join("|");
+
+  return {
+    id,
+    input: result.description ?? result.id,
+    output: lastAssistantText(result.events),
+    error: result.error,
+    scores,
+    metadata,
+    metrics,
+  };
+}
+
+/** agent 的最终回复文本(事件流里最后一条 assistant message);没有就不填。 */
+function lastAssistantText(events: EvalResult["events"]): string | undefined {
+  if (!events) return undefined;
+  for (let i = events.length - 1; i >= 0; i--) {
+    const e = events[i];
+    if (e.type === "message" && e.role === "assistant") return e.text;
+  }
+  return undefined;
+}
+
+/** 动态 import 说明:specifier 放变量里,`braintrust` 不进编译期依赖,没装也能 typecheck。 */
+const BRAINTRUST_PACKAGE = "braintrust";
+
+async function loadBraintrustSdk(): Promise<BraintrustSdk> {
+  try {
+    return (await import(BRAINTRUST_PACKAGE)) as unknown as BraintrustSdk;
+  } catch {
+    throw new Error(
+      "The 'braintrust' package is required for the Braintrust reporter but was not found. Install it with: npm install braintrust (and set BRAINTRUST_API_KEY).",
+    );
+  }
+}

package/src/runner/reporters/index.ts

@@ -1,5 +1,7 @@
-// 内置报告器:Console(默认)/ Artifacts / Json / JUnit。第三方实验跟踪平台也走这条 Reporter 通道。
+// 内置报告器:Console(默认)/ Artifacts / Json / JUnit / Braintrust。
+// 其它第三方实验跟踪平台也走同一条 Reporter 通道。
 
 export { Console } from "./console.ts";
 export { Artifacts } from "./artifacts.ts";
 export { Json, JUnit } from "./json.ts";
+export { Braintrust, type BraintrustConfig } from "./braintrust.ts";