niceeval 0.6.2 → 0.7.0

package/dist/report/built-ins/experiment-comparison.d.ts

@@ -1,39 +0,0 @@
-import { type ReportDefinition } from "../report.ts";
-import { type ReportComponent } from "../tree.ts";
-import { type ReportLocale } from "../locale.ts";
-import type { ExperimentListItem, GroupSummaryData, ScatterData } from "../types.ts";
-import { type SnapshotsInput } from "../aggregate.ts";
-/**
- * 一个可比组的数据。三个子块都只消费本组快照，不能含其它父目录的引用。
- */
-export interface ExperimentComparisonGroupData {
-    /** experiment id 的完整父路径；根目录 experiment 使用自己的完整 id。 */
-    key: string;
-    summary: GroupSummaryData;
-    /** 成本 × 端到端成功率散点(`MetricScatter.data` 的口径:points=experiment、series=agent)。 */
-    scatter: ScatterData;
-    /** 固定列 experiment 比较表(`ExperimentList.data` 的口径,已按端到端成功率排序)。 */
-    experiments: ExperimentListItem[];
-}
-/** `ExperimentComparison.data(selection)` 的穷尽产物：按 key 排序的独立可比组。 */
-export interface ExperimentComparisonData {
-    groups: ExperimentComparisonGroupData[];
-}
-/** 完整父路径是组键；没有父路径的 experiment 不能互相比，自己形成单例组。 */
-export declare function experimentComparisonGroupKey(experimentId: string): string;
-/** 组合件的数据计算：先分区，再让三个官方子块在各组内并行计算。 */
-declare function experimentComparisonData(input: SnapshotsInput): Promise<ExperimentComparisonData>;
-export interface ExperimentComparisonProps {
-    data: ExperimentComparisonData;
-    /** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
-    locale?: ReportLocale;
-    className?: string;
-}
-/**
- * 内置默认报告兼官方组合件:作为组件时收 `data`(`.data()` 的产物),两个渲染面直接摆
- * 数据形态的子组件;作为默认报告时经 build 面走与用户 `--report` 完全相同的管线。
- */
-export declare const ExperimentComparison: ReportComponent<ExperimentComparisonProps> & ReportDefinition & {
-    data: typeof experimentComparisonData;
-};
-export {};

package/dist/report/built-ins/experiment-comparison.js

@@ -1,119 +0,0 @@
-import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-// ExperimentComparison:裸 niceeval show / view 首页渲染的内置默认报告,同时是可整体引用的
-// 官方组合件(docs/feature/reports/library.md「ExperimentComparison」)。
-//
-// 名字描述整份报告回答的问题,不绑定其中某一张图。它是组合件而非新的数据源——每组的
-// GroupSummary、成本 × 端到端成功率散点、固定列 ExperimentList 都消费各自公开 .data() 的结果。
-// 两种用法共享同一份口径:
-//   · 默认报告:宿主把它当普通 ReportDefinition(build 面),没有 renderer 私有通道;
-//   · 报告组件:`<ExperimentComparison data={await ExperimentComparison.data(selection)} />`,
-//     在自定义报告或用户自己的 React 页面里整体引用。
-import { Col, Section, Table } from "../primitives.js";
-import { ExperimentList, GroupSummary, MetricScatter } from "../components.js";
-import { costUSD, endToEndPassRate } from "../metrics.js";
-import { defineReport } from "../report.js";
-import { defineComponent } from "../tree.js";
-import { localeText } from "../locale.js";
-import { resolveInput } from "../aggregate.js";
-import { ExperimentComparisonView } from "../react/ExperimentComparison.js";
-import { wrapDisplay } from "../text/layout.js";
-import { formatUSD } from "../format.js";
-// 每组散点的唯一口径：默认 definition 与公开 `.data()` 共用，不各写一份。
-const SCATTER_OPTIONS = {
-    points: "experiment",
-    series: "agent",
-    x: costUSD,
-    y: endToEndPassRate,
-};
-/** 完整父路径是组键；没有父路径的 experiment 不能互相比，自己形成单例组。 */
-export function experimentComparisonGroupKey(experimentId) {
-    const slash = experimentId.lastIndexOf("/");
-    return slash === -1 ? experimentId : experimentId.slice(0, slash);
-}
-/** 组合件的数据计算：先分区，再让三个官方子块在各组内并行计算。 */
-async function experimentComparisonData(input) {
-    const { snapshots } = resolveInput(input);
-    const snapshotsByGroup = new Map();
-    for (const snapshot of snapshots) {
-        const key = experimentComparisonGroupKey(snapshot.experimentId);
-        const group = snapshotsByGroup.get(key);
-        if (group)
-            group.push(snapshot);
-        else
-            snapshotsByGroup.set(key, [snapshot]);
-    }
-    const groups = await Promise.all([...snapshotsByGroup.entries()]
-        .sort(([a], [b]) => a.localeCompare(b))
-        .map(async ([key, groupSnapshots]) => {
-        const [summary, scatter, experiments] = await Promise.all([
-            GroupSummary.data(groupSnapshots),
-            MetricScatter.data(groupSnapshots, SCATTER_OPTIONS),
-            ExperimentList.data(groupSnapshots),
-        ]);
-        return { key, summary, scatter, experiments };
-    }));
-    return { groups };
-}
-/** 单组详情：散点与实验列表全部来自同一个预分区结果。 */
-function groupComposition(group, locale, className) {
-    return (_jsxs(Col, { className: className, children: [_jsx(MetricScatter, { data: group.scatter, locale: locale }), _jsx(ExperimentList, { items: group.experiments, filter: true, locale: locale, relativeTo: group.key })] }));
-}
-function shellQuote(value) {
-    return /^[A-Za-z0-9._/-]+$/.test(value) ? value : `'${value.replaceAll("'", `'"'"'`)}'`;
-}
-function verdictsText(group, locale) {
-    const parts = [];
-    for (const verdict of ["passed", "failed", "errored", "skipped"]) {
-        const count = group.summary.verdicts[verdict];
-        if (count > 0)
-            parts.push(`${count} ${localeText(locale, `verdict.${verdict}`)}`);
-    }
-    return parts.length > 0 ? parts.join(" / ") : "—";
-}
-function textFace(data, className, ctx) {
-    if (data.groups.length === 0)
-        return localeText(ctx.locale, "experimentComparison.empty");
-    if (data.groups.length === 1) {
-        const group = data.groups[0];
-        return ctx.render(_jsx(Section, { title: group.key, className: className, children: groupComposition(group) }));
-    }
-    const table = ctx.render(_jsx(Table, { columns: [
-            { key: "group", header: localeText(ctx.locale, "experimentComparison.group") },
-            { key: "experiments", header: localeText(ctx.locale, "groupSummary.experiments"), align: "right" },
-            { key: "evals", header: localeText(ctx.locale, "overview.evals"), align: "right" },
-            { key: "passRate", header: localeText(ctx.locale, "overview.passRate"), align: "right" },
-            { key: "results", header: localeText(ctx.locale, "experimentComparison.results") },
-            { key: "cost", header: localeText(ctx.locale, "overview.totalCost"), align: "right" },
-            { key: "lastRun", header: localeText(ctx.locale, "experimentComparison.lastRun") },
-        ], rows: data.groups.map((group) => ({
-            key: group.key,
-            cells: {
-                group: group.key,
-                experiments: String(group.summary.experiments),
-                evals: String(group.summary.evals),
-                passRate: group.summary.passRate.display,
-                results: verdictsText(group, ctx.locale),
-                cost: group.summary.totalCostUSD === null ? null : formatUSD(group.summary.totalCostUSD),
-                lastRun: group.summary.lastRunAt ?? null,
-            },
-        })) }));
-    const commands = data.groups
-        .map((group) => {
-        const command = `niceeval show --experiment ${shellQuote(group.key)}`;
-        return wrapDisplay(localeText(ctx.locale, "experimentComparison.command", { command }), ctx.width).join("\n");
-    })
-        .join("\n");
-    return `${localeText(ctx.locale, "experimentComparison.groups")}\n\n${table}\n\n${commands}`;
-}
-/**
- * 内置默认报告兼官方组合件:作为组件时收 `data`(`.data()` 的产物),两个渲染面直接摆
- * 数据形态的子组件;作为默认报告时经 build 面走与用户 `--report` 完全相同的管线。
- */
-export const ExperimentComparison = Object.assign(defineComponent({
-    web: (props, ctx) => (_jsx(ExperimentComparisonView, { data: props.data, locale: props.locale ?? ctx.locale, className: props.className })),
-    text: ({ data, className }, ctx) => textFace(data, className, ctx),
-}), defineReport(async ({ selection }) => {
-    const data = await experimentComparisonData(selection);
-    return _jsx(ExperimentComparison, { data: data });
-}), { data: experimentComparisonData });
-ExperimentComparison.displayName = "ExperimentComparison";

package/dist/report/built-ins/index.d.ts

@@ -1,2 +0,0 @@
-export { ExperimentComparison } from "./experiment-comparison.tsx";
-export type { ExperimentComparisonData, ExperimentComparisonGroupData, ExperimentComparisonProps, } from "./experiment-comparison.tsx";

package/dist/report/built-ins/index.js

@@ -1,2 +0,0 @@
-// 内置报告目录:一个文件一个按展示内容命名的组合件(组件兼默认报告),显式导出值,不建字符串 registry。
-export { ExperimentComparison } from "./experiment-comparison.js";

package/dist/report/react/GroupSummary.d.ts

@@ -1,8 +0,0 @@
-import type { ReactElement } from "react";
-import type { GroupSummaryData } from "../types.ts";
-import { type ReportLocale } from "../locale.ts";
-export declare function GroupSummary({ data, className, locale, }: {
-    data: GroupSummaryData;
-    className?: string;
-    locale?: ReportLocale;
-}): ReactElement;

package/dist/report/react/GroupSummary.js

@@ -1,8 +0,0 @@
-import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-import { DEFAULT_REPORT_LOCALE, localeText } from "../locale.js";
-import { MetricCellView } from "./cell.js";
-import { cx, formatUSD } from "./format.js";
-export function GroupSummary({ data, className, locale = DEFAULT_REPORT_LOCALE, }) {
-    const missing = _jsx("span", { className: "nre-missing", children: localeText(locale, "cell.missing") });
-    return (_jsxs("div", { className: cx("nre", "nre-group-summary", className), children: [_jsxs("dl", { className: "nre-group-kpis", children: [_jsxs("div", { className: "nre-group-kpi nre-group-kpi-rate", children: [_jsx("dt", { children: localeText(locale, "overview.passRate") }), _jsx("dd", { children: _jsx(MetricCellView, { cell: data.passRate, locale: locale }) })] }), _jsxs("div", { className: "nre-group-kpi", children: [_jsx("dt", { children: localeText(locale, "groupSummary.experiments") }), _jsx("dd", { children: data.experiments })] }), _jsxs("div", { className: "nre-group-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.evals") }), _jsx("dd", { children: data.evals })] }), _jsxs("div", { className: "nre-group-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.attempts") }), _jsx("dd", { children: data.attempts })] }), _jsxs("div", { className: "nre-group-kpi", children: [_jsx("dt", { children: localeText(locale, "verdict.failed") }), _jsx("dd", { className: "nre-verdict-failed", children: data.verdicts.failed })] }), data.verdicts.errored > 0 && (_jsxs("div", { className: "nre-group-kpi", children: [_jsx("dt", { children: localeText(locale, "verdict.errored") }), _jsx("dd", { className: "nre-verdict-errored", children: data.verdicts.errored })] })), _jsxs("div", { className: "nre-group-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.totalCost") }), _jsx("dd", { children: data.totalCostUSD === null ? missing : formatUSD(data.totalCostUSD) })] })] }), data.lastRunAt && (_jsx("p", { className: "nre-group-summary-time", children: localeText(locale, "latestRun", { run: data.lastRunAt }) }))] }));
-}

package/dist/report/react/RunOverview.d.ts

@@ -1,8 +0,0 @@
-import type { ReactElement } from "react";
-import type { OverviewData } from "../types.ts";
-import { type ReportLocale } from "../locale.ts";
-export declare function RunOverview({ data, className, locale, }: {
-    data: OverviewData;
-    className?: string;
-    locale?: ReportLocale;
-}): ReactElement;

package/dist/report/react/RunOverview.js

@@ -1,12 +0,0 @@
-import { Fragment as _Fragment, jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-import { DEFAULT_REPORT_LOCALE, localeText } from "../locale.js";
-import { cx, formatDurationMs, formatUSD } from "./format.js";
-export function RunOverview({ data, className, locale = DEFAULT_REPORT_LOCALE, }) {
-    const { totals } = data;
-    const missing = _jsx("span", { className: "nre-missing", children: localeText(locale, "cell.missing") });
-    // 通过率只渲染 computeCell 算好的 MetricCell,不现场重算——覆盖率角标复用
-    // MetricCellView 的缺数据/coverage 语义(缺数据 = 显式文案,绝不画 0%)
-    const { passRate } = totals;
-    const passRateNode = passRate.value === null ? (missing) : (_jsxs(_Fragment, { children: [passRate.display, passRate.samples < passRate.total && (_jsxs("sup", { className: "nre-coverage", title: localeText(locale, "cell.coverageTitle", { samples: passRate.samples, total: passRate.total }), children: [passRate.samples, "/", passRate.total] }))] }));
-    return (_jsxs("header", { className: cx("nre", "nre-overview", className), children: [_jsxs("dl", { className: "nre-kpis", children: [_jsxs("div", { className: "nre-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.snapshots") }), _jsx("dd", { children: data.snapshots.length })] }), _jsxs("div", { className: "nre-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.evals") }), _jsx("dd", { children: totals.evals })] }), _jsxs("div", { className: "nre-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.attempts") }), _jsx("dd", { children: totals.attempts })] }), _jsxs("div", { className: "nre-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.passRate") }), _jsx("dd", { children: passRateNode })] }), _jsxs("div", { className: "nre-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.totalCost") }), _jsx("dd", { children: totals.costUSD === null ? missing : formatUSD(totals.costUSD) })] }), _jsxs("div", { className: "nre-kpi", children: [_jsx("dt", { children: localeText(locale, "overview.totalDuration") }), _jsx("dd", { children: formatDurationMs(totals.durationMs) })] })] }), _jsxs("p", { className: "nre-verdicts", children: [_jsxs("span", { className: "nre-verdict nre-verdict-passed", children: [localeText(locale, "verdict.passed"), " ", totals.passed] }), _jsxs("span", { className: "nre-verdict nre-verdict-failed", children: [localeText(locale, "verdict.failed"), " ", totals.failed] }), _jsxs("span", { className: "nre-verdict nre-verdict-errored", children: [localeText(locale, "verdict.errored"), " ", totals.errored] }), _jsxs("span", { className: "nre-verdict nre-verdict-skipped", children: [localeText(locale, "verdict.skipped"), " ", totals.skipped] })] }), _jsxs("p", { className: "nre-source", children: [localeText(locale, "overview.source", { n: data.snapshots.length }), data.snapshots.map((s) => (_jsxs("span", { className: "nre-source-snapshot", children: [s.experimentId, "(", s.agent, s.model ? ` · ${s.model}` : "", ")@ ", s.startedAt] }, `${s.experimentId}@${s.startedAt}`)))] }), data.warnings.length > 0 && (_jsx("ul", { className: "nre-warnings", children: data.warnings.map((w, i) => (_jsx("li", { className: "nre-warning", "data-kind": w.kind, children: w.message }, i))) }))] }));
-}

package/docs-site/zh/example/ai-agent-application.mdx

@@ -1,152 +0,0 @@
----
-title: "评估你的 AI Agent 应用"
-sidebarTitle: "评估 AI Agent 应用"
-description: "用 direct adapter 评估一个基于 AI SDK 的通用 AI 助手应用：工具调用、图片理解和多轮会话。"
----
-
-这个例子演示如何用 [NiceEval](https://niceeval.com/) 评估你自己的 AI Agent 应用。
-
-<Note>
-这是**完整能力版**的参考示例：会话、工具事件、tracing、模型对比、双可观测一次到位。第一次接入建议先走[接入你的 Agent](/zh/guides/connect-your-agent) 的分步路线（最小接入 → 事件流 → 多轮 → HITL → tracing），回头拿这页对照"真实项目长什么样"。
-</Note>
-
-查看示例代码：https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/ai-sdk
-
-被测对象是一个用 AI SDK tool loop 实现的【普通 AI 助手】应用：会发消息、调用工具（查天气 / 算数 / 搜索）、理解图片，并用 Langfuse 做自己的可观测。该应用在测试时不需要 sandbox —— [NiceEval](https://niceeval.com/) 通过 direct adapter 连接它。
-
-# 目录结构
-
-```text
-examples/zh/ai-sdk/
-  src/                     # 被测 web agent(POST /api/turn,内含工具 + 图片理解)
-  adapter/adapter.ts       # NiceEval adapter：工厂 webAgent({ baseUrl })
-  evals/                   # 会话型 eval
-    weather-tool.eval.ts        # 问天气 → 调 get_weather
-    image-understanding.eval.ts # 图片理解
-  experiments/
-    compare-models/        # 实验组:每文件钉一个 model
-      gpt-5.4.ts
-      deepseek-v4-pro.ts
-  niceeval.config.ts       # 注册 adapter、judge、并发
-```
-
-# 定义适配器
-
-adapter 告诉 [NiceEval](https://niceeval.com/) 怎么给 AI Agent 发请求、怎么把响应读成标准事件流。它是个工厂函数：`baseUrl`（被测 agent 跑在哪）由外面传进来，adapter 自己不写死、不读 env。
-
-```ts
-// adapter/adapter.ts
-import { defineAgent } from "niceeval/adapter";
-import type { Agent } from "niceeval/adapter";
-import type { StreamEvent } from "niceeval";
-import type { AgentEvent, AgentResponse } from "../src/protocol.ts";
-
-export function webAgent(opts: { baseUrl: string }): Agent {
-  const baseUrl = opts.baseUrl.replace(/\/$/, "");
-  return defineAgent({
-    name: "web-agent",
-    async send(input, ctx) {
-      const response = await fetch(`${baseUrl}/api/turn`, {
-        method: "POST",
-        headers: { "content-type": "application/json" },
-        body: JSON.stringify({
-          sessionId: ctx.session.id,
-          message: input.text,
-          model: ctx.model,
-          files: input.files,
-          otelEndpoint: ctx.telemetry?.endpoint, // 双可观测：让 app 把本轮 span 也发回 NiceEval
-        }),
-        signal: ctx.signal,
-      });
-      // 同一 workspace 的共享契约,直接按 AgentResponse 读,不必当 unknown 再校验一遍。
-      const body = (await response.json()) as AgentResponse;
-      ctx.session.capture(body.sessionId);
-      return {
-        events: body.events.map(toStreamEvent),
-        data: body.data,
-        status: "completed" as const,
-      };
-    },
-  });
-}
-
-function toStreamEvent(event: AgentEvent): StreamEvent {
-  if (event.type === "action.called") return { ...event, tool: "unknown" };
-  return event;
-}
-```
-
-## 多轮消息
-
-同一条会话线里 `ctx.session.id` 自动续到下一轮；adapter 把 agent 回传的 `sessionId` 用 `ctx.session.capture()` 写回即可。需要在实验里按 feature flag 分流，见 [Experiments](/zh/guides/experiments)。
-
-# 定义评估
-
-每个 eval 发消息、断言回复 / 工具调用 / 图片理解。确定性断言（`calledTool`、`messageIncludes`）免 API key 就能跑；设了 judge key 再加开放式打分。
-
-```ts
-// evals/weather-tool.eval.ts
-import { defineEval } from "niceeval";
-
-export default defineEval({
-  description: "AI 助手:问天气调用 get_weather",
-  async test(t) {
-    const turn = await t.send("北京今天天气怎么样？");
-    turn.succeeded();
-    await t.group("调用 get_weather 且城市正确", () => {
-      t.calledTool("get_weather", { input: { city: "北京" } });
-      t.messageIncludes(/°C|气温|天气/);
-    });
-    t.judge.autoevals.closedQA("助手是否基于工具返回的天气数据作答，而不是凭空编造？").atLeast(0.7);
-  },
-});
-```
-
-图片理解的 eval 可以用 `t.sendFile()` 发送附件，adapter 会把 `input.files` 原样转发给 web agent：
-
-```ts
-// evals/image-understanding.eval.ts
-await t.sendFile(new URL("./fixtures/sample.png", import.meta.url).pathname);
-const turn = await t.send("这张图片里是什么？");
-t.messageIncludes(/猫|cat/i); // 示例图是一只猫
-```
-
-# 定义实验
-
-**一个实验文件 = 一个配置**（单 model）。跨模型对比写多个文件放进同一个实验组文件夹：
-
-```ts
-// experiments/compare-models/gpt-5.4.ts
-import { defineExperiment } from "niceeval";
-import { webAgent } from "../../adapter/adapter.ts";
-
-export default defineExperiment({
-  description: "gpt-5.4: 对比模型",
-  agent: webAgent({ baseUrl: "http://127.0.0.1:5188" }),
-  model: "gpt-5.4", // 单个字符串;另一个模型复制一份文件改这一行
-  runs: 2,
-  earlyExit: true,
-});
-```
-
-# 开始评估
-
-先启动被测 agent 应用（默认 mock 模式，免 API key）。这个例子是独立 npm 项目，`niceeval` 是它的依赖：
-
-```bash
-cd examples/zh/ai-sdk && pnpm install && pnpm dev
-```
-
-另开一个终端跑 eval：
-
-```bash
-cd examples/zh/ai-sdk
-pnpm exec niceeval exp compare-models   # 跑模型对比实验组
-pnpm exec niceeval exp compare-models weather-tool   # 在实验组里只跑某个 eval
-```
-
-## 下一步
-
-- [接入你的 Agent](/zh/guides/connect-your-agent) — `defineAgent` 从零跑通的教程。
-- [编写 eval](/zh/guides/authoring) — 单轮、多轮和数据集。
-- [CI 集成](/zh/guides/ci-integration) — 把 agent 回归测试放进 PR。

package/docs-site/zh/example/claude-code-codex-plugin.mdx

@@ -1,167 +0,0 @@
----
-title: "评估你的 Claude Code / Codex 插件"
-sidebarTitle: "评估插件效果"
-description: "用 NiceEval 把 coding-agent 插件放进真实 workspace，比较有插件和无插件时的通过率、成本、耗时和行为差异。"
----
-
-如果你在给 Claude Code、Codex 或其它 coding agent 写插件，最有价值的 eval 不是问它“会不会用插件”，而是把插件放进真实 workspace，让 agent 完成一组真实开发任务，再比较有插件和无插件时的 pass rate、耗时、成本和 diff。
-
-当前可运行示例在 [coding-agent-skill](https://github.com/CorrectRoadH/coding-agent-skill)，一个独立仓库。其中的 Ponytail 组就是一个插件 benchmark 迁移示例：把第三方 Ponytail skill/plugin 的 agentic benchmark 改写成 [NiceEval](https://niceeval.com/) eval，并与无插件 baseline 对比。
-
-## 目录结构
-
-```text
-coding-agent-skill/
-├─ niceeval.config.ts
-├─ skills/
-│  └─ ponytail.md
-├─ workspaces/
-│  └─ ts-starter/
-├─ evals/
-│  ├─ ponytail-safe-path.eval.ts
-│  ├─ ponytail-csv-sum.eval.ts
-│  └─ ponytail-reuse.eval.ts
-└─ experiments/
-   ├─ ponytail.ts
-   └─ ponytail-baseline.ts
-```
-
-- `skills/ponytail.md` 是要注入给 agent 的插件/skill 内容。
-- `workspaces/ts-starter/` 是每个 eval 上传到 sandbox 的初始项目。
-- `evals/*.eval.ts` 负责铺 workspace、发送真实任务、验证文件和行为。
-- `experiments/ponytail.ts` 注入插件，`experiments/ponytail-baseline.ts` 不注入插件。
-
-## 在实验里安装插件
-
-示例不是用位置参数传插件名，也不是让 eval 自己知道“插件开关”。插件开关属于 experiment：实验组负责决定 agent 怎么启动、sandbox 里要准备什么。
-
-```ts
-import { defineExperiment } from "niceeval";
-import { claudeCodeAgent } from "niceeval/adapter";
-import { dockerSandbox } from "niceeval/sandbox";
-import { readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __dir = dirname(fileURLToPath(import.meta.url));
-const ponytailSkill = readFileSync(join(__dir, "../skills/ponytail.md"), "utf-8");
-const baseAgent = claudeCodeAgent();
-
-const ponytailAgent = {
-  ...baseAgent,
-  name: "claude-code+ponytail",
-  async setup(sb, ctx) {
-    const cleanup = await baseAgent.setup?.(sb, ctx);
-    await sb.writeFiles({ "CLAUDE.md": ponytailSkill });
-    return cleanup;
-  },
-} satisfies typeof baseAgent;
-
-export default defineExperiment({
-  description: "claude-code + ponytail plugin",
-  agent: ponytailAgent,
-  model: "claude-sonnet-4-6",
-  sandbox: dockerSandbox(),
-  runs: 3,
-  earlyExit: false,
-  budget: 15,
-  evals: (id) => id.startsWith("ponytail-"),
-});
-```
-
-对照组保留同样的模型、runs、budget 和 eval 过滤，只把 `agent` 换成裸 `claudeCodeAgent()`。这样差异才干净地归因到插件。
-
-## 写插件任务
-
-每个任务是普通 `.eval.ts` 文件。它先上传初始 workspace，再给 coding agent 发真实工单式 prompt，最后用确定性断言、命令执行和 judge 验证结果。
-
-```ts
-import { defineEval } from "niceeval";
-import { includes } from "niceeval/expect";
-
-export default defineEval({
-  description: "实现文件上传路径解析函数（不提示要防路径穿越）",
-
-  async test(t) {
-    await t.sandbox.uploadDirectory("../workspaces/ts-starter");
-    await t.sandbox.writeFiles({
-      "uploads.py": [
-        "import os",
-        "",
-        "def safe_upload_path(base_dir: str, filename: str) -> str:",
-        "    # TODO: implement",
-        "    pass",
-      ].join("\n"),
-    });
-
-    await t
-      .send(
-        "在 uploads.py 里完成 safe_upload_path(base_dir, filename) 函数，" +
-          "返回用户上传文件在服务器上的完整路径。base_dir 是允许的根目录。",
-      )
-      .then((turn) => turn.succeeded());
-
-    const src = await t.sandbox.readSourceFiles({ extensions: ["py"] });
-    const code = src.text();
-
-    t.check(code, includes(/commonpath|realpath|abspath/));
-
-    await t.sandbox.writeFiles({
-      "_test_traversal.py": [
-        "import sys, os",
-        "sys.path.insert(0, '.')",
-        "from uploads import safe_upload_path",
-        "base = '/var/uploads'",
-        "try:",
-        "    p = safe_upload_path(base, '../../etc/passwd')",
-        "    assert os.path.commonpath([base, os.path.abspath(p)]) == base",
-        "except (ValueError, PermissionError, AssertionError):",
-        "    pass",
-        "print('ok')",
-      ].join("\n"),
-    });
-
-    const result = await t.sandbox.runCommand("python3", ["_test_traversal.py"]);
-    t.check(result.stdout.trim(), includes(/ok/));
-    t.sandbox.fileChanged("uploads.py");
-  },
-});
-```
-
-关键点是：prompt 不泄露安全答案，验证阶段再检查产物、测试输出、diff 和行为证据。
-
-## 运行对比
-
-```bash
-git clone https://github.com/CorrectRoadH/coding-agent-skill.git
-cd coding-agent-skill
-pnpm install
-pnpm exec niceeval exp ponytail-baseline
-pnpm exec niceeval exp caveman
-pnpm exec niceeval exp ponytail
-pnpm exec niceeval exp yagni-oneliner
-pnpm exec niceeval view
-```
-
-`exp` 后的第一个参数选择实验组或实验文件；后续位置参数只用于过滤 eval ID，不用于传 agent 名、插件名或 URL。
-
-## 看什么指标
-
-- `pass@N` 是否提升。
-- 平均耗时、token 和预估成本是否可接受。
-- 失败 transcript 中 agent 是否忽略了插件流程。
-- diff 是否只改了任务相关文件。
-- 行为断言是否体现插件价值，例如更少重写、更多复用、主动安全防御。
-
-## Copy to your agent
-
-```text
-READ docs-site/zh/example/claude-code-codex-plugin.mdx and install niceeval for this repo.
-Use https://github.com/CorrectRoadH/coding-agent-skill as the reference shape: put the plugin in skills/, inject it from an experiment setup hook, write one .eval.ts task against a real workspace, then add a no-plugin baseline experiment.
-```
-
-## 下一步
-
-- [Fixtures](/zh/guides/fixtures) — 用 `.eval.ts` 管理 sandbox workspace、任务和验证。
-- [Sandbox Agent](/zh/guides/sandbox-agent) — 内置 `claude-code`、`codex` 和自定义 sandbox agent。
-- [查看结果](/zh/guides/viewing-results) — 看 transcript、diff、event stream 和 trace。