niceeval 0.4.3 → 0.4.5

package/docs-site/docs.json

@@ -0,0 +1,256 @@
+{
+  "$schema": "https://mintlify.com/docs.json",
+  "name": "niceeval",
+  "theme": "luma",
+  "favicon": "/favicon.svg",
+  "logo": {
+    "light": "/images/logo.svg",
+    "dark": "/images/logo.svg"
+  },
+  "colors": {
+    "primary": "#848c8e",
+    "light": "#11161e",
+    "dark": "#cfd0d2"
+  },
+  "navbar": {
+    "primary": {
+      "type": "github",
+      "href": "https://github.com/CorrectRoadH/niceeval"
+    }
+  },
+  "redirects": [
+    {
+      "source": "/concepts/scoring",
+      "destination": "/concepts/assert"
+    },
+    {
+      "source": "/concepts/agents-adapters",
+      "destination": "/concepts/adapter"
+    },
+    {
+      "source": "/guides/connect-agent",
+      "destination": "/guides/connect-your-agent"
+    },
+    {
+      "source": "/concepts/write-send",
+      "destination": "/guides/write-send"
+    },
+    {
+      "source": "/zh/concepts/scoring",
+      "destination": "/zh/concepts/assert"
+    },
+    {
+      "source": "/zh/concepts/agents-adapters",
+      "destination": "/zh/concepts/adapter"
+    },
+    {
+      "source": "/zh/concepts/write-send",
+      "destination": "/zh/guides/write-send"
+    },
+    {
+      "source": "/reference/configuration",
+      "destination": "/reference/define-config"
+    },
+    {
+      "source": "/zh/reference/configuration",
+      "destination": "/zh/reference/define-config"
+    },
+    {
+      "source": "/zh/example/eval-multil-turn-for-your-agent",
+      "destination": "/zh/concepts/drive"
+    }
+  ],
+  "navigation": {
+    "languages": [
+      {
+        "language": "en",
+        "default": true,
+        "tabs": [
+          {
+            "tab": "Documentation",
+            "groups": [
+              {
+                "group": "Get Started",
+                "pages": [
+                  "introduction",
+                  "quickstart"
+                ]
+              },
+              {
+                "group": "Core Concepts",
+                "pages": [
+                  "concepts/overview",
+                  "concepts/evals",
+                  "concepts/experiment",
+                  "concepts/adapter",
+                  "concepts/tier",
+                  "concepts/drive",
+                  "concepts/hitl",
+                  "concepts/assert",
+                  "concepts/judge"
+                ]
+              },
+              {
+                "group": "Connecting Agents",
+                "pages": [
+                  "guides/connect-your-agent",
+                  "guides/write-send",
+                  "guides/connect-otel",
+                  "guides/sandbox-agent",
+                  "guides/sandbox-backends"
+                ]
+              },
+              {
+                "group": "Evals & Experiments",
+                "pages": [
+                  "guides/authoring",
+                  "guides/write-experiment",
+                  "guides/experiments",
+                  "guides/fixtures",
+                  "guides/dataset-fanout",
+                  "guides/scoring-guide"
+                ]
+              },
+              {
+                "group": "Examples",
+                "pages": [
+                  "example/claude-code-codex-plugin",
+                  "example/claude-code-codex-skill",
+                  "example/ai-agent-application",
+                  "example/showcase"
+                ]
+              },
+              {
+                "group": "Running & Observing",
+                "pages": [
+                  "guides/runner",
+                  "guides/ci-integration",
+                  "guides/reporters",
+                  "guides/viewing-results"
+                ]
+              }
+            ]
+          },
+          {
+            "tab": "Reference",
+            "groups": [
+              {
+                "group": "API",
+                "pages": [
+                  "reference/define-eval",
+                  "reference/define-agent",
+                  "reference/events",
+                  "reference/capabilities",
+                  "reference/builtin-agents",
+                  "reference/define-config",
+                  "reference/expect"
+                ]
+              },
+              {
+                "group": "CLI",
+                "pages": [
+                  "reference/cli"
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "language": "zh-Hans",
+        "tabs": [
+          {
+            "tab": "文档",
+            "groups": [
+              {
+                "group": "开始使用",
+                "pages": [
+                  "zh/introduction",
+                  "zh/quickstart"
+                ]
+              },
+              {
+                "group": "核心概念",
+                "pages": [
+                  "zh/concepts/overview",
+                  "zh/concepts/evals",
+                  "zh/concepts/experiment",
+                  "zh/concepts/adapter",
+                  "zh/concepts/tier",
+                  "zh/concepts/drive",
+                  "zh/concepts/hitl",
+                  "zh/concepts/assert",
+                  "zh/concepts/judge"
+                ]
+              },
+              {
+                "group": "连接 Agent",
+                "pages": [
+                  "zh/guides/connect-your-agent",
+                  "zh/guides/official-adapters",
+                  "zh/guides/write-send",
+                  "zh/guides/connect-otel",
+                  "zh/guides/sandbox-agent",
+                  "zh/guides/sandbox-backends"
+                ]
+              },
+              {
+                "group": "评估与实验",
+                "pages": [
+                  "zh/guides/authoring",
+                  "zh/guides/write-experiment",
+                  "zh/guides/experiments",
+                  "zh/guides/fixtures",
+                  "zh/guides/dataset-fanout",
+                  "zh/guides/scoring-guide"
+                ]
+              },
+              {
+                "group": "示例",
+                "pages": [
+                  "zh/example/claude-code-codex-plugin",
+                  "zh/example/claude-code-codex-skill",
+                  "zh/example/ai-agent-application",
+                  "zh/example/showcase"
+                ]
+              },
+              {
+                "group": "运行与观察",
+                "pages": [
+                  "zh/guides/runner",
+                  "zh/guides/ci-integration",
+                  "zh/guides/reporters",
+                  "zh/guides/viewing-results",
+                  "zh/guides/agent-feedback-loop"
+                ]
+              }
+            ]
+          },
+          {
+            "tab": "参考",
+            "groups": [
+              {
+                "group": "API",
+                "pages": [
+                  "zh/reference/define-eval",
+                  "zh/reference/define-agent",
+                  "zh/reference/events",
+                  "zh/reference/capabilities",
+                  "zh/reference/builtin-agents",
+                  "zh/reference/define-config",
+                  "zh/reference/expect"
+                ]
+              },
+              {
+                "group": "CLI",
+                "pages": [
+                  "zh/reference/cli"
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

package/docs-site/example/ai-agent-application.mdx

@@ -0,0 +1,148 @@
+---
+title: "Eval Your AI Agent Application"
+sidebarTitle: "Eval AI Agent Applications"
+description: ""
+---
+
+This example shows how to use niceeval to evaluate your own AI agent application.
+
+See the full example code: https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/ai-sdk
+
+The system under test is a general-purpose AI assistant application built with an AI SDK tool loop. It handles messages, calls tools (weather / calculator / search), understands images, and uses Langfuse for its own observability. The application doesn't need a sandbox during testing — niceeval reaches it through a direct adapter.
+
+## Directory structure
+
+```text
+examples/zh/ai-sdk/
+  src/                     # web agent under test (POST /api/turn, tools + image understanding)
+  adapter/adapter.ts       # niceeval adapter: factory webAgent({ baseUrl })
+  evals/                   # conversational evals
+    weather-tool.eval.ts        # ask about weather → calls get_weather
+    image-understanding.eval.ts # image understanding
+  experiments/
+    compare-models/        # experiment group: one file per model
+      gpt-5.4.ts
+      deepseek-v4-pro.ts
+  niceeval.config.ts       # register adapter, judge, concurrency
+```
+
+## Define the adapter
+
+The adapter tells niceeval how to send requests to the AI agent and how to read responses as a standard event stream. It's a factory function: `baseUrl` (where the service runs) is passed in from the outside so the adapter never hardcodes it or reads from env.
+
+```ts
+// adapter/adapter.ts
+import { defineAgent } from "niceeval/adapter";
+import type { Agent } from "niceeval/adapter";
+import type { StreamEvent } from "niceeval";
+import type { AgentEvent, AgentResponse } from "../src/protocol.ts";
+
+export function webAgent(opts: { baseUrl: string }): Agent {
+  const baseUrl = opts.baseUrl.replace(/\/$/, "");
+  return defineAgent({
+    name: "web-agent",
+    async send(input, ctx) {
+      const response = await fetch(`${baseUrl}/api/turn`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          sessionId: ctx.session.id,
+          message: input.text,
+          model: ctx.model,
+          files: input.files,
+          otelEndpoint: ctx.telemetry?.endpoint, // dual observability: let the app send this turn's spans back to niceeval
+        }),
+        signal: ctx.signal,
+      });
+      // Shared contract within the same workspace — read as AgentResponse directly, no need to validate as unknown.
+      const body = (await response.json()) as AgentResponse;
+      ctx.session.capture(body.sessionId);
+      return {
+        events: body.events.map(toStreamEvent),
+        data: body.data,
+        status: "completed" as const,
+      };
+    },
+  });
+}
+
+function toStreamEvent(event: AgentEvent): StreamEvent {
+  if (event.type === "action.called") return { ...event, tool: "unknown" };
+  return event;
+}
+```
+
+## Multi-turn messages
+
+Within the same conversation line, `ctx.session.id` automatically carries over to the next turn; the adapter just writes the agent's returned `sessionId` back with `ctx.session.capture()`. To split traffic by feature flag within an experiment, see [Experiments](/guides/experiments).
+
+## Define evals
+
+Each eval sends a message and asserts on the reply, tool calls, and image understanding. Deterministic assertions (`calledTool`, `messageIncludes`) run without an API key; open-ended scoring with a judge requires a key to be set.
+
+```ts
+// evals/weather-tool.eval.ts
+import { defineEval } from "niceeval";
+
+export default defineEval({
+  description: "AI assistant: asking about weather calls get_weather",
+  async test(t) {
+    const turn = await t.send("What's the weather like in Brooklyn today?");
+    turn.succeeded();
+    await t.group("calls get_weather with correct city", () => {
+      t.calledTool("get_weather", { input: { city: "Brooklyn" } });
+      t.messageIncludes(/°[CF]|temperature|weather/i);
+    });
+    t.judge.autoevals.closedQA("Does the assistant answer based on the tool's returned weather data, not a hallucination?").atLeast(0.7);
+  },
+});
+```
+
+For image understanding, use `t.sendFile()` to attach an image. The adapter forwards `input.files` to the web agent:
+
+```ts
+// evals/image-understanding.eval.ts
+await t.sendFile(new URL("./fixtures/sample.png", import.meta.url).pathname);
+const turn = await t.send("What's in this image?");
+t.messageIncludes(/cat/i); // the sample image is a cat
+```
+
+## Define experiments
+
+**One experiment file = one configuration** (single model). For cross-model comparison, write multiple files in the same experiment group folder:
+
+```ts
+// experiments/compare-models/gpt-5.4.ts
+import { defineExperiment } from "niceeval";
+import { webAgent } from "../../adapter/adapter.ts";
+
+export default defineExperiment({
+  description: "gpt-5.4: model comparison",
+  agent: webAgent({ baseUrl: "http://127.0.0.1:5188" }),
+  model: "gpt-5.4", // single string; copy this file and change one line for each model
+  runs: 2,
+  earlyExit: true,
+});
+```
+
+## Start evaluating
+
+First start the service under test (defaults to mock mode — no API key required). This example is a standalone npm project where `niceeval` is a local dependency:
+
+```bash
+cd examples/zh/ai-sdk && pnpm install && pnpm dev
+```
+
+Open a second terminal and run evals:
+
+```bash
+cd examples/zh/ai-sdk
+pnpm exec niceeval exp compare-models   # run the model comparison experiment
+pnpm exec niceeval exp compare-models weather-tool   # run one eval inside that experiment
+```
+
+## Next steps
+
+- [Connect Your Agent](/guides/connect-your-agent) — the from-scratch tutorial for `defineAgent`.
+- [Authoring Evals](/guides/authoring) — single-turn, multi-turn, and dataset evals.
+- [CI Integration](/guides/ci-integration) — put agent regression tests in PRs.

package/docs-site/example/claude-code-codex-plugin.mdx

@@ -0,0 +1,162 @@
+---
+title: "Eval Your Claude Code / Codex Plugin"
+sidebarTitle: "Eval CC/Codex Plugins"
+description: "Use niceeval to put a coding-agent plugin in a real workspace and compare pass rate, latency, cost, and behavior with and without the plugin."
+---
+
+If you're building a plugin for Claude Code, Codex, or another coding agent, the useful eval is not "can the agent mention the plugin?" It is: put the plugin in a real workspace, ask the agent to complete real development tasks, and compare pass rate, latency, cost, and diffs against a no-plugin baseline.
+
+The runnable reference is [`examples/zh/coding-agent-skill`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/coding-agent-skill). Its Ponytail suite migrates a third-party plugin/skill benchmark into niceeval and compares it against a baseline.
+
+## Directory structure
+
+```text
+examples/zh/coding-agent-skill/
+├─ niceeval.config.ts
+├─ skills/
+│  └─ ponytail.md
+├─ workspaces/
+│  └─ ts-starter/
+├─ evals/
+│  ├─ ponytail-safe-path.eval.ts
+│  ├─ ponytail-csv-sum.eval.ts
+│  └─ ponytail-reuse.eval.ts
+└─ experiments/
+   ├─ ponytail.ts
+   └─ ponytail-baseline.ts
+```
+
+- `skills/ponytail.md` is the plugin/skill content injected into the agent workspace.
+- `workspaces/ts-starter/` is the initial project uploaded into each sandbox.
+- `evals/*.eval.ts` uploads the workspace, sends the task, and verifies the result.
+- `experiments/ponytail.ts` injects the plugin; `experiments/ponytail-baseline.ts` does not.
+
+## Install the plugin in an experiment
+
+Plugin on/off belongs in the experiment, not in the eval ID or CLI positionals. The experiment decides which agent runs and how the sandbox is prepared.
+
+```ts
+import { defineExperiment } from "niceeval";
+import { claudeCodeAgent } from "niceeval/adapter";
+import { dockerSandbox } from "niceeval/sandbox";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dir = dirname(fileURLToPath(import.meta.url));
+const ponytailSkill = readFileSync(join(__dir, "../skills/ponytail.md"), "utf-8");
+const baseAgent = claudeCodeAgent();
+
+const ponytailAgent = {
+  ...baseAgent,
+  name: "claude-code+ponytail",
+  async setup(sb, ctx) {
+    const cleanup = await baseAgent.setup?.(sb, ctx);
+    await sb.writeFiles({ "CLAUDE.md": ponytailSkill });
+    return cleanup;
+  },
+} satisfies typeof baseAgent;
+
+export default defineExperiment({
+  description: "claude-code + ponytail plugin",
+  agent: ponytailAgent,
+  model: "claude-sonnet-4-6",
+  sandbox: dockerSandbox(),
+  runs: 3,
+  earlyExit: false,
+  budget: 15,
+  evals: (id) => id.startsWith("ponytail-"),
+});
+```
+
+The baseline keeps the same model, runs, budget, and eval filter, but uses plain `claudeCodeAgent()`. That keeps the comparison attributable to the plugin.
+
+## Write plugin tasks
+
+Each task is a normal `.eval.ts` file. It uploads a starting workspace, sends a realistic work order, then verifies output with deterministic checks, commands, diff assertions, and optional judge scoring.
+
+```ts
+import { defineEval } from "niceeval";
+import { includes } from "niceeval/expect";
+
+export default defineEval({
+  description: "Implement an upload path resolver without explicitly mentioning traversal",
+
+  async test(t) {
+    await t.sandbox.uploadDirectory("../workspaces/ts-starter");
+    await t.sandbox.writeFiles({
+      "uploads.py": [
+        "import os",
+        "",
+        "def safe_upload_path(base_dir: str, filename: str) -> str:",
+        "    # TODO: implement",
+        "    pass",
+      ].join("\n"),
+    });
+
+    await t
+      .send(
+        "Complete safe_upload_path(base_dir, filename) in uploads.py. " +
+          "Return the full server path for a user-uploaded file. base_dir is the allowed root.",
+      )
+      .then((turn) => turn.succeeded());
+
+    const src = await t.sandbox.readSourceFiles({ extensions: ["py"] });
+    const code = src.text();
+    t.check(code, includes(/commonpath|realpath|abspath/));
+
+    await t.sandbox.writeFiles({
+      "_test_traversal.py": [
+        "import sys, os",
+        "sys.path.insert(0, '.')",
+        "from uploads import safe_upload_path",
+        "base = '/var/uploads'",
+        "try:",
+        "    p = safe_upload_path(base, '../../etc/passwd')",
+        "    assert os.path.commonpath([base, os.path.abspath(p)]) == base",
+        "except (ValueError, PermissionError, AssertionError):",
+        "    pass",
+        "print('ok')",
+      ].join("\n"),
+    });
+
+    const result = await t.sandbox.runCommand("python3", ["_test_traversal.py"]);
+    t.check(result.stdout.trim(), includes(/ok/));
+    t.sandbox.fileChanged("uploads.py");
+  },
+});
+```
+
+The prompt does not leak the safety answer. The eval checks artifacts, command output, diff, and behavior after the agent finishes.
+
+## Run the comparison
+
+```bash
+cd examples/zh/coding-agent-skill
+pnpm install
+pnpm exec niceeval exp ponytail ponytail-baseline
+pnpm exec niceeval view
+```
+
+The first argument after `exp` selects the experiment group or file. Later positionals only filter eval IDs; they are not agent names, plugin names, URLs, or runtime config.
+
+## What to inspect
+
+- Whether `pass@N` improves.
+- Whether average latency, tokens, and estimated cost remain acceptable.
+- Whether failing transcripts show the agent ignored the plugin process.
+- Whether the diff only touches task-relevant files.
+- Whether behavior assertions capture the plugin's value, such as more reuse, less overengineering, or proactive safety checks.
+
+## Copy to your agent
+
+```text
+READ docs-site/example/claude-code-codex-plugin.mdx and install niceeval for this repo.
+Use examples/zh/coding-agent-skill as the reference shape: put the plugin in skills/, inject it from an experiment setup hook, write one .eval.ts task against a real workspace, then add a no-plugin baseline experiment.
+```
+
+## Next steps
+
+- [Fixtures](/guides/fixtures) — manage sandbox workspaces, tasks, and verification in `.eval.ts`.
+- [Sandbox Agent](/guides/sandbox-agent) — built-in `claude-code`, `codex`, and custom sandbox agents.
+- [Viewing Results](/guides/viewing-results) — inspect transcripts, diffs, event streams, and traces.