niceeval 0.4.5 → 0.5.0

package/docs-site/github-diff.css

@@ -1,135 +0,0 @@
-/* 本文件由 scripts/gen-diff-code.ts 生成（pnpm run gen:diff-code），不要手工编辑 */
-/* GitHub PR 式 diff 视图，配合生成的 diff MDX 页面使用 */
-
-.gd-file {
-  margin: 1rem 0;
-  border: 1px solid #d0d7de;
-  border-radius: 8px;
-  overflow: hidden;
-  font-size: 12px;
-}
-.dark .gd-file { border-color: #30363d; }
-
-.gd-head {
-  display: flex;
-  align-items: center;
-  gap: 8px;
-  padding: 8px 12px;
-  background: #f6f8fa;
-  border-bottom: 1px solid #d0d7de;
-  user-select: none;
-}
-.dark .gd-head { background: #161b22; border-bottom-color: #30363d; }
-
-.gd-name {
-  font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace;
-  font-weight: 600;
-  color: #1f2328;
-}
-.dark .gd-name { color: #e6edf3; }
-
-.gd-stats { margin-left: auto; font-weight: 600; display: flex; gap: 6px; }
-.gd-plus { color: #1a7f37; }
-.gd-minus { color: #cf222e; }
-.dark .gd-plus { color: #3fb950; }
-.dark .gd-minus { color: #f85149; }
-
-.gd-body { overflow-x: auto; background: #ffffff; }
-.dark .gd-body { background: #0d1117; }
-
-.gd-note { padding: 12px; color: #656d76; background: #ffffff; }
-.dark .gd-note { color: #8b949e; background: #0d1117; }
-
-table.gd-table {
-  width: 100%;
-  margin: 0;
-  border-collapse: collapse;
-  font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace;
-  font-size: 12px;
-  line-height: 20px;
-  display: table;
-}
-.gd-table tr { border: 0; background: transparent; }
-.gd-table td { border: 0; padding: 0; background: transparent; }
-
-td.gd-ln {
-  width: 1%;
-  min-width: 40px;
-  padding: 0 10px;
-  text-align: right;
-  color: #656d76;
-  user-select: none;
-  vertical-align: top;
-}
-.dark td.gd-ln { color: #6e7681; }
-
-td.gd-sign {
-  width: 1%;
-  padding: 0 4px;
-  text-align: center;
-  user-select: none;
-  color: #1f2328;
-  vertical-align: top;
-}
-.dark td.gd-sign { color: #e6edf3; }
-
-td.gd-code {
-  padding: 0 10px 0 4px;
-  white-space: pre;
-  color: #1f2328;
-  tab-size: 2;
-}
-.dark td.gd-code { color: #e6edf3; }
-
-/* 背景挂在 td 而不是 tr 上：tr 背景在 Safari / 非整数缩放下行间会出 hairline */
-tr.gd-add td { background: #e6ffec; }
-tr.gd-add td.gd-ln { background: #ccffd8; }
-.dark tr.gd-add td { background: rgba(46, 160, 67, 0.15); }
-.dark tr.gd-add td.gd-ln { background: rgba(63, 185, 80, 0.3); color: #c9d1d9; }
-
-tr.gd-del td { background: #ffebe9; }
-tr.gd-del td.gd-ln { background: #ffd7d5; }
-.dark tr.gd-del td { background: rgba(248, 81, 73, 0.1); }
-.dark tr.gd-del td.gd-ln { background: rgba(248, 81, 73, 0.3); color: #c9d1d9; }
-
-/* 页面开头的变更统计表（JSX 表格：同页有超长 JSX 行时 GFM 管道表格会压爆 MDX 编译栈） */
-table.gd-summary { border-collapse: collapse; margin: 1rem 0; font-size: 14px; display: table; width: auto; }
-.gd-summary th, .gd-summary td { border: 1px solid #d0d7de; padding: 6px 12px; text-align: left; background: transparent; }
-.gd-summary th { background: #f6f8fa; font-weight: 600; }
-.gd-summary th:nth-child(n+2), .gd-summary td:nth-child(n+2) { text-align: right; font-variant-numeric: tabular-nums; }
-tr.gd-total td { font-weight: 600; background: #f6f8fa; }
-.dark .gd-summary th, .dark .gd-summary td { border-color: #30363d; }
-.dark .gd-summary th { background: #161b22; }
-.dark tr.gd-total td { background: #161b22; }
-
-/* 折叠的未变更行 + GitHub 式蓝色展开条（点击逻辑在 github-diff.js） */
-tr.gd-fold { display: none; }
-tr.gd-expand { cursor: pointer; }
-tr.gd-expand td { background: #ddf4ff; }
-tr.gd-expand td.gd-ln { color: #0969da; text-align: center; }
-tr.gd-expand td.gd-code { color: #57606a; }
-tr.gd-expand:hover td { background: #b6e3ff; }
-.dark tr.gd-expand td { background: rgba(56, 139, 253, 0.15); }
-.dark tr.gd-expand td.gd-ln { color: #58a6ff; }
-.dark tr.gd-expand td.gd-code { color: #8b949e; }
-.dark tr.gd-expand:hover td { background: rgba(56, 139, 253, 0.3); }
-
-/* Shiki 调色板（github-light / github-dark） */
-.gdt0 { color: #24292E; }
-.dark .gdt0 { color: #E1E4E8; }
-.gdt1 { color: #005CC5; }
-.dark .gdt1 { color: #79B8FF; }
-.gdt2 { color: #032F62; }
-.dark .gdt2 { color: #9ECBFF; }
-.gdt3 { color: #22863A; }
-.dark .gdt3 { color: #85E89D; }
-.gdt4 { color: #D73A49; }
-.dark .gdt4 { color: #F97583; }
-.gdt5 { color: #6F42C1; }
-.dark .gdt5 { color: #B392F0; }
-.gdt6 { color: #6A737D; }
-.dark .gdt6 { color: #6A737D; }
-.gdt7 { color: #E36209; }
-.dark .gdt7 { color: #FFAB70; }
-.gdt8 { color: #032F62; }
-.dark .gdt8 { color: #DBEDFF; }

package/docs-site/github-diff.js

@@ -1,11 +0,0 @@
-// 本文件由 scripts/gen-diff-code.ts 生成（pnpm run gen:diff-code），不要手工编辑
-// GitHub 式 diff 展开条：点击后显示折叠的未变更行（配合 github-diff.css 的 .gd-fold）
-document.addEventListener("click", (e) => {
-  const tr = e.target && e.target.closest ? e.target.closest("tr.gd-expand") : null;
-  if (!tr) return;
-  const id = tr.getAttribute("data-fold");
-  const tbody = tr.closest("tbody");
-  if (!id || !tbody) return;
-  tbody.querySelectorAll("tr." + id).forEach((row) => row.classList.remove("gd-fold"));
-  tr.remove();
-});

package/docs-site/guides/authoring.mdx

@@ -1,129 +0,0 @@
----
-title: "Authoring evals: single-turn, multi-turn, and dataset patterns"
-sidebarTitle: "Authoring Evals"
-description: "Learn how to write NiceEval evals with defineEval, including single-turn assertions, multi-turn conversations, dataset fan-out, and sandbox workspaces."
----
-
-Every NiceEval eval is a TypeScript file that default-exports `defineEval(...)`. The core rules are simple: **path is identity**, **one file is one eval**, and **write linearly with assertions next to the interaction they check**.
-
-## The `defineEval` shape
-
-```ts
-import { defineEval } from "niceeval";
-
-export default defineEval({
-  description?: string;
-  tags?: string[];
-  judge?: JudgeConfig;
-  reporters?: Reporter[];
-  timeoutMs?: number;
-  metadata?: Record<string, unknown>;
-  async test(t) { /* interactions + assertions */ },
-});
-```
-
-<Warning>
-  Do not set `id` or `name`. `evals/weather/brooklyn.eval.ts` automatically becomes the ID `weather/brooklyn`.
-</Warning>
-
-## Single-turn evals
-
-```ts
-import { defineEval } from "niceeval";
-import { includes } from "niceeval/expect";
-
-export default defineEval({
-  description: "Brooklyn weather query",
-  async test(t) {
-    await t.send("What's the weather like in Brooklyn today?");
-    t.succeeded();
-    t.calledTool("get_weather", { input: { city: "Brooklyn" }, count: 1 });
-    t.check(t.reply, includes("sunny"));
-  },
-});
-```
-
-`t.send()` drives one interaction. `t.succeeded()` and `t.calledTool()` are scoped assertions. `t.check()` records an immediate value assertion.
-
-### The `Turn` object
-
-| Property | Meaning |
-|---|---|
-| `turn.events` | Standard event stream, the main fact source |
-| `turn.data` | Structured output |
-| `turn.status` | `"completed"`, `"failed"`, or `"waiting"` |
-| `turn.usage` | usage such as tokens and cost |
-| `turn.message` | assistant text reply |
-| `turn.toolCalls` | tool calls in this turn |
-
-## Multi-turn evals
-
-```ts
-export default defineEval({
-  description: "Draft an email, then send it on confirmation",
-  async test(t) {
-    const draft = await t.send("Draft a follow-up email.");
-    draft.succeeded();
-    t.check(draft.message, includes("Best"));
-
-    await t.send("Looks good, send it.");
-    t.calledTool("send_email");
-  },
-});
-```
-
-Use `t.newSession()` when you need multiple independent conversations in parallel.
-
-## Dataset fan-out
-
-One file can export an eval array:
-
-```ts
-export default rows.map((row) =>
-  defineEval({
-    description: row.task,
-    async test(t) {
-      await t.send(row.prompt);
-      t.check(t.reply, equals(row.expected));
-    },
-  }),
-);
-```
-
-That generates IDs such as `sql/0000`, `sql/0001`, and so on. See [Dataset fan-out](/guides/dataset-fanout).
-
-## Sandbox workspace
-
-Coding-agent evals are still normal `.eval.ts` files. They just prepare a sandbox workspace, send a task, and inspect files or tests:
-
-```ts
-import { defineEval } from "niceeval";
-
-export default defineEval({
-  description: "Create a Button component",
-  async test(t) {
-    await t.sandbox.uploadDirectory("../workspaces/ts-starter");
-    await t.send("Create src/components/Button.tsx with label and onClick props.").then((turn) => turn.succeeded());
-    t.sandbox.fileChanged("src/components/Button.tsx");
-  },
-});
-```
-
-See [Fixtures](/guides/fixtures) for the full pattern.
-
-## Naming conventions
-
-<CardGroup cols={2}>
-  <Card title="Filename" icon="file">
-    Only `.eval.ts` files are discovered by the runner.
-  </Card>
-  <Card title="Directory grouping" icon="folder">
-    `evals/billing/refund.eval.ts` becomes the ID `billing/refund`.
-  </Card>
-  <Card title="Dataset" icon="database">
-    Good for many cases with the same structure and different inputs.
-  </Card>
-  <Card title="Sandbox workspace" icon="box">
-    Good for coding agents that need a real file system, commands, and diffs.
-  </Card>
-</CardGroup>

package/docs-site/guides/ci-integration.mdx

@@ -1,97 +0,0 @@
----
-title: "Run NiceEval in GitHub Actions and CI"
-sidebarTitle: "CI Integration"
-description: "Add NiceEval to GitHub Actions or any CI. Failed evals exit non-zero, JUnit XML can be emitted, and caching can speed up repeated runs."
----
-
-Evals should live in CI the same way tests do. They catch agent regressions on pull requests and track model, cost, and latency changes in nightly jobs.
-
-## Exit codes
-
-By default, NiceEval exits non-zero when any gate fails. In CI you usually want `--strict` so failure is explicit:
-
-```bash
-npx niceeval exp ci --strict
-```
-
-## GitHub Actions example
-
-```yaml
-name: evals
-on:
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  evals:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: npm
-      - run: npm ci
-      - run: npx niceeval exp ci --strict
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-```
-
-## Configure secrets
-
-<Steps>
-  <Step title="Add repository secrets">
-    Put provider tokens into GitHub Actions secrets.
-  </Step>
-  <Step title="Pass them as env in the workflow">
-    Expose only the variables the eval step needs.
-  </Step>
-  <Step title="Verify they exist">
-    Use a minimal eval, or run `list` / a dry run first to confirm configuration.
-  </Step>
-</Steps>
-
-## JUnit reporter
-
-```ts
-import { defineConfig } from "niceeval";
-import { Console, JUnit } from "niceeval/reporters";
-
-export default defineConfig({
-  reporters: [Console(), JUnit(".niceeval/junit.xml")],
-});
-```
-
-CI can upload the JUnit XML so failures appear in the test-report UI.
-
-## Discovery only
-
-```bash
-npx niceeval list
-```
-
-This is useful when you only want to confirm that eval files load and configuration is sane.
-
-## Cache `.niceeval/`
-
-You can cache `.niceeval/` in CI, but make sure the fingerprint covers every input that affects the result. For nightly benchmarks, keeping the full artifacts is usually more valuable.
-
-## Control concurrency
-
-```bash
-npx niceeval exp ci --max-concurrency 2
-```
-
-Sandbox evals should stay conservative on a standard GitHub-hosted runner. Remote HTTP evals can usually run with higher concurrency if the service allows it.
-
-## Recommended patterns
-
-<CardGroup cols={2}>
-  <Card title="PR runs" icon="code-pull-request">
-    Run only critical evals and high-risk paths so feedback stays fast.
-  </Card>
-  <Card title="Nightly full-matrix runs" icon="moon">
-    Run the full experiment matrix and record pass rate, cost, and latency trends.
-  </Card>
-</CardGroup>

package/docs-site/guides/connect-otel.mdx

@@ -1,209 +0,0 @@
----
-title: "OTel Integration"
-sidebarTitle: "OTel Integration"
-description: "Send a copy of the OTel spans your app already emits to NiceEval and get a per-turn call waterfall in niceeval view. Assertions do not come from here — they are complete the moment send is wired up."
----
-
-First, what this integration does **not** change: assertions. Every verdict — `t.calledTool`, `t.maxTokens`, timing — is based entirely on the `Turn` (`events` + `usage`) your adapter returns from `send`. The moment `send` is wired up, the full assertion set is available, and none of it involves OTel (see [Connect Your Agent](/guides/connect-your-agent)).
-
-What OTel integration buys you is something else: **the call waterfall in `niceeval view`**. Every model call and tool execution inside your app, each with its duration and tokens, laid out as a per-turn timeline — why a failing eval failed, or which step made a slow turn slow, is often obvious at a glance in the waterfall.
-
-If your app already emits OTel traces — AI SDK telemetry, LangGraph's LangSmith export, OpenLLMetry / OpenInference auto-instrumentation, or your own spans following the GenAI semantic conventions — you are already producing the waterfall data: just have the app send a copy of its spans to NiceEval. Application code stays untouched, and the integration remains non-invasive (see [Tier](/concepts/tier)).
-
-## How it works (one paragraph)
-
-At run time NiceEval starts a local OTLP receiver, and your app sends spans to it. Spans received during each `send` are attributed to that turn, normalized into GenAI semantics, and written into `EvalResult.trace`; after the run, `npx niceeval view` is the waterfall. **Spans only feed the waterfall — never the event stream, never assertions.** Missing instrumentation, late spans, or dropped batches only degrade waterfall completeness; they never affect any verdict.
-
-## Wiring it up
-
-**1. Adapter side** — write `send` as usual (the event mapping is still your mapping), plus one extra line: forward this turn's `traceparent` with the request:
-
-```ts
-// agents/support-bot.ts
-import { defineAgent } from "niceeval/adapter";
-
-export default defineAgent({
-  name: "support-bot",
-  async send(input, ctx) {
-    const r = await fetch("http://localhost:5188/chat", {
-      method: "POST",
-      // Forward traceparent with the request: this turn's spans attach to NiceEval's trace,
-      // which is what makes attribution precise under concurrency
-      headers: { "content-type": "application/json", ...ctx.telemetry?.headers },
-      body: JSON.stringify({ message: input.text }),
-      signal: ctx.signal,
-    });
-    const body = await r.json();
-    return {
-      status: r.ok ? "completed" : "failed",
-      events: mapToEvents(body),   // Assertions are based on this, same as without OTel
-    };
-  },
-});
-```
-
-Built-ins do not need this step: `uiMessageStreamAgent` always merges `ctx.telemetry.headers` into the request headers automatically.
-
-**2. How the endpoint reaches your app** — NiceEval's receiver endpoint is **startup-time configuration, never passed through send** (standard OTel SDKs cannot switch endpoints at run time either: `OTEL_*` environment variables are read once at process startup). Pick by deployment shape:
-
-- **Your own long-running service (most common)**: use **fixed-port mode** — pin the receiver port in `niceeval.config.ts`. Writing this config is what turns OTel integration on:
-
-  ```ts
-  // niceeval.config.ts
-  export default defineConfig({
-    telemetry: { port: 4318 },   // Receiver always listens on http://localhost:4318/v1/traces
-  });
-  ```
-
-  Configure `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces` once when the service starts; it stays valid no matter how many eval runs follow. The trade-off: sharing a port means only one niceeval process can run on the machine at a time; OTel Collector fan-out setups point at this fixed endpoint the same way. If `port` is already taken by another process, NiceEval fails immediately with an error telling you to pick a free port — it never fails silently.
-
-  The receiver hostname reported to your app defaults to `127.0.0.1`. When Docker-based sandbox tracing needs `host.docker.internal`, or a tunneled remote setup needs another hostname, override it with `host`: `telemetry: { host: "host.docker.internal", port: 4318 }`. These two fields are the only way to configure the OTLP receiver in NiceEval; it does not read environment variables.
-
-- **Child processes / processes NiceEval spawns** (CLI-style agents): nothing to do. `ctx.telemetry.env` (standard `OTEL_*` environment variables, ready to spread) is injected into the process environment; each run is a fresh process and reads the fresh endpoint.
-
-**3. App side** — a few lines of configuration, depending on your instrumentation ecosystem:
-
-<Tabs>
-  <Tab title="AI SDK">
-    The official OTel integration (`@ai-sdk/otel`, which emits standard GenAI semantics) is recommended; the older `experimental_telemetry` (`ai.*`) renders too:
-
-    ```ts
-    import { generateText } from "ai";
-
-    const result = await generateText({
-      model, tools, messages,
-      experimental_telemetry: { isEnabled: true },
-    });
-    ```
-
-    The exporter is the standard OTel Node SDK, with the endpoint pointed at NiceEval (the injected env or the fixed port). Runnable examples: app-side instrumentation in [`examples/zh/origin/ai-sdk-v7`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/origin/ai-sdk-v7) (`src/backend/otel.ts`, official `@ai-sdk/otel`); the full eval project after integration in [`examples/zh/tier1/ai-sdk-v7`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/tier1/ai-sdk-v7).
-  </Tab>
-  <Tab title="LangGraph / LangChain">
-    The zero-dependency route: three environment variables:
-
-    ```bash
-    LANGSMITH_TRACING=true \
-    LANGSMITH_OTEL_ENABLED=true \
-    LANGSMITH_OTEL_ONLY=true \
-    OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces \
-    node server.js   # Endpoint value per "How the endpoint reaches your app": injected env or fixed port
-    ```
-
-    `LANGSMITH_OTEL_ONLY` means OTLP only, no LangSmith cloud (drop it to dual-send). This is truly zero-code with the **Python** `langsmith` SDK (it hooks OTel automatically at import time); the **JS** version (`langsmith@0.7.x`) additionally needs one explicit call to `initializeOTEL()` (from `langsmith/experimental/otel/setup`), otherwise it only logs a warning and produces no spans — the three environment variables themselves stay the same. Runnable example (Python backend, zero wiring code): [`examples/zh/origin/langgraph`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/origin/langgraph); the full eval project after integration in [`examples/zh/tier1/langgraph`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/tier1/langgraph).
-  </Tab>
-  <Tab title="OpenLLMetry">
-    ```ts
-    import * as traceloop from "@traceloop/node-server-sdk";
-
-    traceloop.initialize({ disableBatch: true });
-    // Endpoint via the standard OTEL_EXPORTER_OTLP_ENDPOINT environment variable
-    ```
-  </Tab>
-  <Tab title="OpenInference">
-    ```python
-    from openinference.instrumentation.langchain import LangChainInstrumentor
-    from phoenix.otel import register
-
-    register()  # Or the standard OTel SDK; endpoint via OTEL_EXPORTER_OTLP_ENDPOINT
-    LangChainInstrumentor().instrument()
-    ```
-  </Tab>
-  <Tab title="Hand-rolled gen_ai">
-    Instrument per the [OTel GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/): name model-call spans `chat {model}` and tool spans `execute_tool {tool}`, with the `gen_ai.operation.name` and `gen_ai.tool.name` / `gen_ai.tool.call.id` attributes. This is the target semantics NiceEval normalizes into, and it produces the most complete waterfall.
-  </Tab>
-</Tabs>
-
-## How spans are attributed to turns
-
-When evals run in parallel, one receiver receives spans from multiple sessions at once. NiceEval attributes them to their turns via two paths:
-
-- **traceparent (recommended, concurrency-safe)**: when `send` makes its request, spread `ctx.telemetry.headers` (W3C trace context, a fresh `traceparent` per turn) into the request headers. If the app's instrumentation supports context propagation (standard OTel HTTP server instrumentation does), this turn's spans automatically attach under the trace NiceEval provided, and attribution is exact by traceId.
-- **Time window (fallback)**: when the app does not propagate trace context, spans are attributed by the time window around `send`. The window is only reliable when turns run serially, so in this case NiceEval executes this agent's turns serially and says so in the logs — it never silently mixes streams. Once traceparent is confirmed working, concurrency resumes automatically.
-
-Export promptly on the app side: what the waterfall cares about is "this turn's spans arrive in time". Use `SimpleSpanProcessor` (or flush every turn) — `BatchSpanProcessor`'s buffering makes spans arrive late across turns; when the waterfall occasionally has a missing tail, that is usually why.
-
-## Already have your own OTel backend? Dual-export, no switch
-
-Your app is most likely already sending traces to its own observability backend (Langfuse / SigNoz / a production collector). Connecting NiceEval **requires neither switching backends nor a second layer of instrumentation**: a TracerProvider supports multiple SpanProcessors — the same spans, two exits:
-
-```ts
-// instrumentation.ts — initialize once at app startup: one set of instrumentation, two exits
-import { NodeTracerProvider, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-node";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-
-export const provider = new NodeTracerProvider({
-  spanProcessors: [
-    // Exit 1: your own backend, always on
-    new SimpleSpanProcessor(new OTLPTraceExporter({ url: process.env.MY_COLLECTOR_URL })),
-    // Exit 2: NiceEval. Without this variable in production, this exit simply
-    // does not exist — same code in production and in evals
-    ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT
-      ? [new SimpleSpanProcessor(new OTLPTraceExporter({ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT }))]
-      : []),
-  ],
-});
-provider.register();
-```
-
-If touching app code is inconvenient, fan out through an OTel Collector — the app sends only to the collector, and the collector configures two exporters (your backend + NiceEval's fixed endpoint). The cost is one more component to operate.
-
-## How accurate is the waterfall: what you map is what lights up
-
-Before drawing the waterfall, NiceEval normalizes every span into GenAI semantics. The verdict looks at exactly two things: `gen_ai.operation.name` (the standard operation name) and the normalized `kind` (semantic role). The mapping table — what you provide on a span, and what lights up in the waterfall:
-
-| What you provide on the span | What lights up in the waterfall |
-|---|---|
-| `gen_ai.operation.name: "chat"` (or `text_completion` / `embeddings`) | Colored and grouped as a **model call** |
-| `gen_ai.operation.name: "execute_tool"`, or a `tool_name` attribute | Colored and grouped as a **tool execution** |
-| `gen_ai.operation.name: "invoke_agent"` / `"create_agent"` | Colored and grouped as a **sub-agent** |
-| A `call_id` attribute equal to a `callId` in your send events | That tool span gets the real input/output backfilled (`io.input` / `io.output`, joined by callId from the events `send` returned — many instrumentations do not capture content by default, and this path is unaffected) |
-| Span status = error | Marked red |
-| Nothing recognizable (normalized to `other`) | The timeline is still there; in large traces (>150 spans) it is folded away as internal noise |
-| Any other attributes | Preserved as-is, drillable in the view |
-
-If your app instruments directly per the GenAI semconv (the "Hand-rolled gen_ai" tab above), all of this holds automatically; the common shapes of mainstream formats (AI SDK, LangSmith, OpenLLMetry / OpenInference) are also within the generic fallback's recognition range.
-
-### Private instrumentation: write your own mapping
-
-When your instrumentation has an app-private shape the generic fallback cannot recognize, there are two routes:
-
-- **Fix the instrumentation (recommended)**: add a `gen_ai.operation.name` attribute to the spans on the app side — a one-line change, and your own observability backend benefits equally.
-- **Write a `spanMapper`**: when touching the app is inconvenient, declare a pure function on the agent that translates private spans into the semantics of the table above before rendering. Both `tagSpan` (writes the verdict back onto the span; existing attributes are only added to, never changed) and `heuristicTag` (the generic fallback verdict) are exported from `niceeval/adapter`:
-
-```ts
-import { defineAgent, tagSpan, heuristicTag } from "niceeval/adapter";
-import type { TraceSpan } from "niceeval";
-
-function mapMySpans(spans: TraceSpan[]): TraceSpan[] {
-  return spans.map((span) => {
-    // Private naming → standard semantics: op goes into gen_ai.operation.name, kind sets the coloring
-    if (span.name === "my.llm.request") return tagSpan(span, { op: "chat", kind: "model" });
-    if (span.name === "my.tool.exec") {
-      // Copy the private call id into call_id while you're at it, so tool I/O joins in from the send events
-      const attributes = { ...span.attributes, call_id: String(span.attributes?.["my.callId"] ?? "") };
-      return tagSpan({ ...span, attributes }, { op: "execute_tool", kind: "tool" });
-    }
-    return tagSpan(span, heuristicTag(span));   // leave the rest to the generic fallback
-  });
-}
-
-export default defineAgent({
-  name: "my-bot",
-  spanMapper: mapMySpans,
-  async send(input, ctx) { /* same wiring as above */ },
-});
-```
-
-The built-in `mapCodexSpans` (also exported from `niceeval/adapter`) is a ready-made spanMapper implementation to use as a reference. A spanMapper is pure observability code: get it wrong and at worst the waterfall coloring is off — assertions are never affected.
-
-## Boundaries
-
-- **Everything assertion-related lives in send**. To assert on tool calls, map them into `events` (an official converter or a hand-written mapping, see [Write send](/guides/write-send)); to assert on usage, include it in the `send` return. There is no path where "the span has it, events do not, so the assertion reads the span".
-- **Multi-turn sessions and HITL are not span concerns**. Spans have no "waiting for human input" semantics, and session continuation is application-protocol work — both keep happening in `send` as usual (session continuation in [Write send](/guides/write-send), the HITL concept in [HITL](/concepts/hitl)).
-- **You get a warning when no spans arrive**. A turn with zero spans usually means the endpoint is not hooked up (env not injected, service not restarted); NiceEval says so in the logs. The waterfall is empty; assertions judge as usual.
-
-## Related reading
-
-- [Connect Your Agent](/guides/connect-your-agent) — where assertions come from: the event mapping in `send`.
-- [Write send](/guides/write-send) — the full tutorial for hand-writing an adapter; step six is this page's adapter-side wiring.
-- [Events reference](/reference/events) — what the events assertions consume look like.