niceeval 0.4.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs-site/zh/concepts/adapter.mdx +6 -6
- package/docs-site/zh/concepts/experiment.mdx +7 -7
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +63 -20
- package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
- package/docs-site/zh/guides/custom-reports.mdx +243 -0
- package/docs-site/zh/guides/experiments.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +252 -0
- package/docs-site/zh/guides/reporters.mdx +1 -1
- package/docs-site/zh/guides/results-data.mdx +224 -0
- package/docs-site/zh/guides/runner.mdx +1 -1
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/viewing-results.mdx +147 -77
- package/docs-site/zh/guides/write-experiment.mdx +9 -9
- package/docs-site/zh/guides/write-send.mdx +6 -6
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/introduction.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +10 -2
- package/docs-site/zh/reference/define-agent.mdx +5 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/package.json +3 -2
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/bub.ts +14 -4
- package/src/agents/claude-code.ts +1 -1
- package/src/agents/codex.ts +2 -2
- package/src/agents/streaming.test.ts +1 -1
- package/src/agents/types.ts +4 -4
- package/src/agents/ui-message-stream.test.ts +1 -1
- package/src/cli.ts +104 -14
- package/src/context/context.test.ts +1 -1
- package/src/context/context.ts +3 -3
- package/src/context/session.ts +2 -2
- package/src/context/types.ts +2 -2
- package/src/i18n/en.ts +29 -7
- package/src/i18n/zh-CN.ts +24 -2
- package/src/report/aggregate.ts +79 -50
- package/src/report/components.tsx +190 -0
- package/src/report/compute.ts +149 -84
- package/src/report/default-report.tsx +222 -0
- package/src/report/dual-face.test.tsx +527 -0
- package/src/report/format.ts +29 -0
- package/src/report/index.ts +79 -24
- package/src/report/load.ts +67 -0
- package/src/report/metrics.ts +2 -1
- package/src/report/param.ts +18 -0
- package/src/report/primitives.tsx +117 -0
- package/src/report/react/CaseList.tsx +2 -2
- package/src/report/react/DeltaTable.tsx +2 -2
- package/src/report/react/MetricBars.tsx +109 -0
- package/src/report/react/MetricLine.tsx +200 -0
- package/src/report/react/MetricMatrix.tsx +1 -1
- package/src/report/react/MetricScatter.tsx +1 -1
- package/src/report/react/MetricTable.tsx +1 -1
- package/src/report/react/RunOverview.tsx +4 -4
- package/src/report/react/Scoreboard.tsx +3 -3
- package/src/report/react/cell.tsx +1 -1
- package/src/report/react/fixtures.ts +98 -36
- package/src/report/react/format.ts +3 -28
- package/src/report/react/index.tsx +41 -24
- package/src/report/react/render.test.tsx +8 -8
- package/src/report/react/styles.css +77 -0
- package/src/report/report.test.ts +248 -128
- package/src/report/report.ts +69 -0
- package/src/report/text/faces.ts +356 -0
- package/src/report/text/layout.ts +125 -0
- package/src/report/text/plot.ts +127 -0
- package/src/report/tree.ts +220 -0
- package/src/report/types.ts +75 -27
- package/src/report/web.ts +40 -0
- package/src/results/copy.ts +95 -41
- package/src/results/format.ts +12 -11
- package/src/results/index.ts +34 -17
- package/src/results/open.ts +201 -81
- package/src/results/results.test.ts +595 -191
- package/src/results/select.ts +107 -55
- package/src/results/types.ts +137 -45
- package/src/results/writer.ts +258 -0
- package/src/runner/attempt.ts +3 -3
- package/src/runner/fingerprint.ts +1 -1
- package/src/runner/reporters/artifacts.ts +38 -78
- package/src/runner/reporters/braintrust.test.ts +2 -2
- package/src/runner/reporters/braintrust.ts +3 -3
- package/src/runner/run.ts +1 -1
- package/src/runner/types.ts +18 -12
- package/src/show/compose.ts +215 -0
- package/src/show/index.ts +263 -0
- package/src/show/render.ts +456 -0
- package/src/show/show.test.ts +505 -0
- package/src/view/app/App.tsx +85 -24
- package/src/view/app/components/CostScoreChart.tsx +29 -10
- package/src/view/app/components/ExperimentTable.tsx +27 -9
- package/src/view/app/components/GroupSelector.tsx +1 -1
- package/src/view/app/i18n.ts +18 -3
- package/src/view/app/lib/attempt-route.test.ts +9 -9
- package/src/view/app/lib/attempt-route.ts +5 -5
- package/src/view/app/lib/outcome.ts +1 -3
- package/src/view/app/lib/rows.ts +95 -15
- package/src/view/app/main.tsx +18 -6
- package/src/view/app/pages/RunsPage.tsx +4 -7
- package/src/view/app/pages/TracesPage.tsx +3 -8
- package/src/view/app/types.ts +50 -6
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.test.ts +254 -0
- package/src/view/data.ts +336 -0
- package/src/view/index.ts +64 -10
- package/src/view/server.ts +31 -9
- package/src/view/shared/types.ts +47 -41
- package/src/view/styles.css +5 -0
- package/src/view/template.html +1 -0
- package/src/view/view-report.test.ts +308 -0
- package/docs-site/.mintignore +0 -13
- package/docs-site/AGENTS.md +0 -46
- package/docs-site/concepts/adapter.mdx +0 -287
- package/docs-site/concepts/assert.mdx +0 -243
- package/docs-site/concepts/drive.mdx +0 -118
- package/docs-site/concepts/evals.mdx +0 -108
- package/docs-site/concepts/experiment.mdx +0 -37
- package/docs-site/concepts/hitl.mdx +0 -83
- package/docs-site/concepts/judge.mdx +0 -129
- package/docs-site/concepts/overview.mdx +0 -98
- package/docs-site/concepts/tier.mdx +0 -42
- package/docs-site/docs-ref/00-index.md +0 -23
- package/docs-site/docs-ref/01-page-types.md +0 -43
- package/docs-site/docs-ref/03-style-rules.md +0 -45
- package/docs-site/docs-ref/04-code-examples.md +0 -37
- package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
- package/docs-site/docs-ref/06-checklists.md +0 -53
- package/docs-site/docs.json +0 -256
- package/docs-site/example/ai-agent-application.mdx +0 -148
- package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
- package/docs-site/example/claude-code-codex-skill.mdx +0 -147
- package/docs-site/example/showcase.mdx +0 -39
- package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
- package/docs-site/example/tier1-claude-sdk.mdx +0 -119
- package/docs-site/example/tier1-codex-sdk.mdx +0 -111
- package/docs-site/example/tier1-langgraph.mdx +0 -119
- package/docs-site/example/tier1-pi-sdk.mdx +0 -119
- package/docs-site/favicon.svg +0 -5
- package/docs-site/github-diff.css +0 -135
- package/docs-site/github-diff.js +0 -11
- package/docs-site/guides/authoring.mdx +0 -129
- package/docs-site/guides/ci-integration.mdx +0 -97
- package/docs-site/guides/connect-otel.mdx +0 -209
- package/docs-site/guides/connect-your-agent.mdx +0 -221
- package/docs-site/guides/dataset-fanout.mdx +0 -98
- package/docs-site/guides/experiments.mdx +0 -71
- package/docs-site/guides/fixtures.mdx +0 -147
- package/docs-site/guides/reporters.mdx +0 -113
- package/docs-site/guides/runner.mdx +0 -79
- package/docs-site/guides/sandbox-agent.mdx +0 -138
- package/docs-site/guides/sandbox-backends.mdx +0 -64
- package/docs-site/guides/scoring-guide.mdx +0 -92
- package/docs-site/guides/viewing-results.mdx +0 -195
- package/docs-site/guides/write-experiment.mdx +0 -81
- package/docs-site/guides/write-send.mdx +0 -347
- package/docs-site/index.mdx +0 -181
- package/docs-site/introduction.mdx +0 -141
- package/docs-site/quickstart.mdx +0 -136
- package/docs-site/reference/builtin-agents.mdx +0 -161
- package/docs-site/reference/capabilities.mdx +0 -76
- package/docs-site/reference/cli.mdx +0 -120
- package/docs-site/reference/define-agent.mdx +0 -168
- package/docs-site/reference/define-config.mdx +0 -41
- package/docs-site/reference/define-eval.mdx +0 -160
- package/docs-site/reference/events.mdx +0 -131
- package/docs-site/reference/expect.mdx +0 -112
- package/docs-site/tracker.js +0 -8
- package/src/report/react/data.ts +0 -17
- package/src/view/aggregate.ts +0 -103
- package/src/view/loader.test.ts +0 -61
- package/src/view/loader.ts +0 -257
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
title: "Integrate Claude Agent SDK with NiceEval non-invasively"
|
|
3
|
-
sidebarTitle: "Claude Agent SDK integration"
|
|
4
|
-
description: "The complete before/after diff for connecting a Claude Agent SDK assistant backend to NiceEval: zero app-side code changes; everything is added on the eval side."
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
{/* Generated by scripts/gen-diff-code.ts (pnpm run gen:diff-code). Do not edit by hand. */}
|
|
8
|
-
|
|
9
|
-
Comparison targets:
|
|
10
|
-
|
|
11
|
-
- **before**:[https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/origin/claude-sdk](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/origin/claude-sdk) -- a standalone `@anthropic-ai/claude-agent-sdk` HTTP service, before any eval integration.
|
|
12
|
-
- **after**:[https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/tier1/claude-sdk](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/tier1/claude-sdk) -- the same app after integrating NiceEval.
|
|
13
|
-
|
|
14
|
-
**Integration path**: official converter -- mapping Claude Agent SDK native `SDKMessage` events into standard events is handled by `fromClaudeSdkMessages` (exported from `"niceeval/adapter"`). The adapter only keeps transport glue and HITL stop-turn detection (`calculate` is gated through the official `canUseTool` callback). You start the app yourself in its normal way (`pnpm start`); evals do not manage the process. App-side `src/backend/*` files are byte-for-byte unchanged.
|
|
15
|
-
|
|
16
|
-
All integration code changes (measured from both directories when generated):
|
|
17
|
-
|
|
18
|
-
<table className="gd-summary"><tbody><tr><th>{"Category"}</th><th>{"Files"}</th><th>{"Lines"}</th></tr><tr><td>{"App-side config (required: dependency declarations)"}</td><td>{"3"}</td><td>{"+6 −1"}</td></tr><tr><td>{"adapter (required: transport glue; protocol mapping is in the official package)"}</td><td>{"2"}</td><td>{"+124"}</td></tr><tr><td>{"evals and experiments (test content, grows as needed)"}</td><td>{"6"}</td><td>{"+120"}</td></tr><tr className="gd-total"><td>{"Total"}</td><td>{"11"}</td><td>{"+250 −1"}</td></tr></tbody></table>
|
|
19
|
-
|
|
20
|
-
## File list
|
|
21
|
-
|
|
22
|
-
```text
|
|
23
|
-
claude-sdk/
|
|
24
|
-
├── package.json modified
|
|
25
|
-
├── tsconfig.json modified
|
|
26
|
-
├── pnpm-workspace.yaml modified
|
|
27
|
-
├── niceeval.config.ts added
|
|
28
|
-
├── agents/
|
|
29
|
-
│ └── claude-sdk.ts added
|
|
30
|
-
├── evals/
|
|
31
|
-
│ ├── basic-qa.eval.ts added
|
|
32
|
-
│ ├── hitl-approve.eval.ts added
|
|
33
|
-
│ ├── hitl-deny.eval.ts added
|
|
34
|
-
│ ├── session-isolation.eval.ts added
|
|
35
|
-
│ └── weather-tool.eval.ts added
|
|
36
|
-
└── experiments/
|
|
37
|
-
└── assistant.ts added
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
## App-side changes (dependency declarations only)
|
|
41
|
-
|
|
42
|
-
<div className="gd-file">
|
|
43
|
-
<div className="gd-head"><span className="gd-name">{"package.json"}</span><span className="gd-stats"><span className="gd-plus">{"+2"}</span></span></div>
|
|
44
|
-
<div className="gd-body">
|
|
45
|
-
<table className="gd-table"><tbody><tr className="gd-expand" data-fold="gdf3"><td className="gd-ln" colSpan={2}>{"⇕"}</td><td className="gd-sign"></td><td className="gd-code">{"Expand 8 unchanged lines"}</td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"1"}</td><td className="gd-ln">{"1"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{"{"}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"2"}</td><td className="gd-ln">{"2"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"name\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"claude-sdk-example\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"3"}</td><td className="gd-ln">{"3"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"version\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"0.1.0\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"4"}</td><td className="gd-ln">{"4"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"private\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"5"}</td><td className="gd-ln">{"5"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"type\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"module\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"6"}</td><td className="gd-ln">{"6"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"scripts\""}</span><span className="gdt0">{": {"}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"7"}</td><td className="gd-ln">{"7"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"dev\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"lsof -ti tcp:32001 | xargs kill -9 2>/dev/null; concurrently -k -n server,web -c blue,green "}</span><span className="gdt1">{"\\\""}</span><span className="gdt2">{"pnpm dev:server"}</span><span className="gdt1">{"\\\""}</span><span className="gdt2">{" "}</span><span className="gdt1">{"\\\""}</span><span className="gdt2">{"pnpm dev:web"}</span><span className="gdt1">{"\\\""}</span><span className="gdt2">{"\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf3"><td className="gd-ln">{"8"}</td><td className="gd-ln">{"8"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"dev:server\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"node --env-file .env --import tsx/esm --watch src/backend/server.ts\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"9"}</td><td className="gd-ln">{"9"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"dev:web\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"wait-on -t 30000 tcp:127.0.0.1:32001 && vite\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"10"}</td><td className="gd-ln">{"10"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"build\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"vite build\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"11"}</td><td className="gd-ln">{"11"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"start\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"node --env-file .env --import tsx/esm src/backend/server.ts\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"eval\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"12"}</td><td className="gd-ln">{"13"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"typecheck\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"tsc --noEmit\""}</span></td></tr><tr><td className="gd-ln">{"13"}</td><td className="gd-ln">{"14"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr><td className="gd-ln">{"14"}</td><td className="gd-ln">{"15"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"dependencies\""}</span><span className="gdt0">{": {"}</span></td></tr><tr className="gd-expand" data-fold="gdf4"><td className="gd-ln" colSpan={2}>{"⇕"}</td><td className="gd-sign"></td><td className="gd-code">{"Expand 9 unchanged lines"}</td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"15"}</td><td className="gd-ln">{"16"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"@anthropic-ai/claude-agent-sdk\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^0.3.198\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"16"}</td><td className="gd-ln">{"17"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"@assistant-ui/react\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^0.14.24\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"17"}</td><td className="gd-ln">{"18"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"react\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^19.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"18"}</td><td className="gd-ln">{"19"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"react-dom\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^19.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"19"}</td><td className="gd-ln">{"20"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"zod\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^4.1.8\""}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"20"}</td><td className="gd-ln">{"21"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"21"}</td><td className="gd-ln">{"22"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"devDependencies\""}</span><span className="gdt0">{": {"}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"22"}</td><td className="gd-ln">{"23"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"@types/node\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^22.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf4"><td className="gd-ln">{"23"}</td><td className="gd-ln">{"24"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"@types/react\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^19.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"24"}</td><td className="gd-ln">{"25"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"@types/react-dom\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^19.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"25"}</td><td className="gd-ln">{"26"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"@vitejs/plugin-react\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^5.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"26"}</td><td className="gd-ln">{"27"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"concurrently\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^9.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"28"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"niceeval\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"file:../../../..\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"27"}</td><td className="gd-ln">{"29"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"tsx\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^4.19.2\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"28"}</td><td className="gd-ln">{"30"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"typescript\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^5.7.2\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"29"}</td><td className="gd-ln">{"31"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"vite\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^7.0.0\""}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"30"}</td><td className="gd-ln">{"32"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"wait-on\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"^9.0.0\""}</span></td></tr><tr><td className="gd-ln">{"31"}</td><td className="gd-ln">{"33"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr><td className="gd-ln">{"32"}</td><td className="gd-ln">{"34"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr></tbody></table>
|
|
46
|
-
</div>
|
|
47
|
-
</div>
|
|
48
|
-
|
|
49
|
-
<div className="gd-file">
|
|
50
|
-
<div className="gd-head"><span className="gd-name">{"tsconfig.json"}</span><span className="gd-stats"><span className="gd-plus">{"+1"}</span><span className="gd-minus">{"−1"}</span></span></div>
|
|
51
|
-
<div className="gd-body">
|
|
52
|
-
<table className="gd-table"><tbody><tr className="gd-expand" data-fold="gdf5"><td className="gd-ln" colSpan={2}>{"⇕"}</td><td className="gd-sign"></td><td className="gd-code">{"Expand 13 unchanged lines"}</td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"1"}</td><td className="gd-ln">{"1"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{"{"}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"2"}</td><td className="gd-ln">{"2"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"compilerOptions\""}</span><span className="gdt0">{": {"}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"3"}</td><td className="gd-ln">{"3"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"target\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"ES2022\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"4"}</td><td className="gd-ln">{"4"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"module\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"NodeNext\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"5"}</td><td className="gd-ln">{"5"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"moduleResolution\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"NodeNext\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"6"}</td><td className="gd-ln">{"6"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"lib\""}</span><span className="gdt0">{": ["}</span><span className="gdt2">{"\"ES2023\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"DOM\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"DOM.Iterable\""}</span><span className="gdt0">{"],"}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"7"}</td><td className="gd-ln">{"7"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"types\""}</span><span className="gdt0">{": ["}</span><span className="gdt2">{"\"node\""}</span><span className="gdt0">{"],"}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"8"}</td><td className="gd-ln">{"8"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"jsx\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"react-jsx\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"9"}</td><td className="gd-ln">{"9"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"strict\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"10"}</td><td className="gd-ln">{"10"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"noUncheckedIndexedAccess\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"false"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"11"}</td><td className="gd-ln">{"11"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"esModuleInterop\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"12"}</td><td className="gd-ln">{"12"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"skipLibCheck\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-fold gdf5"><td className="gd-ln">{"13"}</td><td className="gd-ln">{"13"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"allowImportingTsExtensions\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"14"}</td><td className="gd-ln">{"14"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"noEmit\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span><span className="gdt0">{","}</span></td></tr><tr><td className="gd-ln">{"15"}</td><td className="gd-ln">{"15"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"forceConsistentCasingInFileNames\""}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span></td></tr><tr><td className="gd-ln">{"16"}</td><td className="gd-ln">{"16"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-del"><td className="gd-ln">{"17"}</td><td className="gd-ln"></td><td className="gd-sign">{"−"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"include\""}</span><span className="gdt0">{": ["}</span><span className="gdt2">{"\"src\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"vite.config.ts\""}</span><span className="gdt0">{"]"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt1">{"\"include\""}</span><span className="gdt0">{": ["}</span><span className="gdt2">{"\"src\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"vite.config.ts\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"niceeval.config.ts\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"agents\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"evals\""}</span><span className="gdt0">{", "}</span><span className="gdt2">{"\"experiments\""}</span><span className="gdt0">{"]"}</span></td></tr><tr><td className="gd-ln">{"18"}</td><td className="gd-ln">{"18"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr></tbody></table>
|
|
53
|
-
</div>
|
|
54
|
-
</div>
|
|
55
|
-
|
|
56
|
-
<div className="gd-file">
|
|
57
|
-
<div className="gd-head"><span className="gd-name">{"pnpm-workspace.yaml"}</span><span className="gd-stats"><span className="gd-plus">{"+3"}</span></span></div>
|
|
58
|
-
<div className="gd-body">
|
|
59
|
-
<table className="gd-table"><tbody><tr><td className="gd-ln">{"1"}</td><td className="gd-ln">{"1"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt3">{"packages"}</span><span className="gdt0">{": []"}</span></td></tr><tr><td className="gd-ln">{"2"}</td><td className="gd-ln">{"2"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt3">{"allowBuilds"}</span><span className="gdt0">{":"}</span></td></tr><tr><td className="gd-ln">{"3"}</td><td className="gd-ln">{"3"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt3">{"esbuild"}</span><span className="gdt0">{": "}</span><span className="gdt1">{"true"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt3">{"overrides"}</span><span className="gdt0">{":"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt3">{"niceeval"}</span><span className="gdt0">{": "}</span><span className="gdt2">{"link:../../../.."}</span></td></tr><tr><td className="gd-ln">{"4"}</td><td className="gd-ln">{"6"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt3">{"minimumReleaseAgeExclude"}</span><span className="gdt0">{":"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"niceeval@0.1.1"}</span></td></tr><tr><td className="gd-ln">{"5"}</td><td className="gd-ln">{"8"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.198'"}</span></td></tr><tr><td className="gd-ln">{"6"}</td><td className="gd-ln">{"9"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.198'"}</span></td></tr><tr><td className="gd-ln">{"7"}</td><td className="gd-ln">{"10"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.198'"}</span></td></tr><tr className="gd-expand" data-fold="gdf6"><td className="gd-ln" colSpan={2}>{"⇕"}</td><td className="gd-sign"></td><td className="gd-code">{"Expand 6 unchanged lines"}</td></tr><tr className="gd-fold gdf6"><td className="gd-ln">{"8"}</td><td className="gd-ln">{"11"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.198'"}</span></td></tr><tr className="gd-fold gdf6"><td className="gd-ln">{"9"}</td><td className="gd-ln">{"12"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.198'"}</span></td></tr><tr className="gd-fold gdf6"><td className="gd-ln">{"10"}</td><td className="gd-ln">{"13"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-linux-x64@0.3.198'"}</span></td></tr><tr className="gd-fold gdf6"><td className="gd-ln">{"11"}</td><td className="gd-ln">{"14"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.198'"}</span></td></tr><tr className="gd-fold gdf6"><td className="gd-ln">{"12"}</td><td className="gd-ln">{"15"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk-win32-x64@0.3.198'"}</span></td></tr><tr className="gd-fold gdf6"><td className="gd-ln">{"13"}</td><td className="gd-ln">{"16"}</td><td className="gd-sign"></td><td className="gd-code"><span className="gdt0">{" - "}</span><span className="gdt2">{"'@anthropic-ai/claude-agent-sdk@0.3.198'"}</span></td></tr></tbody></table>
|
|
60
|
-
</div>
|
|
61
|
-
</div>
|
|
62
|
-
|
|
63
|
-
## Added adapter, evals, and experiments
|
|
64
|
-
|
|
65
|
-
<div className="gd-file">
|
|
66
|
-
<div className="gd-head"><span className="gd-name">{"niceeval.config.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+12"}</span></span></div>
|
|
67
|
-
<div className="gd-body">
|
|
68
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineConfig } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineConfig"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" name: { "}</span><span className="gdt2">{"\"zh-CN\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"Claude Agent SDK 示例\""}</span><span className="gdt0">{", en: "}</span><span className="gdt2">{"\"Claude Agent SDK example\""}</span><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" judge: { model: "}</span><span className="gdt2">{"\"gpt-5.4\""}</span><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" timeoutMs: "}</span><span className="gdt1">{"120_000"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// 每个 attempt 都要经一个真实子进程(server.ts)+ Claude Code CLI 子进程 + 网络调用。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// 钉死串行(不是\"偏保守\",是必须):同一 model 分桶下两个并发的"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// HITL 审批打到同一个 server 实例时,POST /api/chat/approve 会对其中一个 toolUseId"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// 永久 404(canUseTool 的 resolver 没注册成功),见 memory/claude-sdk-concurrent-hitl-approve-race.md。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" maxConcurrency: "}</span><span className="gdt1">{"1"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
69
|
-
</div>
|
|
70
|
-
</div>
|
|
71
|
-
|
|
72
|
-
<div className="gd-file">
|
|
73
|
-
<div className="gd-head"><span className="gd-name">{"agents/claude-sdk.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+112"}</span></span></div>
|
|
74
|
-
<div className="gd-body">
|
|
75
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// claude-sdk 的 adapter:无侵入对接一个**已经在跑**的应用(../src/backend/server.ts,原生"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// `SDKMessage` 流原样透传成 SSE,外加自定义 { type: \"server_error\" } 传输帧)。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"//"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// `SDKMessage` → 标准事件的映射是官方转换器 `fromClaudeSdkMessages`(`\"niceeval/adapter\"`"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 导出)的事;逐帧驱动也是官方件(`driveFrameStream`)。HITL 停轮现场(读了一半的流)和会话"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// id 续接都不需要自己声明状态槽——挂在 `ctx.session` 上,取用即可:`ctx.session.hold` /"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// `ctx.session.take` 存取停轮现场,`ctx.session.id` / `ctx.session.capture` 续接会话。这里"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 只剩传输粘合:端点在哪、审批打哪个端点、HITL 停轮怎么判。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 无 OTel(CLI 原生遥测只有 metrics+logs,niceeval 不消费),事件全部来自转换器。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"//"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// HITL 没有显式的\"等审批\"帧——`canUseTool` 把流卡在一个 Promise 上,客户端只能从"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// \"gated 工具的 tool_use 到了、之后没动静\"推断。Tier 1 的确定性做法:被门控的工具就"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"13"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// mcp__demo-tools__calculate 一个(应用 agent.ts 里的 GATED_TOOL_NAME,这里必须写死同一个"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"14"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 字符串),`driveFrameStream` 的 onFrame 钩子扫 derived 事件认出它就返回 `{ pause }`;"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"15"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 下一轮先打 /api/chat/approve 再继续读同一条流。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"16"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineAgent, sseJsonFrames, fromClaudeSdkMessages, driveFrameStream } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval/adapter\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"type"}</span><span className="gdt0">{" { AgentContext, ClaudeSdkStream, SseFrameCursor } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval/adapter\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"18"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"type"}</span><span className="gdt0">{" { Turn, TurnInput } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"19"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"type"}</span><span className="gdt0">{" { SDKMessage } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"@anthropic-ai/claude-agent-sdk\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"20"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"21"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 必须和 ../src/backend/agent.ts 里的 GATED_TOOL_NAME 完全一致(MCP 命名空间下的真实工具名)。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"22"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"GATED_TOOL_NAME"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"mcp__demo-tools__calculate\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"23"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"24"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 被测应用由你自己按它的方式启动(pnpm start / 部署在哪都行),eval 不代管进程、不另开端口。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"25"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"BASE_URL"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" process.env."}</span><span className="gdt1">{"CLAUDE_SDK_URL"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"??"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"http://127.0.0.1:32001\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"26"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"27"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"type"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"TransportFrame"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" { "}</span><span className="gdt7">{"type"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"server_error\""}</span><span className="gdt0">{"; "}</span><span className="gdt7">{"message"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"string"}</span><span className="gdt0">{" };"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"28"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"type"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"ClaudeFrame"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt5">{"SDKMessage"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"|"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"TransportFrame"}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"29"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"30"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"function"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"appFetch"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"path"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"string"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"body"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"unknown"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"signal"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"AbortSignal"}</span><span className="gdt0">{")"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Promise"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"Response"}</span><span className="gdt0">{"> {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"31"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"try"}</span><span className="gdt0">{" {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"32"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"fetch"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"`${"}</span><span className="gdt1">{"BASE_URL"}</span><span className="gdt2">{"}${"}</span><span className="gdt0">{"path"}</span><span className="gdt2">{"}`"}</span><span className="gdt0">{", {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"33"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" method: "}</span><span className="gdt2">{"\"POST\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"34"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" headers: { "}</span><span className="gdt2">{"\"content-type\""}</span><span className="gdt0">{": "}</span><span className="gdt2">{"\"application/json\""}</span><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"35"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" body: "}</span><span className="gdt1">{"JSON"}</span><span className="gdt0">{"."}</span><span className="gdt5">{"stringify"}</span><span className="gdt0">{"(body),"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"36"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" signal,"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"37"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"38"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" } "}</span><span className="gdt4">{"catch"}</span><span className="gdt0">{" (err) {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"39"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" (signal.aborted) "}</span><span className="gdt4">{"throw"}</span><span className="gdt0">{" err;"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"40"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"throw"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"new"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Error"}</span><span className="gdt0">{"("}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"41"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt2">{"`连不上 ${"}</span><span className="gdt1">{"BASE_URL"}</span><span className="gdt2">{"}${"}</span><span className="gdt0">{"path"}</span><span className="gdt2">{"}。被测应用在跑吗?先起它:cd examples/zh/tier1/claude-sdk && pnpm start(或设 CLAUDE_SDK_URL 指向已部署实例)。`"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"42"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" );"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"43"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"44"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"45"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"46"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"interface"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Pending"}</span><span className="gdt0">{" {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"47"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"readonly"}</span><span className="gdt0">{" "}</span><span className="gdt7">{"cursor"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"SseFrameCursor"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"ClaudeFrame"}</span><span className="gdt0">{">;"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"48"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"readonly"}</span><span className="gdt0">{" "}</span><span className="gdt7">{"stream"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"ClaudeSdkStream"}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"49"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"readonly"}</span><span className="gdt0">{" "}</span><span className="gdt7">{"toolUseId"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"string"}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"50"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"51"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 会话续接走「服务端记历史」范式:请求带 ctx.session.id,init 帧回传的 session_id 用"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"52"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// ctx.session.capture 写回(first-writer-wins,不会被之后的重复回传覆盖)。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"53"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"54"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"function"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"readStream"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"cursor"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"SseFrameCursor"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"ClaudeFrame"}</span><span className="gdt0">{">, "}</span><span className="gdt7">{"ctx"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"AgentContext"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"stream"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"ClaudeSdkStream"}</span><span className="gdt0">{")"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Promise"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"Turn"}</span><span className="gdt0">{"> {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"55"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"driveFrameStream"}</span><span className="gdt0">{"(cursor, stream, ctx, ("}</span><span className="gdt7">{"frame"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"derived"}</span><span className="gdt0">{") "}</span><span className="gdt4">{"=>"}</span><span className="gdt0">{" {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"56"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" ctx.session."}</span><span className="gdt5">{"capture"}</span><span className="gdt0">{"(stream.sessionId);"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"57"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"58"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" (frame.type "}</span><span className="gdt4">{"==="}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"server_error\""}</span><span className="gdt0">{") "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{" { fail: (frame "}</span><span className="gdt4">{"as"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"TransportFrame"}</span><span className="gdt0">{").message };"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"59"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"60"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// HITL 停轮:gated 工具的 tool_use 到了(canUseTool 此刻把流卡住,不会再有后续帧)。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"61"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"gated"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" derived."}</span><span className="gdt5">{"find"}</span><span className="gdt0">{"(("}</span><span className="gdt7">{"e"}</span><span className="gdt0">{") "}</span><span className="gdt4">{"=>"}</span><span className="gdt0">{" e.type "}</span><span className="gdt4">{"==="}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"action.called\""}</span><span className="gdt0">{" "}</span><span className="gdt4">{"&&"}</span><span className="gdt0">{" e.name "}</span><span className="gdt4">{"==="}</span><span className="gdt0">{" "}</span><span className="gdt1">{"GATED_TOOL_NAME"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"62"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" (gated "}</span><span className="gdt4">{"&&"}</span><span className="gdt0">{" gated.type "}</span><span className="gdt4">{"==="}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"action.called\""}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"63"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" ctx.session."}</span><span className="gdt5">{"hold"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"Pending"}</span><span className="gdt0">{">({ cursor, stream, toolUseId: gated.callId });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"64"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{" { pause: { id: gated.callId, action: "}</span><span className="gdt1">{"GATED_TOOL_NAME"}</span><span className="gdt0">{", options: [{ id: "}</span><span className="gdt2">{"\"approve\""}</span><span className="gdt0">{" }, { id: "}</span><span className="gdt2">{"\"deny\""}</span><span className="gdt0">{" }] } };"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"65"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"66"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"67"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"68"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"69"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"/**"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"70"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{" * approve 端点在极少数情况下会在 canUseTool 真正把 resolver 存进服务端 pendingApprovals 之前"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"71"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{" * 就被我们打到——tool_use 块本身是\"模型已经决定调用\"的证据,但 SDK 内部要再过几十毫秒才跑到"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"72"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{" * canUseTool 回调去注册 resolver。404 大概率是这个注册竞态,不是真的没有这次审批,短退避重试"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"73"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{" * 几次;其它状态码直接抛。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"74"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{" */"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"75"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"function"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"postApprove"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"toolUseId"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"string"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"approved"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"boolean"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"signal"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"AbortSignal"}</span><span className="gdt0">{")"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Promise"}</span><span className="gdt0">{"<"}</span><span className="gdt1">{"void"}</span><span className="gdt0">{"> {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"76"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"deadline"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" Date."}</span><span className="gdt5">{"now"}</span><span className="gdt0">{"() "}</span><span className="gdt4">{"+"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"3_000"}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"77"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"for"}</span><span className="gdt0">{" (;;) {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"78"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"res"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"appFetch"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"/api/chat/approve\""}</span><span className="gdt0">{", { toolUseId, approved }, signal);"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"79"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" (res.ok) "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"80"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" (res.status "}</span><span className="gdt4">{"!=="}</span><span className="gdt0">{" "}</span><span className="gdt1">{"404"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"||"}</span><span className="gdt0">{" Date."}</span><span className="gdt5">{"now"}</span><span className="gdt0">{"() "}</span><span className="gdt4">{">="}</span><span className="gdt0">{" deadline) {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"81"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"throw"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"new"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Error"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"`POST /api/chat/approve 失败: ${"}</span><span className="gdt0">{"res"}</span><span className="gdt2">{"."}</span><span className="gdt0">{"status"}</span><span className="gdt2">{"} ${"}</span><span className="gdt4">{"await"}</span><span className="gdt2">{" "}</span><span className="gdt0">{"res"}</span><span className="gdt2">{"."}</span><span className="gdt5">{"text"}</span><span className="gdt2">{"()"}</span><span className="gdt2">{"}`"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"82"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"83"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"new"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"Promise"}</span><span className="gdt0">{"(("}</span><span className="gdt7">{"resolve"}</span><span className="gdt0">{") "}</span><span className="gdt4">{"=>"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"setTimeout"}</span><span className="gdt0">{"(resolve, "}</span><span className="gdt1">{"150"}</span><span className="gdt0">{"));"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"84"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"85"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"86"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"87"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"function"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"input"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"TurnInput"}</span><span className="gdt0">{", "}</span><span className="gdt7">{"ctx"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"AgentContext"}</span><span className="gdt0">{")"}</span><span className="gdt4">{":"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Promise"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"Turn"}</span><span className="gdt0">{"> {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"88"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"pending"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" ctx.session."}</span><span className="gdt5">{"take"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"Pending"}</span><span className="gdt0">{">();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"89"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" (pending) {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"90"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// 按 requestId(挂起的 toolUseId)从 input.responses 里对位取裁决,不从 text 猜;"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"91"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt6">{"// 这里每次只挂一条审批,取第一条即可——多请求并停时按 requestId 对位。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"92"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"approved"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" input.responses?.["}</span><span className="gdt1">{"0"}</span><span className="gdt0">{"]?.optionId "}</span><span className="gdt4">{"==="}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"approve\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"93"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" ("}</span><span className="gdt4">{"!"}</span><span className="gdt0">{"approved) pending.stream."}</span><span className="gdt5">{"markRejected"}</span><span className="gdt0">{"(pending.toolUseId);"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"94"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"postApprove"}</span><span className="gdt0">{"(pending.toolUseId, approved, ctx.signal);"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"95"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"readStream"}</span><span className="gdt0">{"(pending.cursor, ctx, pending.stream);"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"96"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"97"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"98"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"res"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"appFetch"}</span><span className="gdt0">{"("}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"99"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt2">{"\"/api/chat\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"100"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" { message: input.text, sessionId: ctx.session.id },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"101"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" ctx.signal,"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"102"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" );"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"103"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"if"}</span><span className="gdt0">{" ("}</span><span className="gdt4">{"!"}</span><span className="gdt0">{"res.ok "}</span><span className="gdt4">{"||"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"!"}</span><span className="gdt0">{"res.body) {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"104"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"throw"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"new"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"Error"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"`POST /api/chat 失败: ${"}</span><span className="gdt0">{"res"}</span><span className="gdt2">{"."}</span><span className="gdt0">{"status"}</span><span className="gdt2">{"} ${"}</span><span className="gdt4">{"await"}</span><span className="gdt2">{" "}</span><span className="gdt0">{"res"}</span><span className="gdt2">{"."}</span><span className="gdt5">{"text"}</span><span className="gdt2">{"()."}</span><span className="gdt5">{"catch"}</span><span className="gdt2">{"(() "}</span><span className="gdt4">{"=>"}</span><span className="gdt2">{" "}</span><span className="gdt2">{"\"\""}</span><span className="gdt2">{")"}</span><span className="gdt2">{"}`"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"105"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"106"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"return"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"readStream"}</span><span className="gdt0">{"("}</span><span className="gdt5">{"sseJsonFrames"}</span><span className="gdt0">{"<"}</span><span className="gdt5">{"ClaudeFrame"}</span><span className="gdt0">{">(res.body), ctx, "}</span><span className="gdt5">{"fromClaudeSdkMessages"}</span><span className="gdt0">{"());"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"107"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"}"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"108"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"109"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineAgent"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"110"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" name: "}</span><span className="gdt2">{"\"claude-sdk\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"111"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" send,"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"112"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
76
|
-
</div>
|
|
77
|
-
</div>
|
|
78
|
-
|
|
79
|
-
<div className="gd-file">
|
|
80
|
-
<div className="gd-head"><span className="gd-name">{"evals/basic-qa.eval.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+21"}</span></span></div>
|
|
81
|
-
<div className="gd-body">
|
|
82
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineEval } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 这条 eval 验证 agent 能正常问答、不瞎调工具,顺带冒烟 usage 有没有从 result 消息的"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// usage/total_cost_usd 正确映射进 Turn.usage。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineEval"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" description: "}</span><span className="gdt2">{"\"测试 agent 能正常问答且不瞎调工具\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"test"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"t"}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"turn"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"用一句话介绍一下你自己,这轮不用查天气也不用算数。\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" turn."}</span><span className="gdt5">{"expectOk"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"group"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"正常收发、没有多余工具调用\""}</span><span className="gdt0">{", () "}</span><span className="gdt4">{"=>"}</span><span className="gdt0">{" {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"13"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"succeeded"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"14"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"usedNoTools"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"15"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"16"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"maxTokens"}</span><span className="gdt0">{"("}</span><span className="gdt1">{"20_000"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"18"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"19"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t.judge.autoevals."}</span><span className="gdt5">{"closedQA"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"助手是否用一两句话正常介绍了自己,而不是报错或答非所问?\""}</span><span className="gdt0">{")."}</span><span className="gdt5">{"gate"}</span><span className="gdt0">{"("}</span><span className="gdt1">{"0.6"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"20"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"21"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
83
|
-
</div>
|
|
84
|
-
</div>
|
|
85
|
-
|
|
86
|
-
<div className="gd-file">
|
|
87
|
-
<div className="gd-head"><span className="gd-name">{"evals/hitl-approve.eval.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+23"}</span></span></div>
|
|
88
|
-
<div className="gd-body">
|
|
89
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineEval } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { equals } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval/expect\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// calculate 工具经 canUseTool 挂了审批(见 agents/claude-sdk.ts、origin src/backend/agent.ts)。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 这条验证批准分支:approve 之后工具正常执行,calledTool 的 status 是 \"completed\"。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"//"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 提示词不提\"审批\"——不同模型在提示词里看到\"审批\"字样时,有的会倾向于用文字问\"可以吗\""}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 而不是真的发起工具调用;审批门是服务端 canUseTool 自动挂的,跟用户怎么问无关。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineEval"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" description: "}</span><span className="gdt2">{"\"HITL:calculate 经批准后正常执行\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"test"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"t"}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"13"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"draft"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"用计算器算一下 (23+19)*3 等于多少\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"14"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"check"}</span><span className="gdt0">{"(draft.status, "}</span><span className="gdt5">{"equals"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"waiting\""}</span><span className="gdt0">{"));"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"15"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"16"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"requireInputRequest"}</span><span className="gdt0">{"({ action: "}</span><span className="gdt2">{"\"mcp__demo-tools__calculate\""}</span><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"18"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"approved"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"respond"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"approve\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"19"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" approved."}</span><span className="gdt5">{"succeeded"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"20"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"calledTool"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"mcp__demo-tools__calculate\""}</span><span className="gdt0">{", { status: "}</span><span className="gdt2">{"\"completed\""}</span><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"21"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"messageIncludes"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"/"}</span><span className="gdt8">{"126"}</span><span className="gdt2">{"/"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"22"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"23"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
90
|
-
</div>
|
|
91
|
-
</div>
|
|
92
|
-
|
|
93
|
-
<div className="gd-file">
|
|
94
|
-
<div className="gd-head"><span className="gd-name">{"evals/hitl-deny.eval.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+25"}</span></span></div>
|
|
95
|
-
<div className="gd-body">
|
|
96
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineEval } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { equals } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval/expect\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// deny 分支:人否决和工具故障是两回事——calledTool 的 status 应该是 \"rejected\""}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// (来自 system/permission_denied 帧,见 agents/claude-sdk.ts),noFailedActions() 依然通过。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"//"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// deepseek-v4-flash 被拒绝一次后,偶尔会不死心、原样再试一次同一个工具调用(新的 tool_use id,"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 同一个 gated 工具),这一轮又会停在新的 input.requested 上——不是 adapter 的映射 bug(每次"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 拒绝都正确落 rejected),是模型行为。deny 到它放弃为止,给个上限避免死循环。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineEval"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" description: "}</span><span className="gdt2">{"\"HITL:calculate 被拒绝后标记 rejected 而不是 failed\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"13"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"test"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"t"}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"14"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"用计算器算一下 (23+19)*3 等于多少\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"15"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"requireInputRequest"}</span><span className="gdt0">{"({ action: "}</span><span className="gdt2">{"\"mcp__demo-tools__calculate\""}</span><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"16"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"let"}</span><span className="gdt0">{" denied "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"respond"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"deny\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"18"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"for"}</span><span className="gdt0">{" ("}</span><span className="gdt4">{"let"}</span><span className="gdt0">{" attempt "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt1">{"0"}</span><span className="gdt0">{"; attempt "}</span><span className="gdt4">{"<"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"3"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"&&"}</span><span className="gdt0">{" denied.status "}</span><span className="gdt4">{"==="}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"waiting\""}</span><span className="gdt0">{"; attempt"}</span><span className="gdt4">{"++"}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"19"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" denied "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"respond"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"deny\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"20"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" }"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"21"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"check"}</span><span className="gdt0">{"(denied.status, "}</span><span className="gdt5">{"equals"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"completed\""}</span><span className="gdt0">{"));"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"22"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"calledTool"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"mcp__demo-tools__calculate\""}</span><span className="gdt0">{", { status: "}</span><span className="gdt2">{"\"rejected\""}</span><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"23"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"noFailedActions"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"24"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"25"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
97
|
-
</div>
|
|
98
|
-
</div>
|
|
99
|
-
|
|
100
|
-
<div className="gd-file">
|
|
101
|
-
<div className="gd-head"><span className="gd-name">{"evals/session-isolation.eval.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+20"}</span></span></div>
|
|
102
|
-
<div className="gd-body">
|
|
103
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineEval } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { includes, excludes } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval/expect\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 这条 eval 专门验证会话续接的两半承诺:同一条会话线里第二轮记得住第一轮说的名字"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// (SDK 的 resume 续接同一个 claude-code 会话历史成功);t.newSession() 造出的新会话线"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 不共享历史(常见 bug:adapter 不管新旧会话线都无条件复用同一个 ctx.session.id,隔离会静默失真且不报错)。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineEval"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" description: "}</span><span className="gdt2">{"\"测试跨轮记忆与 newSession() 隔离\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"test"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"t"}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"我叫小明,帮我记住这个名字。\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"recall"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"我刚才说我叫什么名字?\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"13"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" recall."}</span><span className="gdt5">{"messageIncludes"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"小明\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"14"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"check"}</span><span className="gdt0">{"(t.reply, "}</span><span className="gdt5">{"includes"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"小明\""}</span><span className="gdt0">{"));"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"15"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"16"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"fresh"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"newSession"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" fresh."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"我叫什么名字?\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"18"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"check"}</span><span className="gdt0">{"(fresh.reply, "}</span><span className="gdt5">{"excludes"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"小明\""}</span><span className="gdt0">{"));"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"19"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"20"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
104
|
-
</div>
|
|
105
|
-
</div>
|
|
106
|
-
|
|
107
|
-
<div className="gd-file">
|
|
108
|
-
<div className="gd-head"><span className="gd-name">{"evals/weather-tool.eval.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+22"}</span></span></div>
|
|
109
|
-
<div className="gd-body">
|
|
110
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineEval } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 这条 eval 验证 agent 遇到实时天气问题时会调 get_weather,而不是直接编一个答案。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 工具名是 MCP 命名空间下的真实名字 mcp__demo-tools__get_weather(不是裸的 get_weather),"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 见 agents/claude-sdk.ts 头注释。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineEval"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" description: "}</span><span className="gdt2">{"\"测试 agent 在天气问题中正确调用 get_weather 并基于结果作答\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"async"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"test"}</span><span className="gdt0">{"("}</span><span className="gdt7">{"t"}</span><span className="gdt0">{") {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"10"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"const"}</span><span className="gdt0">{" "}</span><span className="gdt1">{"turn"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"="}</span><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"send"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"北京今天天气怎么样?\""}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"11"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" turn."}</span><span className="gdt5">{"expectOk"}</span><span className="gdt0">{"();"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"12"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"13"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" "}</span><span className="gdt4">{"await"}</span><span className="gdt0">{" t."}</span><span className="gdt5">{"group"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"调用 get_weather 且城市正确\""}</span><span className="gdt0">{", () "}</span><span className="gdt4">{"=>"}</span><span className="gdt0">{" {"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"14"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"calledTool"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"mcp__demo-tools__get_weather\""}</span><span className="gdt0">{", { input: { city: "}</span><span className="gdt2">{"\"北京\""}</span><span className="gdt0">{" } });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"15"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t."}</span><span className="gdt5">{"messageIncludes"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"/"}</span><span className="gdt8">{"°C"}</span><span className="gdt4">{"|"}</span><span className="gdt8">{"气温"}</span><span className="gdt4">{"|"}</span><span className="gdt8">{"天气"}</span><span className="gdt4">{"|"}</span><span className="gdt8">{"晴"}</span><span className="gdt4">{"|"}</span><span className="gdt8">{"多云"}</span><span className="gdt4">{"|"}</span><span className="gdt8">{"雨"}</span><span className="gdt4">{"|"}</span><span className="gdt8">{"阴"}</span><span className="gdt2">{"/"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"16"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" });"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"17"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"18"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" t.judge.autoevals"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"19"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" ."}</span><span className="gdt5">{"closedQA"}</span><span className="gdt0">{"("}</span><span className="gdt2">{"\"助手是否给出了具体的天气数据(温度或天气状况),而不是拒绝回答或含糊其辞?\""}</span><span className="gdt0">{")"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"20"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" ."}</span><span className="gdt5">{"atLeast"}</span><span className="gdt0">{"("}</span><span className="gdt1">{"0.7"}</span><span className="gdt0">{");"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"21"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" },"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"22"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
111
|
-
</div>
|
|
112
|
-
</div>
|
|
113
|
-
|
|
114
|
-
<div className="gd-file">
|
|
115
|
-
<div className="gd-head"><span className="gd-name">{"experiments/assistant.ts"}</span><span className="gd-stats"><span className="gd-plus">{"+9"}</span></span></div>
|
|
116
|
-
<div className="gd-body">
|
|
117
|
-
<table className="gd-table"><tbody><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"1"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" { defineExperiment } "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"niceeval\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"2"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"import"}</span><span className="gdt0">{" agent "}</span><span className="gdt4">{"from"}</span><span className="gdt0">{" "}</span><span className="gdt2">{"\"../agents/claude-sdk.ts\""}</span><span className="gdt0">{";"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"3"}</td><td className="gd-sign">{"+"}</td><td className="gd-code">{" "}</td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"4"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt6">{"// 单配置基线:不比较模型,用 .env 里的默认模型。"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"5"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt4">{"export"}</span><span className="gdt0">{" "}</span><span className="gdt4">{"default"}</span><span className="gdt0">{" "}</span><span className="gdt5">{"defineExperiment"}</span><span className="gdt0">{"({"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"6"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" description: "}</span><span className="gdt2">{"\"claude-sdk:真实 Claude Agent SDK 后端(默认模型)\""}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"7"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" agent,"}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"8"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{" runs: "}</span><span className="gdt1">{"1"}</span><span className="gdt0">{","}</span></td></tr><tr className="gd-add"><td className="gd-ln"></td><td className="gd-ln">{"9"}</td><td className="gd-sign">{"+"}</td><td className="gd-code"><span className="gdt0">{"});"}</span></td></tr></tbody></table>
|
|
118
|
-
</div>
|
|
119
|
-
</div>
|