niceeval 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +13 -13
- package/docs/README.md +119 -0
- package/docs/adapters/README.md +60 -0
- package/docs/adapters/authoring.md +179 -0
- package/docs/adapters/coding-agent-skills-plugins.md +324 -0
- package/docs/adapters/collection.md +128 -0
- package/docs/adapters/contract.md +263 -0
- package/docs/adapters/reference/agent-eval.md +215 -0
- package/docs/adapters/reference/agent-loop-apis.md +69 -0
- package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
- package/docs/adapters/reference/eve-protocol.md +127 -0
- package/docs/adapters/reference/otel-genai.md +107 -0
- package/docs/adapters/reference/otel-instrumentation.md +65 -0
- package/docs/adapters/targets.md +99 -0
- package/docs/architecture.md +140 -0
- package/docs/assertions.md +388 -0
- package/docs/capabilities-by-construction.md +48 -0
- package/docs/cli.md +171 -0
- package/docs/concepts.md +106 -0
- package/docs/e2e-ci.md +295 -0
- package/docs/eval-authoring.md +319 -0
- package/docs/experiments.md +154 -0
- package/docs/getting-started.md +224 -0
- package/docs/multi-agent.md +145 -0
- package/docs/observability.md +333 -0
- package/docs/origin-integration.md +195 -0
- package/docs/references.md +35 -0
- package/docs/results-format.md +228 -0
- package/docs/runner.md +104 -0
- package/docs/sandbox.md +276 -0
- package/docs/scoring.md +119 -0
- package/docs/source-map.md +106 -0
- package/docs/tier-sync.md +177 -0
- package/docs/view.md +157 -0
- package/docs/vision.md +88 -0
- package/package.json +35 -5
- package/src/agents/ai-sdk-otel.ts +0 -0
- package/src/agents/ai-sdk.test.ts +377 -0
- package/src/agents/ai-sdk.ts +577 -0
- package/src/agents/bub.ts +30 -37
- package/src/agents/claude-code.ts +7 -4
- package/src/agents/codex.ts +15 -10
- package/src/agents/index.ts +43 -1
- package/src/agents/sdk-streams.test.ts +145 -0
- package/src/agents/sdk-streams.ts +457 -0
- package/src/agents/shared.ts +77 -39
- package/src/agents/streaming.test.ts +152 -0
- package/src/agents/streaming.ts +195 -0
- package/src/agents/types.ts +218 -0
- package/src/agents/ui-message-stream.test.ts +249 -0
- package/src/agents/ui-message-stream.ts +372 -0
- package/src/cli.ts +124 -77
- package/src/context/context.test.ts +203 -7
- package/src/context/context.ts +158 -49
- package/src/context/session.test.ts +96 -0
- package/src/context/session.ts +119 -9
- package/src/context/types.ts +205 -0
- package/src/define.ts +4 -14
- package/src/expect/index.ts +8 -71
- package/src/i18n/core.ts +28 -0
- package/src/i18n/en.ts +47 -5
- package/src/i18n/index.ts +5 -17
- package/src/i18n/zh-CN.ts +47 -5
- package/src/index.ts +12 -0
- package/src/loaders/index.ts +8 -39
- package/src/o11y/otlp/canonical.ts +7 -6
- package/src/o11y/otlp/mappers/codex.ts +10 -8
- package/src/o11y/otlp/mappers/index.ts +9 -21
- package/src/o11y/otlp/receiver.ts +25 -7
- package/src/o11y/otlp/sandbox-receiver.ts +57 -21
- package/src/o11y/otlp/turn-otel.test.ts +110 -0
- package/src/o11y/otlp/turn-otel.ts +125 -0
- package/src/o11y/parsers/bub.ts +13 -11
- package/src/o11y/parsers/claude-code.ts +27 -51
- package/src/o11y/parsers/codex.ts +29 -46
- package/src/o11y/parsers/index.ts +5 -14
- package/src/o11y/tool-names.test.ts +61 -0
- package/src/o11y/tool-names.ts +74 -0
- package/src/o11y/types.ts +135 -0
- package/src/runner/attempt.ts +456 -0
- package/src/runner/fingerprint.ts +59 -0
- package/src/runner/remote-sandbox.ts +53 -0
- package/src/runner/report.ts +53 -0
- package/src/runner/reporters/artifacts.ts +25 -4
- package/src/runner/reporters/console.ts +2 -8
- package/src/runner/reporters/live.ts +2 -8
- package/src/runner/reporters/shared.ts +15 -0
- package/src/runner/reporters/table.ts +10 -62
- package/src/runner/run.ts +114 -619
- package/src/runner/types.ts +237 -0
- package/src/sandbox/checkpoint.ts +15 -13
- package/src/sandbox/docker-stream.ts +87 -0
- package/src/sandbox/docker.ts +42 -120
- package/src/sandbox/e2b.ts +24 -75
- package/src/sandbox/local-files.ts +33 -0
- package/src/sandbox/paths.test.ts +85 -0
- package/src/sandbox/paths.ts +45 -0
- package/src/sandbox/resolve.ts +10 -34
- package/src/sandbox/shell.ts +17 -0
- package/src/sandbox/source-files.ts +42 -2
- package/src/sandbox/types.ts +158 -0
- package/src/sandbox/vercel.ts +55 -71
- package/src/scoring/collector.ts +3 -2
- package/src/scoring/judge.ts +72 -146
- package/src/scoring/match.ts +79 -0
- package/src/scoring/types.ts +76 -0
- package/src/shared/aggregate.ts +48 -0
- package/src/shared/format.ts +20 -0
- package/src/shared/outcome.ts +48 -0
- package/src/shared/types.ts +37 -0
- package/src/types.ts +20 -911
- package/src/view/aggregate.ts +103 -0
- package/src/view/app/App.tsx +26 -5
- package/src/view/app/components/AttemptModal.tsx +12 -3
- package/src/view/app/components/CodeView.tsx +9 -5
- package/src/view/app/components/CopyControls.tsx +4 -4
- package/src/view/app/components/ExperimentTable.tsx +5 -5
- package/src/view/app/i18n.ts +31 -7
- package/src/view/app/lib/format.ts +17 -20
- package/src/view/app/lib/outcome.ts +4 -16
- package/src/view/app/main.tsx +3 -5
- package/src/view/app/pages/RunsPage.tsx +2 -2
- package/src/view/app/pages/TracesPage.tsx +2 -2
- package/src/view/app/shared.ts +2 -1
- package/src/view/app/types.ts +6 -43
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +24 -401
- package/src/view/loader.ts +234 -0
- package/src/view/server.ts +136 -0
- package/src/view/shared/types.ts +64 -0
- package/src/view/styles.css +60 -9
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
|
|
3
|
+
import type { AgentContext } from "../types.ts";
|
|
4
|
+
import { uiMessageStreamAgent, type UIMessageLike, type UIMessagePartLike } from "./ui-message-stream.ts";
|
|
5
|
+
import { createAgentSession } from "../context/session.ts";
|
|
6
|
+
|
|
7
|
+
// ─────────────────── mock `ai`:协议语义的最小 reducer(readUIMessageStream 同款行为) ───────────────────
|
|
8
|
+
// 只实现测试用到的 chunk 类型;seed(resume 的 message)上的 parts 原样保留、按 toolCallId 更新。
|
|
9
|
+
|
|
10
|
+
vi.mock("ai", () => ({
|
|
11
|
+
readUIMessageStream: ({ message, stream }: { message?: UIMessageLike; stream: ReadableStream<Record<string, unknown>> }) =>
|
|
12
|
+
(async function* () {
|
|
13
|
+
const msg: UIMessageLike = message
|
|
14
|
+
? { ...message, parts: message.parts.map((p) => ({ ...p })) }
|
|
15
|
+
: { id: "m1", role: "assistant", parts: [] };
|
|
16
|
+
const toolPart = (toolCallId: string) =>
|
|
17
|
+
msg.parts.find((p) => (p as UIMessagePartLike).toolCallId === toolCallId) as UIMessagePartLike | undefined;
|
|
18
|
+
const reader = stream.getReader();
|
|
19
|
+
for (;;) {
|
|
20
|
+
const { value: c, done } = await reader.read();
|
|
21
|
+
if (done) break;
|
|
22
|
+
const chunk = c as Record<string, unknown>;
|
|
23
|
+
switch (chunk.type) {
|
|
24
|
+
case "text-delta": {
|
|
25
|
+
const last = msg.parts.at(-1);
|
|
26
|
+
if (last?.type === "text") last.text = (last.text ?? "") + (chunk.delta as string);
|
|
27
|
+
else msg.parts.push({ type: "text", text: chunk.delta as string });
|
|
28
|
+
break;
|
|
29
|
+
}
|
|
30
|
+
case "tool-input-available":
|
|
31
|
+
msg.parts.push({
|
|
32
|
+
type: `tool-${chunk.toolName as string}`,
|
|
33
|
+
state: "input-available",
|
|
34
|
+
toolCallId: chunk.toolCallId as string,
|
|
35
|
+
input: chunk.input,
|
|
36
|
+
});
|
|
37
|
+
break;
|
|
38
|
+
case "tool-output-available": {
|
|
39
|
+
const p = toolPart(chunk.toolCallId as string);
|
|
40
|
+
if (p) {
|
|
41
|
+
p.state = "output-available";
|
|
42
|
+
p.output = chunk.output;
|
|
43
|
+
}
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
case "tool-approval-request": {
|
|
47
|
+
const p = toolPart(chunk.toolCallId as string);
|
|
48
|
+
if (p) {
|
|
49
|
+
p.state = "approval-requested";
|
|
50
|
+
p.approval = { id: chunk.approvalId as string };
|
|
51
|
+
}
|
|
52
|
+
break;
|
|
53
|
+
}
|
|
54
|
+
default:
|
|
55
|
+
break; // start / finish / error 等对归约无影响
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
yield msg;
|
|
59
|
+
})(),
|
|
60
|
+
}));
|
|
61
|
+
|
|
62
|
+
// ─────────────────── fetch mock:每次调用吐一段脚本化的 SSE ───────────────────
|
|
63
|
+
|
|
64
|
+
function sse(chunks: Array<Record<string, unknown>>): Response {
|
|
65
|
+
const payload = [...chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`), "data: [DONE]\n\n"].join("");
|
|
66
|
+
return new Response(payload, { status: 200, headers: { "content-type": "text/event-stream" } });
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const fetchMock = vi.fn<typeof fetch>();
|
|
70
|
+
vi.stubGlobal("fetch", fetchMock);
|
|
71
|
+
|
|
72
|
+
function ctx(overrides: Partial<{ model: string }> = {}): AgentContext {
|
|
73
|
+
return {
|
|
74
|
+
signal: new AbortController().signal,
|
|
75
|
+
model: overrides.model,
|
|
76
|
+
flags: {},
|
|
77
|
+
sandbox: undefined as never,
|
|
78
|
+
// 同一个 ctx 重复用 = 同一条会话线(续接,同一个 ctx.session);新造 = 新线。
|
|
79
|
+
session: createAgentSession(),
|
|
80
|
+
log: () => {},
|
|
81
|
+
} as AgentContext;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function sentBody(call: number): { messages: UIMessageLike[] } & Record<string, unknown> {
|
|
85
|
+
return JSON.parse(fetchMock.mock.calls[call]![1]!.body as string) as { messages: UIMessageLike[] };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
afterEach(() => {
|
|
89
|
+
fetchMock.mockReset();
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe("uiMessageStreamAgent", () => {
|
|
93
|
+
it("纯文本轮:message 事件 + completed;第二轮重放全量历史(客户端带历史的协议语义)", async () => {
|
|
94
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat", body: (c) => ({ model: c.model }) });
|
|
95
|
+
fetchMock.mockResolvedValueOnce(sse([{ type: "text-delta", delta: "你好" }, { type: "text-delta", delta: "!" }]));
|
|
96
|
+
|
|
97
|
+
const c = ctx({ model: "m-1" });
|
|
98
|
+
const turn = await agent.send({ text: "hi" }, c);
|
|
99
|
+
expect(turn.status).toBe("completed");
|
|
100
|
+
expect(turn.events).toEqual([{ type: "message", role: "assistant", text: "你好!" }]);
|
|
101
|
+
expect(c.session.id).toBeTruthy();
|
|
102
|
+
expect(sentBody(0).model).toBe("m-1");
|
|
103
|
+
expect(sentBody(0).messages).toHaveLength(1);
|
|
104
|
+
|
|
105
|
+
fetchMock.mockResolvedValueOnce(sse([{ type: "text-delta", delta: "again" }]));
|
|
106
|
+
const turn2 = await agent.send({ text: "再说一次" }, c);
|
|
107
|
+
// user, assistant, user —— 全量历史重放
|
|
108
|
+
expect(sentBody(1).messages.map((m) => m.role)).toEqual(["user", "assistant", "user"]);
|
|
109
|
+
// 回归:mock 的响应消息 id 每轮都是 "m1"(有的应用真的会这样)。全新轮的文本必须完整
|
|
110
|
+
// 报出,不能被上一轮的已报进度按 id 误当成"已报过的前缀"截掉。
|
|
111
|
+
expect(turn2.events).toEqual([{ type: "message", role: "assistant", text: "again" }]);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("工具 part 到 output-available → action.called + action.result", async () => {
|
|
115
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat" });
|
|
116
|
+
fetchMock.mockResolvedValueOnce(
|
|
117
|
+
sse([
|
|
118
|
+
{ type: "tool-input-available", toolCallId: "c1", toolName: "get_weather", input: { city: "北京" } },
|
|
119
|
+
{ type: "tool-output-available", toolCallId: "c1", output: { temp: 21 } },
|
|
120
|
+
{ type: "text-delta", delta: "21 度" },
|
|
121
|
+
]),
|
|
122
|
+
);
|
|
123
|
+
const turn = await agent.send({ text: "天气" }, ctx());
|
|
124
|
+
expect(turn.events).toEqual([
|
|
125
|
+
{ type: "action.called", callId: "c1", name: "get_weather", input: { city: "北京" } },
|
|
126
|
+
{ type: "action.result", callId: "c1", output: { temp: 21 }, status: "completed" },
|
|
127
|
+
{ type: "message", role: "assistant", text: "21 度" },
|
|
128
|
+
]);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it("HITL:approval-requested → waiting;approve 续跑改写 part 重放,拿到执行结果不重报旧事件", async () => {
|
|
132
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat" });
|
|
133
|
+
fetchMock.mockResolvedValueOnce(
|
|
134
|
+
sse([
|
|
135
|
+
{ type: "tool-input-available", toolCallId: "c1", toolName: "calculate", input: { expr: "1+1" } },
|
|
136
|
+
{ type: "tool-approval-request", toolCallId: "c1", approvalId: "ap1" },
|
|
137
|
+
]),
|
|
138
|
+
);
|
|
139
|
+
const c = ctx();
|
|
140
|
+
const turn1 = await agent.send({ text: "算 1+1" }, c);
|
|
141
|
+
expect(turn1.status).toBe("waiting");
|
|
142
|
+
expect(turn1.events).toEqual([
|
|
143
|
+
{
|
|
144
|
+
type: "input.requested",
|
|
145
|
+
request: { id: "ap1", action: "calculate", input: { expr: "1+1" }, options: [{ id: "approve" }, { id: "deny" }] },
|
|
146
|
+
},
|
|
147
|
+
]);
|
|
148
|
+
|
|
149
|
+
fetchMock.mockResolvedValueOnce(
|
|
150
|
+
sse([{ type: "tool-output-available", toolCallId: "c1", output: 2 }, { type: "text-delta", delta: "= 2" }]),
|
|
151
|
+
);
|
|
152
|
+
const turn2 = await agent.send({ text: "approve" }, c);
|
|
153
|
+
expect(turn2.status).toBe("completed");
|
|
154
|
+
expect(turn2.events).toEqual([
|
|
155
|
+
{ type: "action.called", callId: "c1", name: "calculate", input: { expr: "1+1" } },
|
|
156
|
+
{ type: "action.result", callId: "c1", output: 2, status: "completed" },
|
|
157
|
+
{ type: "message", role: "assistant", text: "= 2" },
|
|
158
|
+
]);
|
|
159
|
+
// 续跑请求:不追加新 user 消息,pending part 已改写成 approval-responded approved:true
|
|
160
|
+
const resumeBody = sentBody(1);
|
|
161
|
+
expect(resumeBody.messages.map((m) => m.role)).toEqual(["user", "assistant"]);
|
|
162
|
+
const mutated = resumeBody.messages[1]!.parts.find((p) => p.toolCallId === "c1")!;
|
|
163
|
+
expect(mutated.state).toBe("approval-responded");
|
|
164
|
+
expect(mutated.approval).toMatchObject({ id: "ap1", approved: true });
|
|
165
|
+
expect(mutated.approval!.reason).toBeUndefined();
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("HITL:deny 合成 rejected 的工具对,并带上劝退重试的 reason", async () => {
|
|
169
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat", denyReason: "别重试" });
|
|
170
|
+
fetchMock.mockResolvedValueOnce(
|
|
171
|
+
sse([
|
|
172
|
+
{ type: "tool-input-available", toolCallId: "c1", toolName: "calculate", input: { expr: "1+1" } },
|
|
173
|
+
{ type: "tool-approval-request", toolCallId: "c1", approvalId: "ap1" },
|
|
174
|
+
]),
|
|
175
|
+
);
|
|
176
|
+
const c = ctx();
|
|
177
|
+
await agent.send({ text: "算" }, c);
|
|
178
|
+
|
|
179
|
+
fetchMock.mockResolvedValueOnce(sse([{ type: "text-delta", delta: "好的,不算了" }]));
|
|
180
|
+
const turn2 = await agent.send({ text: "deny" }, c);
|
|
181
|
+
expect(turn2.status).toBe("completed");
|
|
182
|
+
expect(turn2.events).toEqual([
|
|
183
|
+
{ type: "action.called", callId: "c1", name: "calculate", input: { expr: "1+1" } },
|
|
184
|
+
{ type: "action.result", callId: "c1", status: "rejected" },
|
|
185
|
+
{ type: "message", role: "assistant", text: "好的,不算了" },
|
|
186
|
+
]);
|
|
187
|
+
const mutated = sentBody(1).messages[1]!.parts.find((p) => p.toolCallId === "c1")!;
|
|
188
|
+
expect(mutated.approval).toMatchObject({ approved: false, reason: "别重试" });
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
it("HITL:优先按 requestId 从 input.responses 读裁决,而不是猜 input.text——两者矛盾时以 responses 为准", async () => {
|
|
192
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat" });
|
|
193
|
+
fetchMock.mockResolvedValueOnce(
|
|
194
|
+
sse([
|
|
195
|
+
{ type: "tool-input-available", toolCallId: "c1", toolName: "calculate", input: { expr: "1+1" } },
|
|
196
|
+
{ type: "tool-approval-request", toolCallId: "c1", approvalId: "ap1" },
|
|
197
|
+
]),
|
|
198
|
+
);
|
|
199
|
+
const c = ctx();
|
|
200
|
+
await agent.send({ text: "算 1+1" }, c);
|
|
201
|
+
|
|
202
|
+
fetchMock.mockResolvedValueOnce(sse([{ type: "text-delta", delta: "好的,不算了" }]));
|
|
203
|
+
// input.text 读起来像批准,但 responses 结构化裁决说 deny——必须以 responses 为准。
|
|
204
|
+
const turn2 = await agent.send({ text: "approve", responses: [{ requestId: "ap1", optionId: "deny" }] }, c);
|
|
205
|
+
expect(turn2.events).toEqual([
|
|
206
|
+
{ type: "action.called", callId: "c1", name: "calculate", input: { expr: "1+1" } },
|
|
207
|
+
{ type: "action.result", callId: "c1", status: "rejected" },
|
|
208
|
+
{ type: "message", role: "assistant", text: "好的,不算了" },
|
|
209
|
+
]);
|
|
210
|
+
const mutated = sentBody(1).messages[1]!.parts.find((p) => p.toolCallId === "c1")!;
|
|
211
|
+
expect(mutated.approval).toMatchObject({ approved: false });
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it("HITL:没有 input.responses 时(旧 adapter 用法)退回从 input.text 猜裁决", async () => {
|
|
215
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat" });
|
|
216
|
+
fetchMock.mockResolvedValueOnce(
|
|
217
|
+
sse([
|
|
218
|
+
{ type: "tool-input-available", toolCallId: "c1", toolName: "calculate", input: { expr: "1+1" } },
|
|
219
|
+
{ type: "tool-approval-request", toolCallId: "c1", approvalId: "ap1" },
|
|
220
|
+
]),
|
|
221
|
+
);
|
|
222
|
+
const c = ctx();
|
|
223
|
+
await agent.send({ text: "算 1+1" }, c);
|
|
224
|
+
|
|
225
|
+
fetchMock.mockResolvedValueOnce(
|
|
226
|
+
sse([{ type: "tool-output-available", toolCallId: "c1", output: 2 }, { type: "text-delta", delta: "= 2" }]),
|
|
227
|
+
);
|
|
228
|
+
const turn2 = await agent.send({ text: "approve" }, c); // 没有 responses 字段
|
|
229
|
+
expect(turn2.events).toEqual([
|
|
230
|
+
{ type: "action.called", callId: "c1", name: "calculate", input: { expr: "1+1" } },
|
|
231
|
+
{ type: "action.result", callId: "c1", output: 2, status: "completed" },
|
|
232
|
+
{ type: "message", role: "assistant", text: "= 2" },
|
|
233
|
+
]);
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
it("error 帧 → failed + error 事件", async () => {
|
|
237
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat" });
|
|
238
|
+
fetchMock.mockResolvedValueOnce(sse([{ type: "text-delta", delta: "部分" }, { type: "error", errorText: "boom" }]));
|
|
239
|
+
const turn = await agent.send({ text: "hi" }, ctx());
|
|
240
|
+
expect(turn.status).toBe("failed");
|
|
241
|
+
expect(turn.events.at(-1)).toEqual({ type: "error", message: "boom" });
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it("非 2xx 响应:报错里带状态码和下一步提示", async () => {
|
|
245
|
+
const agent = uiMessageStreamAgent({ name: "t", url: "http://x/api/chat" });
|
|
246
|
+
fetchMock.mockResolvedValueOnce(new Response("nope", { status: 500 }));
|
|
247
|
+
await expect(agent.send({ text: "hi" }, ctx())).rejects.toThrow(/500/);
|
|
248
|
+
});
|
|
249
|
+
});
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
// uiMessageStreamAgent():AI SDK UI Message Stream Protocol 的无侵入 HTTP adapter 工厂。
|
|
2
|
+
//
|
|
3
|
+
// 协议:https://ai-sdk.dev/docs/ai-sdk-ui/stream-protocol —— `useChat` 后端的标准 SSE
|
|
4
|
+
// (`data: {UIMessageChunk}\n\n`,以 `data: [DONE]\n\n` 收尾)。这是「对着一个已部署的
|
|
5
|
+
// AI SDK 应用的 HTTP 接口无侵入接入」:adapter 只 fetch,不 import 被测应用的任何代码。
|
|
6
|
+
//
|
|
7
|
+
// · 会话:协议是服务端零状态、「客户端带全量历史」——工厂用 ctx.session.history() 存整份
|
|
8
|
+
// UIMessage[],每轮原样重放;ctx.session.id 未记录时开新 chat id 并 capture 回写。
|
|
9
|
+
// · 事件流:从归约后的 assistant 消息 parts 直构(text → message,tool part 的
|
|
10
|
+
// output-available / output-error / 审批拒绝 → action.called + action.result),
|
|
11
|
+
// 不要求应用接 OTel;跨 resume 轮次按 callId / 已报文本长度去重。
|
|
12
|
+
// · HITL:v7 tool approval(`needsApproval` 工具)——part 停在 `approval-requested` 时
|
|
13
|
+
// 整轮 `waiting` + `input.requested`;下一轮输入(approve / yes / 同意 / 批准 开头 =
|
|
14
|
+
// 批准,其余拒绝)翻译成 `approval-responded` 原地改写该 part、原样重发 messages 触发
|
|
15
|
+
// 服务端续跑 —— 和真实前端 `addToolApprovalResponse()` + `sendMessage()` 的协议行为
|
|
16
|
+
// 完全一致,没有单独的 approve 端点。拒绝的调用协议里不会有任何 tool-output 帧
|
|
17
|
+
// (从没真正执行),由工厂合成 `status: "rejected"` 的 action.result。
|
|
18
|
+
// · chunk 归约用 `ai` 包官方导出的框架无关 reducer `readUIMessageStream`(`useChat`
|
|
19
|
+
// 内部同款),保证重放回服务端的 UIMessage 形状协议正确 —— `ai` 是可选 peer 依赖,
|
|
20
|
+
// 只在用到本工厂时需要安装。
|
|
21
|
+
//
|
|
22
|
+
// tracing / spanMapper 原样透传:应用有 OTel 时接上拿瀑布图(span 只进瀑布图,不喂断言),
|
|
23
|
+
// 事件流始终从协议帧直构。
|
|
24
|
+
|
|
25
|
+
import { randomUUID } from "node:crypto";
|
|
26
|
+
|
|
27
|
+
import { defineAgent } from "../define.ts";
|
|
28
|
+
import type { Agent, AgentContext, AgentTracing, InputResponse, JsonValue, SpanMapper, StreamEvent, TurnInput } from "../types.ts";
|
|
29
|
+
|
|
30
|
+
// ───────────────────────── 协议的结构化类型(structural,不依赖 ai 包的类型) ─────────────────────────
|
|
31
|
+
|
|
32
|
+
/** UIMessage 的最小结构面:只声明工厂真正要读的字段,其余原样透传。 */
|
|
33
|
+
export interface UIMessageLike {
|
|
34
|
+
id: string;
|
|
35
|
+
role: string;
|
|
36
|
+
parts: UIMessagePartLike[];
|
|
37
|
+
[key: string]: unknown;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface UIMessagePartLike {
|
|
41
|
+
type: string;
|
|
42
|
+
state?: string;
|
|
43
|
+
text?: string;
|
|
44
|
+
toolCallId?: string;
|
|
45
|
+
toolName?: string;
|
|
46
|
+
input?: unknown;
|
|
47
|
+
output?: unknown;
|
|
48
|
+
errorText?: string;
|
|
49
|
+
approval?: { id: string; approved?: boolean; reason?: string };
|
|
50
|
+
[key: string]: unknown;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface UIMessageChunkLike {
|
|
54
|
+
type: string;
|
|
55
|
+
errorText?: string;
|
|
56
|
+
[key: string]: unknown;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
type ReadUIMessageStream = (options: {
|
|
60
|
+
message?: UIMessageLike;
|
|
61
|
+
stream: ReadableStream<UIMessageChunkLike>;
|
|
62
|
+
}) => AsyncIterable<UIMessageLike>;
|
|
63
|
+
|
|
64
|
+
// ai 是可选 peer 依赖:动态 import,缺了就把「装什么」直接说清楚。
|
|
65
|
+
// 说明符经变量传入,避免 TS 对字面量模块名做安装检查(niceeval 自身不依赖 ai)。
|
|
66
|
+
let aiModule: Promise<{ readUIMessageStream: ReadUIMessageStream }> | undefined;
|
|
67
|
+
function loadAi(): Promise<{ readUIMessageStream: ReadUIMessageStream }> {
|
|
68
|
+
if (!aiModule) {
|
|
69
|
+
const specifier = "ai";
|
|
70
|
+
aiModule = (import(specifier) as Promise<{ readUIMessageStream: ReadUIMessageStream }>).catch(() => {
|
|
71
|
+
aiModule = undefined;
|
|
72
|
+
throw new Error(
|
|
73
|
+
"uiMessageStreamAgent 需要 `ai` 包(AI SDK v5+,协议 reducer readUIMessageStream 来自它)。在你的 eval 项目里安装:npm install -D ai",
|
|
74
|
+
);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return aiModule;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ───────────────────────── SSE 解析 ─────────────────────────
|
|
81
|
+
|
|
82
|
+
async function* parseSseChunks(body: ReadableStream<Uint8Array>): AsyncGenerator<UIMessageChunkLike> {
|
|
83
|
+
const reader = body.getReader();
|
|
84
|
+
const decoder = new TextDecoder();
|
|
85
|
+
let buffer = "";
|
|
86
|
+
for (;;) {
|
|
87
|
+
const sepIndex = buffer.indexOf("\n\n");
|
|
88
|
+
if (sepIndex !== -1) {
|
|
89
|
+
const rawEvent = buffer.slice(0, sepIndex);
|
|
90
|
+
buffer = buffer.slice(sepIndex + 2);
|
|
91
|
+
const line = rawEvent.split("\n").find((l) => l.startsWith("data: "));
|
|
92
|
+
if (line) {
|
|
93
|
+
const payload = line.slice("data: ".length);
|
|
94
|
+
if (payload !== "[DONE]") yield JSON.parse(payload) as UIMessageChunkLike;
|
|
95
|
+
}
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
const { value, done } = await reader.read();
|
|
99
|
+
if (done) return;
|
|
100
|
+
buffer += decoder.decode(value, { stream: true });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** 把 SSE 包成 readUIMessageStream 要的 ReadableStream,顺带旁路探测(错误帧等)。 */
|
|
105
|
+
function toChunkStream(
|
|
106
|
+
body: ReadableStream<Uint8Array>,
|
|
107
|
+
onChunk: (c: UIMessageChunkLike) => void,
|
|
108
|
+
): ReadableStream<UIMessageChunkLike> {
|
|
109
|
+
const gen = parseSseChunks(body);
|
|
110
|
+
return new ReadableStream<UIMessageChunkLike>({
|
|
111
|
+
async pull(controller) {
|
|
112
|
+
const { value, done } = await gen.next();
|
|
113
|
+
if (done) {
|
|
114
|
+
controller.close();
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
onChunk(value);
|
|
118
|
+
controller.enqueue(value);
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ───────────────────────── parts → 事件 ─────────────────────────
|
|
124
|
+
|
|
125
|
+
function isToolPart(part: UIMessagePartLike): boolean {
|
|
126
|
+
return part.type === "dynamic-tool" || part.type.startsWith("tool-");
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function toolNameOf(part: UIMessagePartLike): string {
|
|
130
|
+
if (part.type === "dynamic-tool") return part.toolName ?? "dynamic-tool";
|
|
131
|
+
return part.type.slice("tool-".length);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function isApprovalRequested(part: UIMessagePartLike): boolean {
|
|
135
|
+
return isToolPart(part) && part.state === "approval-requested";
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** 和 aiSdkAgent 同一词法:approve / yes / 同意 / 批准 开头 = 批准,其余一律拒绝。 */
|
|
139
|
+
function isApproved(text: string): boolean {
|
|
140
|
+
return /^(approve|yes|同意|批准)/i.test(text.trim());
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* 优先按 requestId 从 input.responses 里取结构化裁决(`optionId === "approve"`);
|
|
145
|
+
* 拿不到 responses(旧 adapter / 手写 send 未接结构化回答)才退回从 input.text 猜的路径,
|
|
146
|
+
* 防御式保留,不删旧分支。
|
|
147
|
+
*/
|
|
148
|
+
function approvedFromResponse(
|
|
149
|
+
responses: readonly InputResponse[] | undefined,
|
|
150
|
+
requestId: string | undefined,
|
|
151
|
+
fallbackText: string,
|
|
152
|
+
): boolean {
|
|
153
|
+
const matched = responses?.find((r) => r.requestId === requestId);
|
|
154
|
+
if (matched?.optionId !== undefined) return matched.optionId === "approve";
|
|
155
|
+
if (matched?.text !== undefined) return isApproved(matched.text);
|
|
156
|
+
return isApproved(fallbackText);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** 已报告的进度:resume 续跑的是同一条 assistant 消息,跨轮去重靠它。 */
|
|
160
|
+
interface ReportedState {
|
|
161
|
+
calls: Set<string>;
|
|
162
|
+
textLen: number;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* 从归约后的最终消息派生本轮事件。工具事件按 part 顺序;文本合并成一条 message 事件
|
|
167
|
+
* (resume 轮只报新增的后缀)。停在 approval-requested 的调用不报 called —— 它还没执行,
|
|
168
|
+
* 裁决后那一轮才落成 completed(批准)或 rejected(拒绝)。
|
|
169
|
+
*/
|
|
170
|
+
function deriveTurnEvents(message: UIMessageLike, reported: ReportedState): StreamEvent[] {
|
|
171
|
+
const events: StreamEvent[] = [];
|
|
172
|
+
let fullText = "";
|
|
173
|
+
for (const part of message.parts) {
|
|
174
|
+
if (part.type === "text") {
|
|
175
|
+
fullText += part.text ?? "";
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (!isToolPart(part)) continue;
|
|
179
|
+
const callId = part.toolCallId ?? "";
|
|
180
|
+
if (!callId || reported.calls.has(callId)) continue;
|
|
181
|
+
const name = toolNameOf(part);
|
|
182
|
+
const input = (part.input ?? null) as JsonValue;
|
|
183
|
+
if (part.state === "output-available") {
|
|
184
|
+
events.push({ type: "action.called", callId, name, input });
|
|
185
|
+
events.push({ type: "action.result", callId, output: part.output as JsonValue, status: "completed" });
|
|
186
|
+
reported.calls.add(callId);
|
|
187
|
+
} else if (part.state === "output-error") {
|
|
188
|
+
events.push({ type: "action.called", callId, name, input });
|
|
189
|
+
events.push({ type: "action.result", callId, output: part.errorText, status: "failed" });
|
|
190
|
+
reported.calls.add(callId);
|
|
191
|
+
} else if (part.state === "approval-responded" && part.approval?.approved === false) {
|
|
192
|
+
// 拒绝的调用从没真正执行,协议里不会再有它的任何帧 —— 在裁决落地的这一轮合成。
|
|
193
|
+
events.push({ type: "action.called", callId, name, input });
|
|
194
|
+
events.push({ type: "action.result", callId, status: "rejected" });
|
|
195
|
+
reported.calls.add(callId);
|
|
196
|
+
}
|
|
197
|
+
// approval-requested(还没裁决)/ input-* 中间态:先不报,等它到终态。
|
|
198
|
+
}
|
|
199
|
+
const newText = fullText.slice(reported.textLen);
|
|
200
|
+
if (newText.trim()) events.push({ type: "message", role: "assistant", text: newText });
|
|
201
|
+
reported.textLen = fullText.length;
|
|
202
|
+
return events;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// ───────────────────────── 工厂 ─────────────────────────
|
|
206
|
+
|
|
207
|
+
export interface UiMessageStreamAgentOptions {
|
|
208
|
+
/** agent 名(报告 / 结果聚合的身份)。默认 "ui-message-stream"。 */
|
|
209
|
+
name?: string;
|
|
210
|
+
/** 被测应用的 chat 端点(完整 URL,应用在哪部署就指哪);函数形式每轮解析。 */
|
|
211
|
+
url: string | ((ctx: AgentContext) => string | Promise<string>);
|
|
212
|
+
/** 附加请求头(鉴权等);`ctx.telemetry.headers`(traceparent)总会自动并入。 */
|
|
213
|
+
headers?: Record<string, string> | ((ctx: AgentContext) => Record<string, string>);
|
|
214
|
+
/** 除 `messages` 外并入请求体的字段,如 `(ctx) => ({ model: ctx.model })`(undefined 字段序列化时自动丢弃)。 */
|
|
215
|
+
body?: (ctx: AgentContext) => Record<string, JsonValue | undefined>;
|
|
216
|
+
/**
|
|
217
|
+
* 拒绝审批时随 `approval-responded` 带出的理由。应用/SDK 会把它作为模型看到的工具结果
|
|
218
|
+
* 文本 —— 写清楚「不要重试」能明显降低模型原样重发同一调用的概率(实测)。
|
|
219
|
+
*/
|
|
220
|
+
denyReason?: string;
|
|
221
|
+
/** 流结束后再等这么久才返回(毫秒),给应用侧的观测导出(如 BatchSpanProcessor)留时间。 */
|
|
222
|
+
settleMs?: number;
|
|
223
|
+
/** 应用有 OTel 时的端点投递方式(拿瀑布图);事件流不依赖它。 */
|
|
224
|
+
tracing?: AgentTracing;
|
|
225
|
+
spanMapper?: SpanMapper;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const DEFAULT_DENY_REASON = "用户拒绝了这次调用,不要重试,直接告知用户操作未执行。";
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* UI Message Stream Protocol(AI SDK `useChat` 后端的标准 SSE 协议)的内置无侵入 adapter。
|
|
232
|
+
* 对着已部署应用的 HTTP 端点收发,不 import 应用代码:
|
|
233
|
+
*
|
|
234
|
+
* ```typescript
|
|
235
|
+
* import { uiMessageStreamAgent } from "niceeval/adapter";
|
|
236
|
+
*
|
|
237
|
+
* export default uiMessageStreamAgent({
|
|
238
|
+
* name: "my-assistant",
|
|
239
|
+
* url: "https://my-app.example.com/api/chat",
|
|
240
|
+
* body: (ctx) => ({ model: ctx.model }), // 应用支持请求级选模型时,模型对比零改动
|
|
241
|
+
* });
|
|
242
|
+
* ```
|
|
243
|
+
*/
|
|
244
|
+
export function uiMessageStreamAgent(options: UiMessageStreamAgentOptions): Agent {
|
|
245
|
+
return defineAgent({
|
|
246
|
+
name: options.name ?? "ui-message-stream",
|
|
247
|
+
tracing: options.tracing,
|
|
248
|
+
spanMapper: options.spanMapper,
|
|
249
|
+
|
|
250
|
+
async send(input: TurnInput, ctx: AgentContext) {
|
|
251
|
+
const { readUIMessageStream } = await loadAi();
|
|
252
|
+
|
|
253
|
+
// 会话续接是「客户端带全量历史」模式:历史槽直接挂在 ctx.session 上,新线自然为空。
|
|
254
|
+
// chat id(useChat 协议要求请求带,每条会话线固定一个)和 reported(HITL 续跑时跨轮
|
|
255
|
+
// 去重要用)是本协议特有的簿记,存进 ctx.session.state(逃生舱,随会话线创建/丢弃)。
|
|
256
|
+
const history = ctx.session.history<UIMessageLike>();
|
|
257
|
+
const priorMessages = history.get();
|
|
258
|
+
const bookkeeping = ctx.session.state as { chatId?: string; reported?: ReportedState };
|
|
259
|
+
let id = bookkeeping.chatId;
|
|
260
|
+
if (!id) {
|
|
261
|
+
id = `uims-${randomUUID()}`;
|
|
262
|
+
bookkeeping.chatId = id;
|
|
263
|
+
ctx.session.capture(id); // 镜像:t.sessionId / 报告可见
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const lastMessage = priorMessages.at(-1);
|
|
267
|
+
const pendingPart =
|
|
268
|
+
lastMessage?.role === "assistant" ? lastMessage.parts.find(isApprovalRequested) : undefined;
|
|
269
|
+
|
|
270
|
+
let messagesToSend: UIMessageLike[];
|
|
271
|
+
let resumeFrom: UIMessageLike | undefined;
|
|
272
|
+
|
|
273
|
+
if (pendingPart && lastMessage) {
|
|
274
|
+
// HITL 续跑:不追加新 user 消息 —— 把停在 approval-requested 的 part 原地改成
|
|
275
|
+
// approval-responded,原样重发,服务端续跑同一条被打断的 assistant 消息。
|
|
276
|
+
// 裁决优先按 requestId 从 input.responses 读 optionId;没有 responses 才退回猜文本。
|
|
277
|
+
const requestId = pendingPart.approval?.id ?? pendingPart.toolCallId;
|
|
278
|
+
const approved = approvedFromResponse(input.responses, requestId, input.text);
|
|
279
|
+
const mutatedParts = lastMessage.parts.map((part) =>
|
|
280
|
+
isApprovalRequested(part) && part.approval?.id === pendingPart.approval?.id
|
|
281
|
+
? {
|
|
282
|
+
...part,
|
|
283
|
+
state: "approval-responded",
|
|
284
|
+
approval: {
|
|
285
|
+
id: pendingPart.approval!.id,
|
|
286
|
+
approved,
|
|
287
|
+
...(approved ? {} : { reason: options.denyReason ?? DEFAULT_DENY_REASON }),
|
|
288
|
+
},
|
|
289
|
+
}
|
|
290
|
+
: part,
|
|
291
|
+
);
|
|
292
|
+
resumeFrom = { ...lastMessage, parts: mutatedParts };
|
|
293
|
+
messagesToSend = [...priorMessages.slice(0, -1), resumeFrom];
|
|
294
|
+
} else {
|
|
295
|
+
const parts: UIMessagePartLike[] = [{ type: "text", text: input.text }];
|
|
296
|
+
for (const f of input.files ?? []) {
|
|
297
|
+
parts.push({ type: "file", mediaType: f.mimeType, url: `data:${f.mimeType};base64,${f.dataBase64}` });
|
|
298
|
+
}
|
|
299
|
+
messagesToSend = [...priorMessages, { id: randomUUID(), role: "user", parts }];
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const url = typeof options.url === "function" ? await options.url(ctx) : options.url;
|
|
303
|
+
const extraHeaders = typeof options.headers === "function" ? options.headers(ctx) : (options.headers ?? {});
|
|
304
|
+
let res: Response;
|
|
305
|
+
try {
|
|
306
|
+
res = await fetch(url, {
|
|
307
|
+
method: "POST",
|
|
308
|
+
// traceparent 随请求带过去:应用埋点支持 context 传播时,span 归属精确到本轮。
|
|
309
|
+
headers: { "content-type": "application/json", ...extraHeaders, ...ctx.telemetry?.headers },
|
|
310
|
+
body: JSON.stringify({ ...options.body?.(ctx), messages: messagesToSend }),
|
|
311
|
+
signal: ctx.signal,
|
|
312
|
+
});
|
|
313
|
+
} catch (err) {
|
|
314
|
+
if (ctx.signal.aborted) throw err;
|
|
315
|
+
const cause = err instanceof Error ? (err.cause instanceof Error ? err.cause.message : err.message) : String(err);
|
|
316
|
+
throw new Error(`连不上 ${url}(${cause})。被测应用在跑吗?先按它自己的方式启动,或用配置把 url 指向已部署实例。`);
|
|
317
|
+
}
|
|
318
|
+
if (!res.ok || !res.body) {
|
|
319
|
+
throw new Error(
|
|
320
|
+
`POST ${url} 失败:${res.status} ${await res.text().catch(() => "")}。确认应用在跑、端点是 UI Message Stream 协议(useChat 的后端)。`,
|
|
321
|
+
);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
let sawError: string | undefined;
|
|
325
|
+
const chunkStream = toChunkStream(res.body, (c) => {
|
|
326
|
+
if (c.type === "error") sawError = c.errorText;
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
let finalMessage: UIMessageLike | undefined;
|
|
330
|
+
for await (const msg of readUIMessageStream({ message: resumeFrom, stream: chunkStream })) {
|
|
331
|
+
finalMessage = msg;
|
|
332
|
+
}
|
|
333
|
+
if (!finalMessage) {
|
|
334
|
+
throw new Error(`POST ${url} 的流结束了但一条 assistant 消息都没归约出来 —— 端点吐的不是 UI Message Stream 帧?`);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// 续跑轮:finalMessage 是同一条消息的完整版,替换末尾半成品;全新轮:追加。
|
|
338
|
+
history.commit(
|
|
339
|
+
resumeFrom ? [...messagesToSend.slice(0, -1), finalMessage] : [...messagesToSend, finalMessage],
|
|
340
|
+
);
|
|
341
|
+
|
|
342
|
+
const reported: ReportedState = resumeFrom && bookkeeping.reported ? bookkeeping.reported : { calls: new Set(), textLen: 0 };
|
|
343
|
+
bookkeeping.reported = reported;
|
|
344
|
+
const events = deriveTurnEvents(finalMessage, reported);
|
|
345
|
+
|
|
346
|
+
const request = finalMessage.parts.find(isApprovalRequested);
|
|
347
|
+
if (request) {
|
|
348
|
+
return {
|
|
349
|
+
status: "waiting" as const,
|
|
350
|
+
events: [
|
|
351
|
+
...events,
|
|
352
|
+
{
|
|
353
|
+
type: "input.requested" as const,
|
|
354
|
+
request: {
|
|
355
|
+
id: request.approval?.id ?? request.toolCallId,
|
|
356
|
+
action: toolNameOf(request),
|
|
357
|
+
input: (request.input ?? null) as JsonValue,
|
|
358
|
+
options: [{ id: "approve" }, { id: "deny" }],
|
|
359
|
+
},
|
|
360
|
+
},
|
|
361
|
+
],
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
if (options.settleMs) await new Promise((resolve) => setTimeout(resolve, options.settleMs));
|
|
366
|
+
return {
|
|
367
|
+
status: sawError ? ("failed" as const) : ("completed" as const),
|
|
368
|
+
events: [...events, ...(sawError ? [{ type: "error" as const, message: sawError }] : [])],
|
|
369
|
+
};
|
|
370
|
+
},
|
|
371
|
+
});
|
|
372
|
+
}
|