niceeval 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +13 -13
- package/docs/README.md +119 -0
- package/docs/adapters/README.md +60 -0
- package/docs/adapters/authoring.md +179 -0
- package/docs/adapters/coding-agent-skills-plugins.md +324 -0
- package/docs/adapters/collection.md +128 -0
- package/docs/adapters/contract.md +263 -0
- package/docs/adapters/reference/agent-eval.md +215 -0
- package/docs/adapters/reference/agent-loop-apis.md +69 -0
- package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
- package/docs/adapters/reference/eve-protocol.md +127 -0
- package/docs/adapters/reference/otel-genai.md +107 -0
- package/docs/adapters/reference/otel-instrumentation.md +65 -0
- package/docs/adapters/targets.md +99 -0
- package/docs/architecture.md +140 -0
- package/docs/assertions.md +388 -0
- package/docs/capabilities-by-construction.md +48 -0
- package/docs/cli.md +171 -0
- package/docs/concepts.md +106 -0
- package/docs/e2e-ci.md +295 -0
- package/docs/eval-authoring.md +319 -0
- package/docs/experiments.md +154 -0
- package/docs/getting-started.md +224 -0
- package/docs/multi-agent.md +145 -0
- package/docs/observability.md +333 -0
- package/docs/origin-integration.md +195 -0
- package/docs/references.md +35 -0
- package/docs/results-format.md +228 -0
- package/docs/runner.md +104 -0
- package/docs/sandbox.md +276 -0
- package/docs/scoring.md +119 -0
- package/docs/source-map.md +106 -0
- package/docs/tier-sync.md +177 -0
- package/docs/view.md +157 -0
- package/docs/vision.md +88 -0
- package/package.json +35 -5
- package/src/agents/ai-sdk-otel.ts +0 -0
- package/src/agents/ai-sdk.test.ts +377 -0
- package/src/agents/ai-sdk.ts +577 -0
- package/src/agents/bub.ts +30 -37
- package/src/agents/claude-code.ts +7 -4
- package/src/agents/codex.ts +15 -10
- package/src/agents/index.ts +43 -1
- package/src/agents/sdk-streams.test.ts +145 -0
- package/src/agents/sdk-streams.ts +457 -0
- package/src/agents/shared.ts +77 -39
- package/src/agents/streaming.test.ts +152 -0
- package/src/agents/streaming.ts +195 -0
- package/src/agents/types.ts +218 -0
- package/src/agents/ui-message-stream.test.ts +249 -0
- package/src/agents/ui-message-stream.ts +372 -0
- package/src/cli.ts +124 -77
- package/src/context/context.test.ts +203 -7
- package/src/context/context.ts +158 -49
- package/src/context/session.test.ts +96 -0
- package/src/context/session.ts +119 -9
- package/src/context/types.ts +205 -0
- package/src/define.ts +4 -14
- package/src/expect/index.ts +8 -71
- package/src/i18n/core.ts +28 -0
- package/src/i18n/en.ts +47 -5
- package/src/i18n/index.ts +5 -17
- package/src/i18n/zh-CN.ts +47 -5
- package/src/index.ts +12 -0
- package/src/loaders/index.ts +8 -39
- package/src/o11y/otlp/canonical.ts +7 -6
- package/src/o11y/otlp/mappers/codex.ts +10 -8
- package/src/o11y/otlp/mappers/index.ts +9 -21
- package/src/o11y/otlp/receiver.ts +25 -7
- package/src/o11y/otlp/sandbox-receiver.ts +57 -21
- package/src/o11y/otlp/turn-otel.test.ts +110 -0
- package/src/o11y/otlp/turn-otel.ts +125 -0
- package/src/o11y/parsers/bub.ts +13 -11
- package/src/o11y/parsers/claude-code.ts +27 -51
- package/src/o11y/parsers/codex.ts +29 -46
- package/src/o11y/parsers/index.ts +5 -14
- package/src/o11y/tool-names.test.ts +61 -0
- package/src/o11y/tool-names.ts +74 -0
- package/src/o11y/types.ts +135 -0
- package/src/runner/attempt.ts +456 -0
- package/src/runner/fingerprint.ts +59 -0
- package/src/runner/remote-sandbox.ts +53 -0
- package/src/runner/report.ts +53 -0
- package/src/runner/reporters/artifacts.ts +25 -4
- package/src/runner/reporters/console.ts +2 -8
- package/src/runner/reporters/live.ts +2 -8
- package/src/runner/reporters/shared.ts +15 -0
- package/src/runner/reporters/table.ts +10 -62
- package/src/runner/run.ts +114 -619
- package/src/runner/types.ts +237 -0
- package/src/sandbox/checkpoint.ts +15 -13
- package/src/sandbox/docker-stream.ts +87 -0
- package/src/sandbox/docker.ts +42 -120
- package/src/sandbox/e2b.ts +24 -75
- package/src/sandbox/local-files.ts +33 -0
- package/src/sandbox/paths.test.ts +85 -0
- package/src/sandbox/paths.ts +45 -0
- package/src/sandbox/resolve.ts +10 -34
- package/src/sandbox/shell.ts +17 -0
- package/src/sandbox/source-files.ts +42 -2
- package/src/sandbox/types.ts +158 -0
- package/src/sandbox/vercel.ts +55 -71
- package/src/scoring/collector.ts +3 -2
- package/src/scoring/judge.ts +72 -146
- package/src/scoring/match.ts +79 -0
- package/src/scoring/types.ts +76 -0
- package/src/shared/aggregate.ts +48 -0
- package/src/shared/format.ts +20 -0
- package/src/shared/outcome.ts +48 -0
- package/src/shared/types.ts +37 -0
- package/src/types.ts +20 -911
- package/src/view/aggregate.ts +103 -0
- package/src/view/app/App.tsx +26 -5
- package/src/view/app/components/AttemptModal.tsx +12 -3
- package/src/view/app/components/CodeView.tsx +9 -5
- package/src/view/app/components/CopyControls.tsx +4 -4
- package/src/view/app/components/ExperimentTable.tsx +5 -5
- package/src/view/app/i18n.ts +31 -7
- package/src/view/app/lib/format.ts +17 -20
- package/src/view/app/lib/outcome.ts +4 -16
- package/src/view/app/main.tsx +3 -5
- package/src/view/app/pages/RunsPage.tsx +2 -2
- package/src/view/app/pages/TracesPage.tsx +2 -2
- package/src/view/app/shared.ts +2 -1
- package/src/view/app/types.ts +6 -43
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +24 -401
- package/src/view/loader.ts +234 -0
- package/src/view/server.ts +136 -0
- package/src/view/shared/types.ts +64 -0
- package/src/view/styles.css +60 -9
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
// AI SDK(Vercel `ai` 包)结果 → 标准事件流的共享转换器(采集通道 0:进程内直构)。
|
|
2
|
+
//
|
|
3
|
+
// 结构化 typing,不依赖 `ai` 包:niceeval 只认识 generateText / streamText 完整结果的
|
|
4
|
+
// 【形状子集】。字段名跨 AI SDK 大版本漂移(v4 args/result/promptTokens、v5+ input/output/
|
|
5
|
+
// inputTokens、v7 inputTokenDetails)在这里统一兜住,adapter 作者不必各自写防御代码,也不必
|
|
6
|
+
// 像手工 recorder 那样包住每个工具的 execute —— AI SDK 的 steps 里本来就有带 toolCallId 的
|
|
7
|
+
// 完整调用记录。
|
|
8
|
+
//
|
|
9
|
+
// v7 的 tool approval(工具带 `needsApproval`)也在这里兜:
|
|
10
|
+
// · `tool-approval-request` part → `input.requested` 事件 + 整轮 `status: "waiting"`,
|
|
11
|
+
// 直接满足 HITL 契约的「waiting + input.requested」两条义务(resume 仍归 adapter 管);
|
|
12
|
+
// · 批准 / 拒绝后的 resume 结果里,执行结果只出现在 `responseMessages` 的 tool 消息中
|
|
13
|
+
// (不在 steps 里),这里挖出来补成 `action.result` —— 拒绝(execution-denied)映射成
|
|
14
|
+
// `status: "rejected"`,喂 `calledTool(..., { status: "rejected" })` 与 `noFailedActions()`。
|
|
15
|
+
|
|
16
|
+
import { randomUUID } from "node:crypto";
|
|
17
|
+
|
|
18
|
+
import { defineAgent } from "../define.ts";
|
|
19
|
+
import { normalizeToolName as normalizeShared } from "../o11y/tool-names.ts";
|
|
20
|
+
import type { Agent, InputRequest, InputResponse, JsonValue, StreamEvent, ToolName, Usage } from "../types.ts";
|
|
21
|
+
|
|
22
|
+
// ───────────────────────── AI SDK 结果的形状子集 ─────────────────────────
|
|
23
|
+
|
|
24
|
+
/** 一次工具调用(v5+ 用 `input`,v4 用 `args`;两者都认)。 */
|
|
25
|
+
export interface AiSdkToolCallLike {
|
|
26
|
+
toolCallId: string;
|
|
27
|
+
toolName: string;
|
|
28
|
+
input?: unknown;
|
|
29
|
+
args?: unknown;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** 一次工具结果(v5+ 用 `output`,v4 用 `result`)。 */
|
|
33
|
+
export interface AiSdkToolResultLike {
|
|
34
|
+
toolCallId: string;
|
|
35
|
+
toolName?: string;
|
|
36
|
+
output?: unknown;
|
|
37
|
+
result?: unknown;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* step.content 的一个 part(v5+)。带类型序:同一 step 里 reasoning / text / tool-call /
|
|
42
|
+
* tool-result / tool-error / tool-approval-request 按真实发生顺序排 —— 有它就优先用它,时序保真。
|
|
43
|
+
*/
|
|
44
|
+
export interface AiSdkContentPartLike {
|
|
45
|
+
type: string;
|
|
46
|
+
text?: unknown;
|
|
47
|
+
toolCallId?: unknown;
|
|
48
|
+
toolName?: unknown;
|
|
49
|
+
input?: unknown;
|
|
50
|
+
args?: unknown;
|
|
51
|
+
output?: unknown;
|
|
52
|
+
result?: unknown;
|
|
53
|
+
error?: unknown;
|
|
54
|
+
/** v7 tool approval:`tool-approval-request` part 的请求 id 与被拦下的调用。 */
|
|
55
|
+
approvalId?: unknown;
|
|
56
|
+
isAutomatic?: unknown;
|
|
57
|
+
toolCall?: {
|
|
58
|
+
toolCallId?: unknown;
|
|
59
|
+
toolName?: unknown;
|
|
60
|
+
input?: unknown;
|
|
61
|
+
args?: unknown;
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** 一个 step = 一次模型调用(工具循环里的一圈)。 */
|
|
66
|
+
export interface AiSdkStepLike {
|
|
67
|
+
content?: readonly AiSdkContentPartLike[];
|
|
68
|
+
text?: string;
|
|
69
|
+
reasoningText?: string;
|
|
70
|
+
toolCalls?: readonly AiSdkToolCallLike[];
|
|
71
|
+
toolResults?: readonly AiSdkToolResultLike[];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface AiSdkUsageLike {
|
|
75
|
+
inputTokens?: number;
|
|
76
|
+
outputTokens?: number;
|
|
77
|
+
promptTokens?: number;
|
|
78
|
+
completionTokens?: number;
|
|
79
|
+
cachedInputTokens?: number;
|
|
80
|
+
/** v7:cache 细分挪进了 inputTokenDetails。 */
|
|
81
|
+
inputTokenDetails?: {
|
|
82
|
+
cacheReadTokens?: number;
|
|
83
|
+
cacheWriteTokens?: number;
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** result.responseMessages 的一条(只认 tool 消息里的 tool-result part,其余略过)。 */
|
|
88
|
+
export interface AiSdkResponseMessageLike {
|
|
89
|
+
role?: unknown;
|
|
90
|
+
content?: unknown;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** generateText / streamText 完整结果的形状子集。没有 steps 的老结果退回顶层字段。 */
|
|
94
|
+
export interface AiSdkResultLike {
|
|
95
|
+
text?: string;
|
|
96
|
+
steps?: readonly AiSdkStepLike[];
|
|
97
|
+
/** v7:跨全部 step 聚合的 content parts(approval 请求也在里面)。 */
|
|
98
|
+
content?: readonly AiSdkContentPartLike[];
|
|
99
|
+
toolCalls?: readonly AiSdkToolCallLike[];
|
|
100
|
+
toolResults?: readonly AiSdkToolResultLike[];
|
|
101
|
+
/** v5/v6 聚合全部 step 的用量;优先于 usage(v6 的 usage 只是最后一个 step,v7 起也是聚合)。 */
|
|
102
|
+
totalUsage?: AiSdkUsageLike;
|
|
103
|
+
usage?: AiSdkUsageLike;
|
|
104
|
+
/**
|
|
105
|
+
* 本次调用新产生的消息(v7)。tool approval 批准 / 拒绝后,被拦工具的执行结果只出现在
|
|
106
|
+
* 这里的 tool 消息里(不进 steps),fromAiSdk 从中补齐 `action.result`。
|
|
107
|
+
*/
|
|
108
|
+
responseMessages?: readonly AiSdkResponseMessageLike[];
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** fromAiSdk 的返回:铺进 `Turn` 即可(data 仍归调用方 —— 结构化输出是应用自己的语义)。 */
|
|
112
|
+
export interface AiSdkTurn {
|
|
113
|
+
events: StreamEvent[];
|
|
114
|
+
usage?: Usage;
|
|
115
|
+
/** 有待人批准的工具调用(非 automatic 的 tool-approval-request)→ "waiting",否则 "completed"。 */
|
|
116
|
+
status: "completed" | "waiting";
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// ───────────────────────── 转换 ─────────────────────────
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* AI SDK 结果 → `{ events, usage, status }`,直接铺进 `Turn` 返回:
|
|
123
|
+
*
|
|
124
|
+
* ```typescript
|
|
125
|
+
* const result = await generateText({ model, tools, prompt: input.text });
|
|
126
|
+
* return { ...fromAiSdk(result), data: result.text };
|
|
127
|
+
* ```
|
|
128
|
+
*
|
|
129
|
+
* `callId` 用 AI SDK 原生的 `toolCallId`(显式配对,不合成);工具名保留原名进 `name`,
|
|
130
|
+
* canonical 名进 `tool`(认不出的域内工具落 "unknown",`calledTool("get_weather")`
|
|
131
|
+
* 仍按原名匹配)。工具执行失败(v5+ 的 `tool-error` part)映射成 `status: "failed"` 的
|
|
132
|
+
* `action.result`,喂 `noFailedActions()`。
|
|
133
|
+
*
|
|
134
|
+
* v7 tool approval:`needsApproval` 工具被拦下时返回 `status: "waiting"` + 每个待批准
|
|
135
|
+
* 调用一条 `input.requested`(`id` = approvalId、`action` = 工具原名、`options` =
|
|
136
|
+
* approve / deny)。resume(把 `tool-approval-response` 塞回 messages 再跑一次)后的结果
|
|
137
|
+
* 再交给 fromAiSdk,会从 `responseMessages` 里补出执行 / 拒绝的 `action.result`。
|
|
138
|
+
*/
|
|
139
|
+
export function fromAiSdk(result: AiSdkResultLike): AiSdkTurn {
|
|
140
|
+
const stepEvents: StreamEvent[] = [];
|
|
141
|
+
const steps: readonly AiSdkStepLike[] =
|
|
142
|
+
result.steps && result.steps.length > 0
|
|
143
|
+
? result.steps
|
|
144
|
+
: [{ text: result.text, toolCalls: result.toolCalls, toolResults: result.toolResults }];
|
|
145
|
+
|
|
146
|
+
const seen = { called: new Set<string>(), resolved: new Set<string>() };
|
|
147
|
+
for (const step of steps) {
|
|
148
|
+
if (Array.isArray(step.content) && step.content.length > 0) {
|
|
149
|
+
pushContentParts(stepEvents, step.content, seen);
|
|
150
|
+
} else {
|
|
151
|
+
pushStepFields(stepEvents, step, seen);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// approval resume 时被拦工具的执行结果只在 responseMessages 里;补出的 action.result
|
|
156
|
+
// 排在本轮 step 事件之前 —— 时间上它确实先于模型看到结果后的新输出。
|
|
157
|
+
const minedEvents = mineResponseMessages(result.responseMessages, seen.resolved);
|
|
158
|
+
|
|
159
|
+
const events = [...minedEvents, ...stepEvents];
|
|
160
|
+
const waiting = events.some((e) => e.type === "input.requested");
|
|
161
|
+
return { events, usage: readUsage(result, steps.length), status: waiting ? "waiting" : "completed" };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
interface SeenCallIds {
|
|
165
|
+
called: Set<string>;
|
|
166
|
+
resolved: Set<string>;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** v5+ 路径:content parts 自带真实顺序,逐个翻译。 */
|
|
170
|
+
function pushContentParts(
|
|
171
|
+
events: StreamEvent[],
|
|
172
|
+
parts: readonly AiSdkContentPartLike[],
|
|
173
|
+
seen: SeenCallIds,
|
|
174
|
+
): void {
|
|
175
|
+
for (const part of parts) {
|
|
176
|
+
switch (part.type) {
|
|
177
|
+
case "text": {
|
|
178
|
+
const text = str(part.text);
|
|
179
|
+
if (text) events.push({ type: "message", role: "assistant", text });
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
182
|
+
case "reasoning": {
|
|
183
|
+
const text = str(part.text);
|
|
184
|
+
if (text) events.push({ type: "thinking", text });
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
187
|
+
case "tool-call": {
|
|
188
|
+
pushCalled(events, seen, {
|
|
189
|
+
toolCallId: str(part.toolCallId) ?? "unknown",
|
|
190
|
+
toolName: str(part.toolName) ?? "unknown",
|
|
191
|
+
input: part.input ?? part.args,
|
|
192
|
+
});
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
case "tool-result": {
|
|
196
|
+
const callId = str(part.toolCallId) ?? "unknown";
|
|
197
|
+
seen.resolved.add(callId);
|
|
198
|
+
events.push({
|
|
199
|
+
type: "action.result",
|
|
200
|
+
callId,
|
|
201
|
+
output: asJson(part.output ?? part.result),
|
|
202
|
+
status: "completed",
|
|
203
|
+
});
|
|
204
|
+
break;
|
|
205
|
+
}
|
|
206
|
+
case "tool-error": {
|
|
207
|
+
const callId = str(part.toolCallId) ?? "unknown";
|
|
208
|
+
seen.resolved.add(callId);
|
|
209
|
+
events.push({
|
|
210
|
+
type: "action.result",
|
|
211
|
+
callId,
|
|
212
|
+
output: { error: part.error instanceof Error ? part.error.message : String(part.error) },
|
|
213
|
+
status: "failed",
|
|
214
|
+
});
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
case "tool-approval-request": {
|
|
218
|
+
// 同一 step 里通常已有配对的 tool-call part(去重);没有时从 toolCall 字段补。
|
|
219
|
+
const call = part.toolCall;
|
|
220
|
+
if (call) {
|
|
221
|
+
pushCalled(events, seen, {
|
|
222
|
+
toolCallId: str(call.toolCallId) ?? "unknown",
|
|
223
|
+
toolName: str(call.toolName) ?? "unknown",
|
|
224
|
+
input: call.input ?? call.args,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
if (part.isAutomatic === true) break; // SDK 自动裁决,不用等人
|
|
228
|
+
const request: InputRequest = {
|
|
229
|
+
id: str(part.approvalId),
|
|
230
|
+
action: call ? str(call.toolName) : undefined,
|
|
231
|
+
input: call ? asJson(call.input ?? call.args) : undefined,
|
|
232
|
+
options: [{ id: "approve" }, { id: "deny" }],
|
|
233
|
+
};
|
|
234
|
+
events.push({ type: "input.requested", request });
|
|
235
|
+
break;
|
|
236
|
+
}
|
|
237
|
+
default:
|
|
238
|
+
break; // source / file 等其余 part 类型对断言无意义,丢弃
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** 退路(v4 / 无 content parts):toolCalls + toolResults + text,顺序按「调用 → 结果 → 文本」近似。 */
|
|
244
|
+
function pushStepFields(events: StreamEvent[], step: AiSdkStepLike, seen: SeenCallIds): void {
|
|
245
|
+
if (step.reasoningText) events.push({ type: "thinking", text: step.reasoningText });
|
|
246
|
+
for (const call of step.toolCalls ?? []) {
|
|
247
|
+
pushCalled(events, seen, { toolCallId: call.toolCallId, toolName: call.toolName, input: call.input ?? call.args });
|
|
248
|
+
}
|
|
249
|
+
for (const res of step.toolResults ?? []) {
|
|
250
|
+
seen.resolved.add(res.toolCallId);
|
|
251
|
+
events.push({
|
|
252
|
+
type: "action.result",
|
|
253
|
+
callId: res.toolCallId,
|
|
254
|
+
output: asJson(res.output ?? res.result),
|
|
255
|
+
status: "completed",
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
if (step.text?.trim()) events.push({ type: "message", role: "assistant", text: step.text });
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** 同一 callId 只发一条 action.called(tool-call part 与 approval-request 的 toolCall 会重)。 */
|
|
262
|
+
function pushCalled(
|
|
263
|
+
events: StreamEvent[],
|
|
264
|
+
seen: SeenCallIds,
|
|
265
|
+
call: { toolCallId: string; toolName: string; input: unknown },
|
|
266
|
+
): void {
|
|
267
|
+
if (seen.called.has(call.toolCallId)) return;
|
|
268
|
+
seen.called.add(call.toolCallId);
|
|
269
|
+
events.push({
|
|
270
|
+
type: "action.called",
|
|
271
|
+
callId: call.toolCallId,
|
|
272
|
+
name: call.toolName,
|
|
273
|
+
input: asJson(call.input),
|
|
274
|
+
tool: normalizeToolName(call.toolName),
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* 从 responseMessages 的 tool 消息里挖 steps 没有的 tool-result(v7 approval resume 的
|
|
280
|
+
* 执行结果只在这里)。output 是 ToolResultOutput 包装({ type: "json" | "text" |
|
|
281
|
+
* "execution-denied" | "error-text" | … }),按类型解包;`execution-denied` → "rejected"。
|
|
282
|
+
*/
|
|
283
|
+
function mineResponseMessages(
|
|
284
|
+
messages: readonly AiSdkResponseMessageLike[] | undefined,
|
|
285
|
+
resolved: ReadonlySet<string>,
|
|
286
|
+
): StreamEvent[] {
|
|
287
|
+
const events: StreamEvent[] = [];
|
|
288
|
+
for (const msg of messages ?? []) {
|
|
289
|
+
if (msg.role !== "tool" || !Array.isArray(msg.content)) continue;
|
|
290
|
+
for (const part of msg.content as AiSdkContentPartLike[]) {
|
|
291
|
+
if (part.type !== "tool-result") continue;
|
|
292
|
+
const callId = str(part.toolCallId);
|
|
293
|
+
if (!callId || resolved.has(callId)) continue;
|
|
294
|
+
events.push({ type: "action.result", callId, ...unwrapToolOutput(part.output) });
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return events;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function unwrapToolOutput(output: unknown): { output?: JsonValue; status: "completed" | "failed" | "rejected" } {
|
|
301
|
+
if (output && typeof output === "object" && !Array.isArray(output)) {
|
|
302
|
+
const wrapped = output as { type?: unknown; value?: unknown; reason?: unknown };
|
|
303
|
+
switch (wrapped.type) {
|
|
304
|
+
case "json":
|
|
305
|
+
case "text":
|
|
306
|
+
case "content":
|
|
307
|
+
return { output: asJson(wrapped.value), status: "completed" };
|
|
308
|
+
case "execution-denied":
|
|
309
|
+
return {
|
|
310
|
+
output: str(wrapped.reason) ? { reason: str(wrapped.reason)! } : undefined,
|
|
311
|
+
status: "rejected",
|
|
312
|
+
};
|
|
313
|
+
case "error-text":
|
|
314
|
+
case "error-json":
|
|
315
|
+
return { output: { error: asJson(wrapped.value) }, status: "failed" };
|
|
316
|
+
default:
|
|
317
|
+
break;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
return { output: asJson(output), status: "completed" };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** totalUsage(v5/v6 的全 steps 聚合)优先;v7 的 usage 本身就是聚合;requests = step 数。 */
|
|
324
|
+
function readUsage(result: AiSdkResultLike, stepCount: number): Usage | undefined {
|
|
325
|
+
const u = result.totalUsage ?? result.usage;
|
|
326
|
+
if (!u) return undefined;
|
|
327
|
+
const inputTokens = num(u.inputTokens) ?? num(u.promptTokens) ?? 0;
|
|
328
|
+
const outputTokens = num(u.outputTokens) ?? num(u.completionTokens) ?? 0;
|
|
329
|
+
if (inputTokens === 0 && outputTokens === 0) return undefined;
|
|
330
|
+
const usage: Usage = { inputTokens, outputTokens, requests: Math.max(stepCount, 1) };
|
|
331
|
+
const cacheRead = num(u.cachedInputTokens) ?? num(u.inputTokenDetails?.cacheReadTokens);
|
|
332
|
+
if (cacheRead) usage.cacheReadTokens = cacheRead;
|
|
333
|
+
const cacheWrite = num(u.inputTokenDetails?.cacheWriteTokens);
|
|
334
|
+
if (cacheWrite) usage.cacheWriteTokens = cacheWrite;
|
|
335
|
+
return usage;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** AI SDK 应用的工具多为域内自定义名(get_weather…),canonical 落 "unknown" 即可;仅认通用别名基表。 */
|
|
339
|
+
function normalizeToolName(name: string): ToolName {
|
|
340
|
+
return normalizeShared(name);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** 工具入参 / 出参在 AI SDK 里经 schema 校验,本就是 JSON 值;这里只做形状断言。 */
|
|
344
|
+
function asJson(value: unknown): JsonValue {
|
|
345
|
+
return (value ?? null) as JsonValue;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function str(value: unknown): string | undefined {
|
|
349
|
+
return typeof value === "string" && value ? value : undefined;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function num(value: unknown): number | undefined {
|
|
353
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// ───────────────────────── 内建 agent 工厂 ─────────────────────────
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* `generate` 收到的上下文。`messages` 是本会话的完整 ModelMessage 历史(含本轮的用户消息 /
|
|
360
|
+
* approval 裁决),直接透传给 generateText / streamText 即可;泛型 `M` 由调用方钉成自己
|
|
361
|
+
* AI SDK 版本的 `ModelMessage`,niceeval 不引 `ai` 包。
|
|
362
|
+
*/
|
|
363
|
+
export interface AiSdkGenerateContext<M = unknown> {
|
|
364
|
+
readonly messages: M[];
|
|
365
|
+
/** 实验钉的 model(ctx.model);省略 → 用应用自己的默认。 */
|
|
366
|
+
readonly model?: string;
|
|
367
|
+
readonly signal: AbortSignal;
|
|
368
|
+
readonly flags: Readonly<Record<string, unknown>>;
|
|
369
|
+
/**
|
|
370
|
+
* 开了 `tracing: true` 才有:直接放进 generateText / streamText 的 `telemetry`
|
|
371
|
+
* 选项。OTel provider、per-attempt 端点绑定和轮末 flush 都由工厂做,应用侧原样透传即可。
|
|
372
|
+
*/
|
|
373
|
+
readonly telemetry?: AiSdkTelemetrySettings;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* generateText / streamText 的 `telemetry` 选项的形状子集。integrations 的元素是
|
|
378
|
+
* `@ai-sdk/otel` 的集成实例——类型属于用户的 ai 版本,niceeval 不引 `ai` 包,所以是 any。
|
|
379
|
+
*/
|
|
380
|
+
export interface AiSdkTelemetrySettings {
|
|
381
|
+
readonly integrations: any[];
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
export interface AiSdkAgentOptions<M = unknown> {
|
|
385
|
+
/** agent 名(报告 / 结果聚合的身份)。默认 "ai-sdk"。 */
|
|
386
|
+
name?: string;
|
|
387
|
+
/**
|
|
388
|
+
* 开 OTel 管线(拿 `niceeval view` 的瀑布图)。true 时运行器为这个 agent 开一个
|
|
389
|
+
* per-attempt OTLP 接收器,`ctx.telemetry` 才会出现;工厂据此建好绑定接收端点的
|
|
390
|
+
* `@ai-sdk/otel` 集成(经 `generate` 的 ctx.telemetry 交给应用,原样透传给
|
|
391
|
+
* generateText / streamText 的 `telemetry` 选项即可)。省略或 false 则整个 OTel
|
|
392
|
+
* 管线不开,`generate` 拿到的 `telemetry` 恒为 undefined,应用侧零开销。
|
|
393
|
+
*/
|
|
394
|
+
tracing?: boolean;
|
|
395
|
+
/**
|
|
396
|
+
* 每轮一召:拿会话历史跑一次 generateText / streamText(await 完整结果)并原样返回。
|
|
397
|
+
* model / tools / system prompt / stopWhen 都在这里配 —— 那是应用的事,工厂不掺和。
|
|
398
|
+
*/
|
|
399
|
+
generate(ctx: AiSdkGenerateContext<M>): Promise<AiSdkResultLike>;
|
|
400
|
+
/** 本轮的结构化输出(Turn.data,喂 outputEquals / outputMatches)。省略则 data 为 undefined。 */
|
|
401
|
+
data?(result: AiSdkResultLike, turn: AiSdkTurn): unknown;
|
|
402
|
+
/**
|
|
403
|
+
* 可选双发:tracing 的 span 除发给 niceeval 的接收器外,同一批也发到你自己的 OTLP
|
|
404
|
+
* 后端(Langfuse / SigNoz / 生产 collector)。只在 `tracing: true` 时生效。
|
|
405
|
+
*/
|
|
406
|
+
otlpBackendUrl?: string;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* 内建的 AI SDK 进程内 agent 工厂:把「一个 generateText 调用」变成完整的 niceeval agent。
|
|
411
|
+
* 应用只写 `generate`(怎么召模型),协议侧的活全部由工厂承担:
|
|
412
|
+
*
|
|
413
|
+
* · 多轮会话:ctx.session.id 未记录时开新会话线并 capture 回写,同一 id 续接同一份
|
|
414
|
+
* messages 历史;
|
|
415
|
+
* · 事件流:结果经 {@link fromAiSdk} 直构(toolCallId 配对、时序保真、usage);
|
|
416
|
+
* · HITL:`needsApproval` 工具停轮 → `waiting` + `input.requested`;下一轮输入按行翻译成
|
|
417
|
+
* tool-approval-response(以 approve / yes / 同意 / 批准 开头 = 批准,其余一律拒绝)塞回
|
|
418
|
+
* messages 再召 `generate`,SDK 才会执行(或跳过)被拦的工具;
|
|
419
|
+
* · 失败兜底:`generate` 抛错或结果完全为空 → `status: "failed"` + error 事件;
|
|
420
|
+
* · tracing:开 `tracing: true` 后,工厂替应用做完 OTel 管线——为每轮建好绑定 niceeval
|
|
421
|
+
* 接收端点的 `@ai-sdk/otel` 集成(经 ctx.telemetry 交给 generate,原样透传给
|
|
422
|
+
* generateText 的 `telemetry` 即可)并在轮末 flush;`otlpBackendUrl` 可选
|
|
423
|
+
* 双发到你自己的观测后端。应用侧零埋点代码。
|
|
424
|
+
*
|
|
425
|
+
* ```typescript
|
|
426
|
+
* import { aiSdkAgent } from "niceeval/adapter";
|
|
427
|
+
* import { generateText, isStepCount, type ModelMessage } from "ai";
|
|
428
|
+
*
|
|
429
|
+
* export const assistant = aiSdkAgent<ModelMessage>({
|
|
430
|
+
* name: "my-assistant",
|
|
431
|
+
* generate: ({ messages, model, signal }) =>
|
|
432
|
+
* generateText({ model: resolveModel(model), system: SYSTEM_PROMPT,
|
|
433
|
+
* tools, stopWhen: isStepCount(5), messages, abortSignal: signal }),
|
|
434
|
+
* data: (result) => ({ reply: result.text }),
|
|
435
|
+
* });
|
|
436
|
+
* ```
|
|
437
|
+
*/
|
|
438
|
+
let otelModule: Promise<typeof import("./ai-sdk-otel.ts")> | undefined;
|
|
439
|
+
|
|
440
|
+
/** OTel 管线按需加载:三个 OTel 包是可选 peer,只有开了 tracing 的用户需要装。 */
|
|
441
|
+
function loadOtel(): Promise<typeof import("./ai-sdk-otel.ts")> {
|
|
442
|
+
otelModule ??= import("./ai-sdk-otel.ts").catch((error: unknown) => {
|
|
443
|
+
otelModule = undefined;
|
|
444
|
+
throw new Error(
|
|
445
|
+
"tracing: true 需要 OTel 依赖:请在被测项目里安装 @ai-sdk/otel、@opentelemetry/sdk-trace-node、@opentelemetry/exporter-trace-otlp-http(niceeval 的可选 peer 依赖)。",
|
|
446
|
+
{ cause: error },
|
|
447
|
+
);
|
|
448
|
+
});
|
|
449
|
+
return otelModule;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
export function aiSdkAgent<M = unknown>(options: AiSdkAgentOptions<M>): Agent {
|
|
453
|
+
interface SessionState {
|
|
454
|
+
messages: M[];
|
|
455
|
+
/** 上一轮停下的 tool approval:下一轮输入按行翻译成裁决。 */
|
|
456
|
+
pendingApprovals: string[];
|
|
457
|
+
}
|
|
458
|
+
const sessions = new Map<string, SessionState>();
|
|
459
|
+
|
|
460
|
+
return defineAgent({
|
|
461
|
+
name: options.name ?? "ai-sdk",
|
|
462
|
+
// tracing 开了才让运行器为这个 agent 起 OTLP 接收器(ctx.telemetry 才会出现);
|
|
463
|
+
// AgentTracing 的其余字段(env/configure/scope)这里都用不上,空对象即可。
|
|
464
|
+
tracing: options.tracing ? {} : undefined,
|
|
465
|
+
|
|
466
|
+
async send(input, ctx) {
|
|
467
|
+
// 会话续接:ctx.session.id 未记录时开新会话并 capture 回写;否则按 id 续接同一份历史。
|
|
468
|
+
const id = ctx.session.id ?? `ai-sdk-${randomUUID()}`;
|
|
469
|
+
ctx.session.capture(id);
|
|
470
|
+
let state = sessions.get(id);
|
|
471
|
+
if (!state) {
|
|
472
|
+
state = { messages: [], pendingApprovals: [] };
|
|
473
|
+
sessions.set(id, state);
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (state.pendingApprovals.length > 0) {
|
|
477
|
+
// HITL:t.respond 的回答到 adapter 就是一次普通的带 resume 的 send。
|
|
478
|
+
// 优先按 requestId(= approvalId)从 input.responses 里对位取裁决;没有匹配的
|
|
479
|
+
// responses(旧 adapter 用法 / 手写 send 未接结构化回答)才退回逐行文本猜
|
|
480
|
+
// (行数不够沿用最后一行)——内置件要防御式兼容,回退路径保留。
|
|
481
|
+
const lines = input.text.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
482
|
+
const content = state.pendingApprovals.map((approvalId, i) => ({
|
|
483
|
+
type: "tool-approval-response" as const,
|
|
484
|
+
approvalId,
|
|
485
|
+
approved: approvedFromResponse(input.responses, approvalId, lines[Math.min(i, lines.length - 1)] ?? ""),
|
|
486
|
+
}));
|
|
487
|
+
state.pendingApprovals = [];
|
|
488
|
+
state.messages.push({ role: "tool", content } as M);
|
|
489
|
+
} else {
|
|
490
|
+
state.messages.push(userMessage(input.text, input.files) as M);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// tracing 开了才建管线;OTel peer 依赖缺失在这里就报清楚,不伪装成失败的 Turn。
|
|
494
|
+
const otel = ctx.telemetry ? (await loadOtel()).telemetryForEndpoint(ctx.telemetry.endpoint, options.otlpBackendUrl) : undefined;
|
|
495
|
+
|
|
496
|
+
let result: AiSdkResultLike;
|
|
497
|
+
try {
|
|
498
|
+
result = await options.generate({
|
|
499
|
+
messages: state.messages,
|
|
500
|
+
model: ctx.model,
|
|
501
|
+
signal: ctx.signal,
|
|
502
|
+
flags: ctx.flags,
|
|
503
|
+
telemetry: otel?.settings,
|
|
504
|
+
});
|
|
505
|
+
} catch (error) {
|
|
506
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
507
|
+
return { status: "failed", events: [{ type: "error", message }] };
|
|
508
|
+
} finally {
|
|
509
|
+
// 轮次归属靠时间窗口:本轮 span 必须在 send 返回前送到接收器,不能等 batch。
|
|
510
|
+
await otel?.flush();
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// resume 的另一半义务:本次调用新产生的消息(含 approval 执行结果)进历史。
|
|
514
|
+
state.messages.push(...((result.responseMessages ?? []) as M[]));
|
|
515
|
+
state.pendingApprovals = collectPendingApprovals(result);
|
|
516
|
+
|
|
517
|
+
const turn = fromAiSdk(result);
|
|
518
|
+
// 上游偶尔退化返回完全空的结果;当正常回复会把故障伪装成通过,按失败处理。
|
|
519
|
+
if (turn.events.length === 0) {
|
|
520
|
+
return { status: "failed", events: [{ type: "error", message: "AI SDK 返回了空结果(无文本、无工具调用)" }] };
|
|
521
|
+
}
|
|
522
|
+
return { ...turn, data: options.data?.(result, turn) };
|
|
523
|
+
},
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
function isApproved(line: string): boolean {
|
|
528
|
+
return /^(approve|yes|同意|批准)/i.test(line);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* 优先按 requestId(= approvalId)从 input.responses 里取结构化裁决(optionId === "approve");
|
|
533
|
+
* 拿不到匹配的 responses 才退回从整行文本猜(见 isApproved)——防御式兼容旧 adapter 用法,
|
|
534
|
+
* 不删旧分支。
|
|
535
|
+
*/
|
|
536
|
+
function approvedFromResponse(
|
|
537
|
+
responses: readonly InputResponse[] | undefined,
|
|
538
|
+
requestId: string,
|
|
539
|
+
fallbackText: string,
|
|
540
|
+
): boolean {
|
|
541
|
+
const matched = responses?.find((r) => r.requestId === requestId);
|
|
542
|
+
if (matched?.optionId !== undefined) return matched.optionId === "approve";
|
|
543
|
+
if (matched?.text !== undefined) return isApproved(matched.text);
|
|
544
|
+
return isApproved(fallbackText);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/** 待人批准的 approval 请求 id(非 automatic)。steps 优先,退回 v7 顶层聚合 content。 */
|
|
548
|
+
function collectPendingApprovals(result: AiSdkResultLike): string[] {
|
|
549
|
+
const parts: AiSdkContentPartLike[] = [];
|
|
550
|
+
if (result.steps?.length) {
|
|
551
|
+
for (const step of result.steps) if (Array.isArray(step.content)) parts.push(...step.content);
|
|
552
|
+
} else if (Array.isArray(result.content)) {
|
|
553
|
+
parts.push(...result.content);
|
|
554
|
+
}
|
|
555
|
+
const ids: string[] = [];
|
|
556
|
+
for (const part of parts) {
|
|
557
|
+
if (part.type !== "tool-approval-request" || part.isAutomatic === true) continue;
|
|
558
|
+
const approvalId = str(part.approvalId);
|
|
559
|
+
if (approvalId) ids.push(approvalId);
|
|
560
|
+
}
|
|
561
|
+
return ids;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* t.sendFile 带来的文件(base64)转成 AI SDK 的多模态用户消息;纯文本就是纯文本。
|
|
566
|
+
* 用 `file` part(v7 推荐;`image` part 已废弃),mediaType 原样透传,图片之外的附件也能走。
|
|
567
|
+
*/
|
|
568
|
+
function userMessage(text: string, files: readonly { mimeType: string; dataBase64: string }[] | undefined): unknown {
|
|
569
|
+
if (!files || files.length === 0) return { role: "user", content: text };
|
|
570
|
+
return {
|
|
571
|
+
role: "user",
|
|
572
|
+
content: [
|
|
573
|
+
{ type: "text", text: text || "请描述这个附件。" },
|
|
574
|
+
...files.map((f) => ({ type: "file", mediaType: f.mimeType, data: f.dataBase64 })),
|
|
575
|
+
],
|
|
576
|
+
};
|
|
577
|
+
}
|