@opentiny/next-sdk 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/dist/{SimulatorMask-BHVXyogh-CARX3Rff.js → SimulatorMask-BHVXyogh-CKmEDsYG.js} +1 -1
  2. package/dist/{index-R_HIbfUX.js → index-CHMa2zG4.js} +1065 -1042
  3. package/dist/index.js +1 -1
  4. package/dist/page-tools/a11y/config.d.ts +6 -3
  5. package/dist/page-tools/a11y/constants.d.ts +20 -0
  6. package/dist/page-tools/a11y/utils.d.ts +14 -0
  7. package/dist/page-tools/constants.d.ts +4 -2
  8. package/dist/page-tools/context.d.ts +2 -0
  9. package/dist/page-tools/handlers/hover.d.ts +9 -0
  10. package/dist/page-tools/schema.d.ts +3 -3
  11. package/dist/page-tools/utils/dom.d.ts +8 -2
  12. package/dist/runtime.js +19 -172
  13. package/package.json +6 -1
  14. package/WebMcpClient.ts +0 -505
  15. package/WebMcpServer.ts +0 -447
  16. package/agent/AgentModelProvider.ts +0 -932
  17. package/agent/type.ts +0 -91
  18. package/agent/utils/generateReActPrompt.ts +0 -71
  19. package/agent/utils/getAISDKTools.ts +0 -34
  20. package/agent/utils/getBuiltinMcpTools.ts +0 -86
  21. package/agent/utils/parseReActAction.ts +0 -34
  22. package/core.ts +0 -26
  23. package/index.ts +0 -91
  24. package/page-tools/a11y/build.ts +0 -74
  25. package/page-tools/a11y/config.ts +0 -465
  26. package/page-tools/a11y/constants.ts +0 -131
  27. package/page-tools/a11y/search.ts +0 -127
  28. package/page-tools/a11y/types.ts +0 -105
  29. package/page-tools/a11y/utils.ts +0 -239
  30. package/page-tools/a11y/vnode.ts +0 -439
  31. package/page-tools/a11y-tree.ts +0 -14
  32. package/page-tools/bridge.ts +0 -722
  33. package/page-tools/configs/console-cloud.ts +0 -172
  34. package/page-tools/constants.ts +0 -12
  35. package/page-tools/context.ts +0 -50
  36. package/page-tools/effects.ts +0 -345
  37. package/page-tools/handlers/browserState.ts +0 -12
  38. package/page-tools/handlers/click.ts +0 -30
  39. package/page-tools/handlers/executeJavascript.ts +0 -22
  40. package/page-tools/handlers/fill.ts +0 -65
  41. package/page-tools/handlers/scroll.ts +0 -66
  42. package/page-tools/handlers/searchTree.ts +0 -27
  43. package/page-tools/handlers/select.ts +0 -39
  44. package/page-tools/initialize-builtin-WebMCP.ts +0 -20
  45. package/page-tools/page-agent-highlight/index.ts +0 -245
  46. package/page-tools/page-agent-mask/SimulatorMask.module.css +0 -14
  47. package/page-tools/page-agent-mask/SimulatorMask.ts +0 -299
  48. package/page-tools/page-agent-mask/checkDarkMode.ts +0 -181
  49. package/page-tools/page-agent-mask/cursor-border.svg +0 -3
  50. package/page-tools/page-agent-mask/cursor-fill.svg +0 -5
  51. package/page-tools/page-agent-mask/cursor.module.css +0 -70
  52. package/page-tools/page-agent-mask/hauwei.svg +0 -25
  53. package/page-tools/page-agent-prompt.md +0 -139
  54. package/page-tools/page-agent-tool-event.ts +0 -113
  55. package/page-tools/page-agent-tool.ts +0 -207
  56. package/page-tools/page-state-cache.ts +0 -78
  57. package/page-tools/schema.ts +0 -52
  58. package/page-tools/tool-config.ts +0 -100
  59. package/page-tools/utils/dom.ts +0 -158
  60. package/page-tools/utils/scroll.ts +0 -58
  61. package/remoter/QrCode.ts +0 -54
  62. package/remoter/createRemoter.ts +0 -957
  63. package/remoter/svgs/chat.svg +0 -43
  64. package/remoter/svgs/icon-copy.svg +0 -25
  65. package/remoter/svgs/link.svg +0 -28
  66. package/remoter/svgs/logo.svg +0 -45
  67. package/remoter/svgs/qrcode.svg +0 -35
  68. package/remoter/svgs/scan.svg +0 -21
  69. package/remoter/tooltips.ts +0 -260
  70. package/runtime.html +0 -39
  71. package/runtime.ts +0 -44
  72. package/script/utils.ts +0 -26
  73. package/skills/index.ts +0 -287
  74. package/test/page-tools/a11y/build.test.ts +0 -638
  75. package/test/page-tools/a11y/config.test.ts +0 -370
  76. package/test/page-tools/configs/console-cloud.test.ts +0 -168
  77. package/test/page-tools/page-agent-highlight.test.ts +0 -110
  78. package/test/page-tools/page-agent-tool-dispatch.test.ts +0 -208
  79. package/test/page-tools/page-agent-tool.test.ts +0 -102
  80. package/test/page-tools/tool-config.test.ts +0 -112
  81. package/test/page-tools/utils/dom.test.ts +0 -122
  82. package/tsconfig.json +0 -16
  83. package/utils/builtinProxy.ts +0 -144
  84. package/utils/env.ts +0 -13
  85. package/utils/uuid.ts +0 -10
  86. package/vite.config.runtime.ts +0 -22
  87. package/vite.config.ts +0 -63
  88. package/vitest.config.ts +0 -10
@@ -1,932 +0,0 @@
1
- import { streamText, stepCountIs, generateText } from 'ai'
2
- import { MCPClientConfig, createMCPClient } from '@ai-sdk/mcp'
3
- import type { ToolSet } from 'ai'
4
- import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
5
- import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'
6
- import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'
7
- import type { IAgentModelProviderOption, McpServerConfig } from './type'
8
- import { ProviderV2 } from '@ai-sdk/provider'
9
- import { OpenAIProvider } from '@ai-sdk/openai'
10
- import { createOpenAI } from '@ai-sdk/openai'
11
- import { createDeepSeek, DeepSeekProvider } from '@ai-sdk/deepseek'
12
- import { MessageChannelTransport } from '@opentiny/next'
13
- import { WebMcpClient } from '../WebMcpClient'
14
- import { getAISDKTools } from './utils/getAISDKTools'
15
- import { getBuiltinMcpTools } from './utils/getBuiltinMcpTools'
16
- import { generateReActToolsPrompt } from './utils/generateReActPrompt'
17
- import { parseReActAction } from './utils/parseReActAction'
18
-
19
- export const AIProviderFactories = {
20
- ['openai']: createOpenAI,
21
- ['deepseek']: createDeepSeek
22
- }
23
-
24
- type ChatMethodFn = typeof streamText | typeof generateText
25
-
26
- /** 一个通用的ai-sdk的Agent封装
27
- * @summary 内部自动管理了 llm, mcpServer, ai-sdk的clients 和 tools
28
- * @returns 暴露了 chat, chatStream方法
29
- */
30
- export class AgentModelProvider {
31
- llm: ProviderV2 | OpenAIProvider | DeepSeekProvider
32
- /** 当前mcpServers对象集合。键为服务器名称,值为 McpServerConfig 或任意的 MCPTransport
33
- * 参考: https://ai-sdk.dev/docs/ai-sdk-core/tools-and-tool-calling#initializing-an-mcp-client */
34
- mcpServers: Record<string, McpServerConfig> = {}
35
- /** 当前ai-sdk的 mcpClient 对象集合,键为服务器名称 */
36
- mcpClients: Record<string, any> = {}
37
- /** 当前 mcpClients 所对应的tools,键为服务器名称 */
38
- mcpTools: Record<string, Record<string, any>> = {}
39
- /** 需要实时过滤掉的tools name*/
40
- ignoreToolnames: string[] = []
41
- /** Agent 自动更新所有的tools 后的事件 */
42
- onUpdatedTools: (() => void) | undefined
43
- /** Agent 内部报错时,抛出的错误事件 */
44
- onError: ((msg: string, err?: any) => void) | undefined
45
- /** 缓存 ai-sdk response 中的 多轮会话的上下文 */
46
- responseMessages: any[] = []
47
- /** 是否使用 ReAct 模式(通过提示词而非 function calling 进行工具调用) */
48
- useReActMode: boolean = false
49
-
50
- constructor({ llmConfig, mcpServers }: IAgentModelProviderOption) {
51
- if (!llmConfig) {
52
- throw new Error('llmConfig is required to initialize AgentModelProvider')
53
- }
54
- this.mcpServers = mcpServers || {}
55
- this.mcpClients = {}
56
- this.mcpTools = {}
57
-
58
- if (llmConfig.llm) {
59
- this.llm = llmConfig.llm
60
- } else if (llmConfig.providerType) {
61
- const providerType = llmConfig.providerType
62
- let providerFn: (options: any) => ProviderV2 | OpenAIProvider | DeepSeekProvider
63
-
64
- if (typeof providerType === 'string') {
65
- providerFn = AIProviderFactories[providerType]
66
- } else {
67
- providerFn = providerType
68
- }
69
- this.llm = providerFn({
70
- apiKey: llmConfig.apiKey,
71
- baseURL: llmConfig.baseURL
72
- })
73
- } else {
74
- throw new Error('Either llmConfig.llm or llmConfig.providerType must be provided')
75
- }
76
-
77
- // 读取 ReAct 模式配置
78
- this.useReActMode = (llmConfig as any).useReActMode ?? false
79
- }
80
-
81
- /** 创建一个 ai-sdk的 mcpClient, 创建失败则返回 null */
82
- private async _createOneClient(serverConfig: McpServerConfig) {
83
- try {
84
- let transport: MCPClientConfig['transport']
85
- // builtin 类型:使用 document.modelContext 作为工具数据源
86
- if ('type' in serverConfig && serverConfig.type === 'builtin') {
87
- const builtinClient = serverConfig.client
88
- // 包装成与 ai-sdk MCPClient 相同接口:提供 tools() 方法
89
- return {
90
- tools: () => getBuiltinMcpTools(builtinClient),
91
- close: async () => {}
92
- }
93
- }
94
- // transport 一定是 streamableHttp/sse 或者就是: ai-sdk允许的 transport
95
- if ('type' in serverConfig && serverConfig.type.toLocaleLowerCase() === 'streamablehttp') {
96
- const configWithHeaders = serverConfig as { url: string; headers?: Record<string, string> }
97
- const requestInit = configWithHeaders.headers ? { headers: configWithHeaders.headers } : undefined
98
- transport = new StreamableHTTPClientTransport(new URL(configWithHeaders.url), { requestInit })
99
- } else if ('type' in serverConfig && serverConfig.type === 'sse') {
100
- const configWithHeaders = serverConfig as { url: string; headers?: Record<string, string> }
101
- const requestInit = configWithHeaders.headers ? { headers: configWithHeaders.headers } : undefined
102
- transport = new SSEClientTransport(new URL(configWithHeaders.url), { requestInit })
103
- } else if ('transport' in serverConfig) {
104
- transport = serverConfig.transport
105
- } else {
106
- transport = serverConfig as unknown as MCPClientConfig['transport']
107
- }
108
-
109
- // 根据 useAISdkClient 配置决定使用哪种 client 创建方式
110
- const useAISdkClient = serverConfig.useAISdkClient ?? false
111
-
112
- if (useAISdkClient) {
113
- // 使用 ai-sdk 的 createMCPClient
114
- const client = await createMCPClient({ transport: transport as MCPClientConfig['transport'] })
115
- //@ts-ignore
116
- client['__transport__'] = transport
117
- return client
118
- } else {
119
- // 使用 WebMcpClient
120
- const client = new WebMcpClient(
121
- { name: 'mcp-web-client', version: '1.0.0' },
122
- { capabilities: { roots: { listChanged: true }, sampling: {}, elicitation: {} } }
123
- )
124
- // @ts-ignore transport 已经在前面的条件分支中转换为 Transport 实例,类型系统无法正确推断
125
- await client.connect(transport)
126
-
127
- //@ts-ignore
128
- client['__transport__'] = transport
129
-
130
- return client
131
- }
132
- } catch (error: unknown) {
133
- if (this.onError) {
134
- this.onError((error as Error)?.message || `Failed to create MCP client`, error)
135
- }
136
- console.error(`Failed to create MCP client`, serverConfig, error)
137
- return null
138
- }
139
- }
140
- /** 关闭一个 mcpClient */
141
- private async _closeOneClient(client: any) {
142
- try {
143
- const transport = client['__transport__']
144
-
145
- // 如果是 InMemoryTransport 或 MessageChannelTransport,不关闭传输层
146
- // 因为它们是配对的,关闭一端会影响另一端(服务端)
147
- if (
148
- (transport && transport instanceof InMemoryTransport) ||
149
- (transport && transport instanceof MessageChannelTransport)
150
- ) {
151
- return
152
- }
153
-
154
- // 其他类型的传输正常关闭
155
- await transport?.terminateSession?.()
156
- await transport?.close?.()
157
- await client?.close?.()
158
- } catch (error) {}
159
- }
160
- /** 创建所有 mcpClients */
161
- private async _createMpcClients() {
162
- // 使用 Promise.all 并行处理所有 mcpServer 项
163
- const serverEntries = Object.entries(this.mcpServers)
164
- const clients = await Promise.all(
165
- serverEntries.map(async ([serverName, server]) => {
166
- const client = await this._createOneClient(server)
167
- return { serverName, client }
168
- })
169
- )
170
- // 将结果存储到对象中,使用 serverName 作为键
171
- this.mcpClients = {}
172
- clients.forEach(({ serverName, client }) => {
173
- this.mcpClients[serverName] = client
174
- })
175
- }
176
- /** 兼容两种 client 类型的 tools 获取方法 */
177
- private async _getClientTools(client: any, serverName: string): Promise<ToolSet | null> {
178
- if (!client) {
179
- return null
180
- }
181
-
182
- try {
183
- // 判断是否为 ai-sdk 的 client(有 tools 方法)
184
- if (typeof client.tools === 'function') {
185
- // ai-sdk 的 client,直接调用 tools() 方法
186
- return await client.tools()
187
- } else {
188
- // WebMcpClient,使用 getAISDKTools 函数
189
- return await getAISDKTools(client)
190
- }
191
- } catch (error: unknown) {
192
- if (this.onError) {
193
- this.onError((error as Error)?.message || `Failed to query tools for ${serverName}`, error)
194
- }
195
- console.error(`Failed to query tools for ${serverName}`, error)
196
- return null
197
- }
198
- }
199
-
200
- /** 查询所有 mcpClients 的 tools, 失败则保存为null */
201
- private async _createMpcTools() {
202
- const clientEntries = Object.entries(this.mcpClients)
203
- const tools = await Promise.all(
204
- clientEntries.map(async ([serverName, client]) => {
205
- const result = await this._getClientTools(client, serverName)
206
- return { serverName, tools: result }
207
- })
208
- )
209
- // 将结果存储到对象中,使用 serverName 作为键
210
- this.mcpTools = {}
211
- tools.forEach(({ serverName, tools: toolsData }) => {
212
- const normalizedTools = toolsData && typeof toolsData === 'object' ? (toolsData as Record<string, any>) : {}
213
- this.mcpTools[serverName] = normalizedTools
214
- })
215
- }
216
- /** 关闭所有的 clients */
217
- async closeAll() {
218
- await Promise.all(
219
- Object.values(this.mcpClients).map(async (client) => {
220
- try {
221
- await this._closeOneClient(client)
222
- } catch (error: unknown) {
223
- if (this.onError) {
224
- this.onError((error as Error)?.message || `Failed to close client`, error)
225
- }
226
- console.error(`Failed to close client`, error)
227
- }
228
- })
229
- )
230
- }
231
-
232
- /** 创建所有的 mcpClients,并更新它们的tools */
233
- async initClientsAndTools() {
234
- await this._createMpcClients()
235
- await this._createMpcTools()
236
- this.onUpdatedTools?.()
237
- }
238
-
239
- /** 仅刷新已连接 clients 的 tools(不重建 client,适合工具目录动态变化场景) */
240
- async refreshTools() {
241
- await this._createMpcTools()
242
- this.onUpdatedTools?.()
243
- }
244
-
245
- /** 全量更新所有的 mcpServers */
246
- async updateMcpServers(mcpServers?: Record<string, McpServerConfig>) {
247
- await this.closeAll()
248
- this.mcpServers = mcpServers || this.mcpServers
249
- await this.initClientsAndTools()
250
- }
251
-
252
- /** 插入一个新的mcpServer,如果已经存在则返回false */
253
- async insertMcpServer(serverName: string, mcpServer: McpServerConfig) {
254
- // 检查是否已存在相同名称的服务器
255
- if (this.mcpServers[serverName]) {
256
- return false
257
- }
258
-
259
- const client = await this._createOneClient(mcpServer)
260
- if (!client) {
261
- // 创建客户端失败时直接返回,避免后续出现空指针问题
262
- this.onError?.(`Failed to create MCP client: ${serverName}`)
263
- return null
264
- }
265
- this.mcpClients[serverName] = client
266
- // 使用兼容的工具获取方法
267
- const tools = await this._getClientTools(client, serverName)
268
- // 工具列表可能为 null,统一兜底为空对象,确保类型安全
269
- this.mcpTools[serverName] = tools && typeof tools === 'object' ? (tools as Record<string, any>) : {}
270
- this.mcpServers[serverName] = mcpServer
271
- this.onUpdatedTools?.()
272
-
273
- return client
274
- }
275
- /** 通过服务器名称删除mcpServer: mcpServers mcpClients mcpTools ignoreToolnames */
276
- async removeMcpServer(serverName: string) {
277
- if (!this.mcpServers[serverName]) {
278
- return
279
- }
280
-
281
- // 删除 mcpServer
282
- delete this.mcpServers[serverName]
283
-
284
- // 关闭并删除 client
285
- const delClient = this.mcpClients[serverName]
286
- delete this.mcpClients[serverName]
287
- try {
288
- await this._closeOneClient(delClient)
289
- } catch (error) {}
290
-
291
- // 删除 tools 并清理 ignoreToolnames
292
- const delTool = this.mcpTools[serverName]
293
- delete this.mcpTools[serverName]
294
-
295
- if (delTool) {
296
- Object.keys(delTool).forEach((toolName) => {
297
- this.ignoreToolnames = this.ignoreToolnames.filter((name) => name !== toolName)
298
- })
299
- }
300
-
301
- this.onUpdatedTools?.()
302
- }
303
-
304
- /** 创建临时允许调用的 tools 集合,合并 mcpTools 与 extraTool */
305
- private _tempMergeTools(extraTool: ToolSet = {} as ToolSet, deleteIgnored = true): ToolSet {
306
- const toolsResult: ToolSet = Object.values(this.mcpTools).reduce(
307
- (acc, curr) => ({ ...acc, ...curr }) as ToolSet,
308
- {} as ToolSet
309
- )
310
- Object.assign(toolsResult, extraTool)
311
-
312
- if (deleteIgnored) {
313
- this.ignoreToolnames.forEach((name) => {
314
- delete toolsResult[name]
315
- })
316
- }
317
- return toolsResult
318
- }
319
-
320
- /** 获取当前激活的 tools 名称列表(过滤 ignoreToolnames) */
321
- private _getActiveToolNames(tools: ToolSet): string[] {
322
- return Object.keys(tools).filter((name) => !this.ignoreToolnames.includes(name))
323
- }
324
-
325
- /** 生成 ReAct 模式的系统提示词(包含工具描述) */
326
- private _generateReActSystemPrompt(tools: ToolSet, modelName: string, baseSystemPrompt?: string): string {
327
- // 统一使用 XML 格式的 ReAct 提示词(所有 ReAct 模式都使用相同格式)
328
- const toolsPrompt = generateReActToolsPrompt(tools)
329
-
330
- if (baseSystemPrompt) {
331
- return `${baseSystemPrompt}${toolsPrompt}`
332
- }
333
- return `你是一个智能助手,可以通过调用工具来完成任务。\n${toolsPrompt}`
334
- }
335
-
336
- /** 执行 ReAct 模式下的工具调用 */
337
- private async _executeReActToolCall(
338
- toolName: string,
339
- args: any,
340
- tools: ToolSet
341
- ): Promise<{ success: boolean; result?: any; error?: string }> {
342
- const tool = tools[toolName]
343
- if (!tool) {
344
- return { success: false, error: `工具 ${toolName} 不存在` }
345
- }
346
-
347
- try {
348
- const toolInfo = tool as any
349
- const executeFn = toolInfo.execute || toolInfo.call
350
- if (typeof executeFn !== 'function') {
351
- return { success: false, error: `工具 ${toolName} 没有可执行的函数` }
352
- }
353
-
354
- const result = await executeFn(args, {})
355
- return { success: true, result }
356
- } catch (error: any) {
357
- const errorMsg = error?.message || String(error) || '工具执行失败'
358
- return { success: false, error: errorMsg }
359
- }
360
- }
361
-
362
- /** ReAct 模式的对话实现 */
363
- private async _chatReAct(
364
- chatMethod: ChatMethodFn,
365
- { model, maxSteps = 5, ...options }: Parameters<typeof generateText>[0] & { maxSteps?: number; message?: string }
366
- ): Promise<any> {
367
- if (!this.llm) {
368
- throw new Error('LLM is not initialized')
369
- }
370
-
371
- await this.initClientsAndTools()
372
-
373
- // 合并所有可用工具
374
- const allTools = this._tempMergeTools(options.tools)
375
- const toolNames = Object.keys(allTools)
376
-
377
- // 如果没有工具,回退到普通模式
378
- if (toolNames.length === 0) {
379
- return this._chat(chatMethod, { model, maxSteps, ...options })
380
- }
381
-
382
- // 准备消息历史
383
- let currentMessages: any[] = []
384
- if (options.message && !options.messages) {
385
- currentMessages.push({ role: 'user', content: options.message })
386
- } else if (options.messages) {
387
- currentMessages = [...options.messages]
388
- } else {
389
- currentMessages = [...this.responseMessages]
390
- }
391
-
392
- // 确保 model 是字符串类型(ReAct 模式下 model 应该是模型名称字符串)
393
- const modelName = typeof model === 'string' ? model : (model as any)?.modelId || 'default-model'
394
-
395
- // 生成包含工具描述的系统提示词
396
- const systemPrompt = this._generateReActSystemPrompt(allTools, modelName, options.system as string)
397
-
398
- const systemMessage = { role: 'system', content: systemPrompt }
399
-
400
- // 确保第一条消息是系统提示词
401
- const messagesWithSystem =
402
- currentMessages[0]?.role === 'system' ? currentMessages : [systemMessage, ...currentMessages]
403
-
404
- // 判断是否为流式输出
405
- const isStream = chatMethod === streamText
406
-
407
- if (isStream) {
408
- // 流式输出模式:创建一个包装的流
409
- return this._chatReActStream(messagesWithSystem, allTools, modelName, maxSteps, options)
410
- } else {
411
- // 非流式输出模式:循环对话直到完成
412
- return this._chatReActNonStream(messagesWithSystem, allTools, modelName, maxSteps, options)
413
- }
414
- }
415
-
416
- /**
417
- * 检查消息内容是否包含图片
418
- * @param content 消息内容
419
- * @returns 是否包含图片
420
- */
421
- private _messageHasImage(content: any): boolean {
422
- if (!content) return false
423
-
424
- // 如果 content 是数组,检查是否有 image 类型的项
425
- if (Array.isArray(content)) {
426
- return content.some((item) => item && item.type === 'image')
427
- }
428
-
429
- return false
430
- }
431
-
432
- /**
433
- * 从消息中移除图片,但保留文本内容
434
- * @param message 原始消息
435
- * @returns 移除图片后的消息(如果只有图片没有文本,返回 null)
436
- */
437
- private _removeImageFromMessage(message: any): any | null {
438
- if (!message || !message.content) {
439
- return null
440
- }
441
-
442
- // 如果 content 不是数组,直接返回(没有图片)
443
- if (!Array.isArray(message.content)) {
444
- return message
445
- }
446
-
447
- // 过滤掉图片类型的内容,保留文本
448
- const textContent = message.content.filter((item: any) => item && item.type !== 'image')
449
-
450
- // 如果过滤后没有内容,返回 null
451
- if (textContent.length === 0) {
452
- return null
453
- }
454
-
455
- // 返回只包含文本的消息副本
456
- return {
457
- ...message,
458
- content: textContent
459
- }
460
- }
461
-
462
- /**
463
- * 构建用于模型调用的消息列表
464
- * 策略:保留所有文本消息,仅限制图片数量(类似 magentic-ui 的 maybe_remove_old_screenshots)
465
- *
466
- * @param systemMessage 系统提示词
467
- * @param allMessages 所有消息历史(包括初始消息和后续对话)
468
- * @param maxImages 最多保留的图片数量(默认3张)
469
- * @returns 构建好的消息列表
470
- */
471
- private _buildMessagesForModel(systemMessage: any | null, allMessages: any[], maxImages: number = 3): any[] {
472
- const messages: any[] = []
473
-
474
- // 1. 添加系统提示词
475
- if (systemMessage) {
476
- messages.push(systemMessage)
477
- }
478
-
479
- // 2. 保留所有文本消息,但限制图片数量
480
- // 从后往前遍历,优先保留最新的图片
481
- let imageCount = 0
482
- const processedMessages: any[] = []
483
-
484
- for (let i = allMessages.length - 1; i >= 0; i--) {
485
- const msg = allMessages[i]
486
-
487
- // 检查消息是否包含图片
488
- const hasImage = this._messageHasImage(msg.content)
489
-
490
- if (hasImage) {
491
- if (imageCount < maxImages) {
492
- // 图片数量未超限,保留完整消息
493
- processedMessages.unshift(msg)
494
- imageCount++
495
- } else {
496
- // 图片数量超限,移除图片但保留文本(如果有)
497
- const textOnly = this._removeImageFromMessage(msg)
498
- if (textOnly) {
499
- processedMessages.unshift(textOnly)
500
- }
501
- }
502
- } else {
503
- // 非图片消息:全部保留
504
- processedMessages.unshift(msg)
505
- }
506
- }
507
-
508
- messages.push(...processedMessages)
509
-
510
- return messages
511
- }
512
-
513
- /** ReAct 模式非流式对话 */
514
- private async _chatReActNonStream(
515
- messages: any[],
516
- tools: ToolSet,
517
- model: string,
518
- maxSteps: number,
519
- options: any
520
- ): Promise<any> {
521
- // 保存完整的消息历史(用于最终返回和传递给模型)
522
- let fullMessageHistory = [...messages]
523
- // 提取系统提示词(第一条消息)
524
- const systemMessage = messages[0]?.role === 'system' ? messages[0] : null
525
- // 提取所有非系统消息
526
- const allUserMessages = systemMessage ? messages.slice(1) : messages
527
-
528
- let stepCount = 0
529
- // 配置:最多保留的图片数量
530
- const maxImages = (options as any).maxImages ?? 3
531
-
532
- while (stepCount < maxSteps) {
533
- stepCount++
534
-
535
- // 构建用于模型调用的消息列表
536
- const messagesForModel = this._buildMessagesForModel(systemMessage, allUserMessages, maxImages)
537
-
538
- // 调用 LLM(ReAct 模式下不传递 tools,因为工具调用通过提示词实现)
539
- const { tools: _, ...restOptions } = options
540
- const result = await generateText({
541
- // @ts-ignore ProviderV2 是所有llm的父类,在每一个具体的llm类都有一个选择model的函数用法
542
- model: this.llm(model),
543
- messages: messagesForModel,
544
- ...restOptions
545
- })
546
-
547
- const assistantMessage = result.text
548
- // 添加到所有消息和完整历史
549
- const assistantMsg = { role: 'assistant', content: assistantMessage }
550
- allUserMessages.push(assistantMsg)
551
- fullMessageHistory.push(assistantMsg)
552
-
553
- // 解析工具调用
554
- const action = parseReActAction(assistantMessage, tools)
555
-
556
- if (!action) {
557
- // 没有工具调用,返回最终结果
558
- this.responseMessages = fullMessageHistory
559
- return {
560
- text: assistantMessage,
561
- response: { messages: fullMessageHistory }
562
- }
563
- }
564
-
565
- // 执行工具调用
566
- const toolResult = await this._executeReActToolCall(action.toolName, action.arguments, tools)
567
-
568
- // 统一使用 XML 格式的 Observation
569
- const resultString = toolResult.success ? JSON.stringify(toolResult.result) : `工具执行失败 - ${toolResult.error}`
570
- const observation = `<tool_response>\n${resultString}\n</tool_response>`
571
-
572
- // 添加到所有消息和完整历史
573
- const observationMessage = {
574
- role: 'user',
575
- content: observation
576
- }
577
- allUserMessages.push(observationMessage)
578
- fullMessageHistory.push(observationMessage)
579
- }
580
-
581
- // 达到最大步数,返回最后一条消息
582
- this.responseMessages = fullMessageHistory
583
- const lastMessage = fullMessageHistory[fullMessageHistory.length - 2]?.content || ''
584
- return {
585
- text: lastMessage,
586
- response: { messages: fullMessageHistory }
587
- }
588
- }
589
-
590
- /** ReAct 模式流式对话 */
591
- private _chatReActStream(messages: any[], tools: ToolSet, model: string, maxSteps: number, options: any): any {
592
- // 保存 this 引用,以便在异步生成器中使用
593
- const self = this
594
- // @ts-ignore ProviderV2 是所有llm的父类,在每一个具体的llm类都有一个选择model的函数用法
595
- const llmModel = this.llm(model)
596
-
597
- // 创建一个 Promise 来跟踪流完成状态,用于触发 onFinish
598
- let streamCompleteResolver: (value: any) => void
599
- let streamCompleteRejecter: (error: any) => void
600
- const streamCompletePromise = new Promise((resolve, reject) => {
601
- streamCompleteResolver = resolve
602
- streamCompleteRejecter = reject
603
- })
604
-
605
- // 创建一个异步生成器来模拟流式输出
606
- const stream = new ReadableStream({
607
- async start(controller) {
608
- // 保存完整的消息历史(用于最终返回和传递给模型)
609
- let fullMessageHistory = [...messages]
610
- // 提取系统提示词(第一条消息)
611
- const systemMessage = messages[0]?.role === 'system' ? messages[0] : null
612
- // 提取所有非系统消息
613
- const allUserMessages = systemMessage ? messages.slice(1) : [...messages]
614
-
615
- let stepCount = 0
616
- let accumulatedText = ''
617
- // 配置:最多保留的图片数量
618
- const maxImages = (options as any).maxImages ?? 3
619
-
620
- // 模拟开启流
621
- controller.enqueue({ type: 'start' })
622
- controller.enqueue({ type: 'start-step' })
623
-
624
- try {
625
- while (stepCount < maxSteps) {
626
- stepCount++
627
-
628
- // 构建用于模型调用的消息列表
629
- const messagesForModel = self._buildMessagesForModel(systemMessage, allUserMessages, maxImages)
630
-
631
- // 移除 tools 选项,ReAct 模式下不传递 tools
632
- const { tools: _, ...restOptions } = options
633
-
634
- // 删除影响多轮对话的配置
635
- delete restOptions.system
636
- delete restOptions.onFinish
637
- const result = await streamText({
638
- ...restOptions,
639
- model: llmModel,
640
- messages: messagesForModel
641
- })
642
-
643
- // 1、直接问,收集流式输出
644
- let assistantText = ''
645
- for await (const part of result.fullStream) {
646
- if (part.type === 'text-delta') {
647
- assistantText += part.text || ''
648
- // 转发文本增量
649
- controller.enqueue({
650
- type: 'text-delta',
651
- text: part.text
652
- })
653
- } else if (part.type === 'text-start') {
654
- controller.enqueue({ type: 'text-start' })
655
- } else if (part.type === 'text-end' || part.type === 'finish-step' || part.type === 'finish') {
656
- // 暂时不关闭,等待检查是否有工具调用
657
- } else if (part.type === 'start' || part.type === 'start-step') {
658
- // 后面每一轮的start 也要忽略
659
- } else {
660
- // 转发其他类型的事件
661
- controller.enqueue(part)
662
- }
663
- }
664
-
665
- accumulatedText += assistantText
666
- // 添加到所有消息和完整历史
667
- const assistantMsg = { role: 'assistant', content: accumulatedText }
668
- allUserMessages.push(assistantMsg)
669
- fullMessageHistory.push(assistantMsg)
670
-
671
- // 2、解析工具调用
672
- const action = parseReActAction(accumulatedText, tools)
673
-
674
- if (!action) {
675
- // 没有工具调用,结束流
676
- controller.enqueue({ type: 'text-end' })
677
- controller.enqueue({ type: 'finish-step' })
678
- controller.enqueue({ type: 'finish' })
679
- controller.close()
680
- self.responseMessages = fullMessageHistory
681
- // 触发 onFinish 回调
682
- streamCompleteResolver({ messages: fullMessageHistory })
683
- return
684
- }
685
-
686
- // 特殊处理: computer 工具的 terminate 操作
687
- if (action.toolName === 'computer' && action.arguments?.action === 'terminate') {
688
- // 视为对话结束
689
- controller.enqueue({ type: 'text-end' })
690
- controller.enqueue({ type: 'finish-step' })
691
- controller.enqueue({ type: 'finish' })
692
- controller.close()
693
- self.responseMessages = fullMessageHistory
694
- streamCompleteResolver({ messages: fullMessageHistory })
695
- return
696
- }
697
-
698
- // 3、是工具调用,发送工具调用开始事件(符合 tiny-robot 格式)
699
- const toolCallId = `react-${Date.now()}`
700
- controller.enqueue({
701
- type: 'tool-input-start',
702
- id: toolCallId,
703
- toolName: action.toolName
704
- })
705
-
706
- // 发送工具调用参数(显示调用中状态)
707
- const argsString = JSON.stringify(action.arguments, null, 2)
708
- controller.enqueue({
709
- type: 'tool-input-delta',
710
- id: toolCallId,
711
- delta: argsString
712
- })
713
- controller.enqueue({
714
- type: 'tool-input-end',
715
- id: toolCallId
716
- })
717
-
718
- // 4、执行工具调用
719
- controller.enqueue({
720
- type: 'tool-call',
721
- toolCallId: toolCallId,
722
- toolName: action.toolName,
723
- input: action.arguments
724
- })
725
-
726
- const toolResult = await self._executeReActToolCall(action.toolName, action.arguments, tools)
727
-
728
- // 如果结果包含 screenshot,先提取出来,避免 JSON stringify 导致过大
729
- let screenshot = undefined
730
- let resultData = toolResult.result
731
- if (
732
- toolResult.success &&
733
- toolResult.result &&
734
- typeof toolResult.result === 'object' &&
735
- toolResult.result.screenshot
736
- ) {
737
- screenshot = toolResult.result.screenshot
738
- const { screenshot: _, ...rest } = toolResult.result
739
- resultData = rest
740
- }
741
-
742
- // 构造 Observation 文本(统一使用 XML 格式)
743
- let observationText = ''
744
-
745
- if (toolResult.success) {
746
- // 尝试从 resultData 中提取纯文本信息
747
- if (
748
- resultData &&
749
- Array.isArray(resultData.content) &&
750
- resultData.content.length > 0 &&
751
- resultData.content[0].text
752
- ) {
753
- observationText = resultData.content[0].text
754
- } else {
755
- observationText = JSON.stringify(resultData)
756
- }
757
-
758
- // 统一使用 XML 格式的 Observation,如果有截图,添加验证提示
759
- let finalObservation = `<tool_response>\n${observationText}\n</tool_response>`
760
-
761
- if (screenshot) {
762
- finalObservation += `\n请检查截图以确认操作是否成功。如果成功,请继续下一步;如果失败,请重试。`
763
- }
764
-
765
- // 发送工具结果(符合 tiny-robot 格式,给 UI 展示用的,不包含 base64 防止卡顿)
766
- controller.enqueue({
767
- type: 'tool-result',
768
- toolCallId: toolCallId,
769
- result: finalObservation
770
- })
771
-
772
- // 添加工具结果到消息历史(ReAct 模式下,工具结果作为 user 消息添加)
773
- const observationMessage = screenshot
774
- ? {
775
- role: 'user',
776
- content: [
777
- { type: 'text', text: finalObservation },
778
- { type: 'image', image: screenshot }
779
- ]
780
- }
781
- : {
782
- role: 'user',
783
- content: finalObservation
784
- }
785
-
786
- // 添加到所有消息和完整历史
787
- allUserMessages.push(observationMessage)
788
- fullMessageHistory.push(observationMessage)
789
-
790
- // 重置累积文本,准备下一轮
791
- accumulatedText = ''
792
- } else {
793
- observationText = `工具执行失败 - ${toolResult.error}`
794
- controller.enqueue({
795
- type: 'tool-error',
796
- toolCallId: toolCallId,
797
- input: action.arguments,
798
- error: { message: observationText }
799
- })
800
- }
801
- }
802
-
803
- // 达到最大步数
804
- controller.enqueue({ type: 'text-end' })
805
- controller.enqueue({ type: 'finish-step' })
806
- controller.enqueue({ type: 'finish' })
807
- controller.close()
808
- self.responseMessages = fullMessageHistory
809
- // 触发 onFinish 回调
810
- streamCompleteResolver({ messages: fullMessageHistory })
811
- } catch (error: any) {
812
- controller.error(error)
813
- streamCompleteRejecter(error)
814
- }
815
- }
816
- })
817
-
818
- // 返回一个类似 streamText 的结果对象
819
- // response Promise 需要在流结束时 resolve,这样才能触发 onFinish 回调
820
- return {
821
- fullStream: stream,
822
- response: streamCompletePromise
823
- }
824
- }
825
-
826
- private async _chat(
827
- chatMethod: ChatMethodFn,
828
- { model, maxSteps = 5, ...options }: Parameters<typeof generateText>[0] & { maxSteps?: number; message?: string }
829
- ): Promise<any> {
830
- // 如果启用 ReAct 模式,使用 ReAct 实现
831
- if (this.useReActMode) {
832
- return this._chatReAct(chatMethod, { model, maxSteps, ...options })
833
- }
834
-
835
- // 否则使用原有的 function calling 模式
836
- if (!this.llm) {
837
- throw new Error('LLM is not initialized')
838
- }
839
-
840
- await this.initClientsAndTools()
841
-
842
- const extraTools = (options.tools || {}) as ToolSet
843
- const allTools = this._tempMergeTools(extraTools, false)
844
-
845
- // 让 tools 保持“同一个对象引用”并在每个 step 前同步:
846
- // - prepareStep 只能控制 activeTools,不能直接返回 tools
847
- // - 因此需要在进入每个 step 前,把最新 mcpTools 合并回当前 tools 对象
848
- // - 这样 activeTools 新增的工具名在当步就能命中真实工具定义
849
- const syncToolsForStep = () => {
850
- const latestTools = this._tempMergeTools(extraTools, false)
851
-
852
- // upsert
853
- Object.entries(latestTools).forEach(([name, tool]) => {
854
- ;(allTools as any)[name] = tool
855
- })
856
-
857
- // remove stale
858
- Object.keys(allTools).forEach((name) => {
859
- if (!(name in latestTools)) {
860
- delete (allTools as any)[name]
861
- }
862
- })
863
- }
864
-
865
- const userPrepareStep = (options as any).prepareStep
866
- const wrappedPrepareStep = async (stepOptions: any) => {
867
- syncToolsForStep()
868
-
869
- const latestActiveTools = this._getActiveToolNames(allTools)
870
- const userStepPatch = typeof userPrepareStep === 'function' ? await userPrepareStep(stepOptions) : undefined
871
- const safeUserStepPatch =
872
- userStepPatch && typeof userStepPatch === 'object' ? userStepPatch : ({} as Record<string, any>)
873
-
874
- return {
875
- ...safeUserStepPatch,
876
- activeTools: Array.isArray(safeUserStepPatch.activeTools)
877
- ? safeUserStepPatch.activeTools.filter((name: string) => latestActiveTools.includes(name))
878
- : latestActiveTools
879
- }
880
- }
881
-
882
- const chatOptions = {
883
- // @ts-ignore ProviderV2 是所有llm的父类, 在每一个具体的llm 类都有一个选择model的函数用法
884
- model: this.llm(model),
885
- stopWhen: stepCountIs(maxSteps),
886
- ...options,
887
- tools: allTools,
888
- prepareStep: wrappedPrepareStep,
889
- activeTools: this._getActiveToolNames(allTools)
890
- }
891
-
892
- // 保存最后一条 user 消息,用于后续缓存
893
- let lastUserMessage: any = null
894
-
895
- if (options.message && !options.messages) {
896
- // 使用 options.message 创建 user 消息
897
- lastUserMessage = { role: 'user', content: options.message }
898
- this.responseMessages.push(lastUserMessage)
899
- chatOptions.messages = [...this.responseMessages]
900
- } else if (options.messages && options.messages.length > 0) {
901
- // 从 options.messages 中获取最后一条 user 消息
902
- const lastMessage = options.messages[options.messages.length - 1]
903
- if (lastMessage.role === 'user') {
904
- lastUserMessage = lastMessage
905
- }
906
- }
907
-
908
- const result = chatMethod(chatOptions)
909
-
910
- // 缓存 ai-sdk 的多轮对话的消息
911
- ;(result as any)?.response?.then((res: any) => {
912
- // 检查 res.messages 的第一条消息是否是 user 消息
913
- // 如果不是,且有 lastUserMessage,则先添加 lastUserMessage
914
- const firstMessage = res.messages?.[0]
915
- if (lastUserMessage && firstMessage?.role !== 'user') {
916
- this.responseMessages.push(lastUserMessage)
917
- }
918
- // 然后添加 AI 返回的消息
919
- this.responseMessages.push(...res.messages)
920
- })
921
-
922
- return result
923
- }
924
-
925
- async chat(options: Parameters<typeof generateText>[0] & { maxSteps?: number; message?: string }): Promise<any> {
926
- return this._chat(generateText, options)
927
- }
928
-
929
- async chatStream(options: Parameters<typeof streamText>[0] & { maxSteps?: number; message?: string }): Promise<any> {
930
- return this._chat(streamText, options as any)
931
- }
932
- }