@opentiny/next-sdk 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/agent/AgentModelProvider.ts +78 -81
  2. package/agent/type.ts +6 -9
  3. package/agent/utils/getAISDKTools.ts +0 -1
  4. package/agent/utils/getBuiltinMcpTools.ts +7 -7
  5. package/core.ts +0 -3
  6. package/dist/SimulatorMask-BHVXyogh-CARX3Rff.js +361 -0
  7. package/dist/agent/type.d.ts +4 -12
  8. package/dist/agent/utils/getBuiltinMcpTools.d.ts +3 -3
  9. package/dist/core.d.ts +0 -1
  10. package/dist/core.js +16 -17
  11. package/dist/index-R_HIbfUX.js +6604 -0
  12. package/dist/index.d.ts +11 -3
  13. package/dist/index.js +76 -4969
  14. package/dist/{initialize-builtin-WebMCP-HgObT902.js → initialize-builtin-WebMCP-JaoKwVlm.js} +1156 -1037
  15. package/dist/page-tools/a11y/build.d.ts +10 -0
  16. package/dist/page-tools/a11y/config.d.ts +96 -0
  17. package/dist/page-tools/a11y/constants.d.ts +11 -0
  18. package/dist/page-tools/a11y/search.d.ts +17 -0
  19. package/dist/page-tools/a11y/types.d.ts +95 -0
  20. package/dist/page-tools/a11y/utils.d.ts +55 -0
  21. package/dist/page-tools/a11y/vnode.d.ts +40 -0
  22. package/dist/page-tools/a11y-tree.d.ts +9 -99
  23. package/dist/page-tools/configs/console-cloud.d.ts +6 -0
  24. package/dist/page-tools/constants.d.ts +10 -0
  25. package/dist/page-tools/context.d.ts +40 -0
  26. package/dist/page-tools/handlers/browserState.d.ts +8 -0
  27. package/dist/page-tools/handlers/click.d.ts +8 -0
  28. package/dist/page-tools/handlers/executeJavascript.d.ts +8 -0
  29. package/dist/page-tools/handlers/fill.d.ts +8 -0
  30. package/dist/page-tools/handlers/scroll.d.ts +8 -0
  31. package/dist/page-tools/handlers/searchTree.d.ts +9 -0
  32. package/dist/page-tools/handlers/select.d.ts +8 -0
  33. package/dist/page-tools/page-agent-highlight/index.d.ts +21 -0
  34. package/dist/page-tools/page-agent-mask/SimulatorMask.d.ts +16 -0
  35. package/dist/page-tools/page-agent-mask/checkDarkMode.d.ts +5 -0
  36. package/dist/page-tools/page-agent-tool-event.d.ts +26 -0
  37. package/dist/page-tools/page-agent-tool.d.ts +3 -8
  38. package/dist/page-tools/schema.d.ts +44 -0
  39. package/dist/page-tools/tool-config.d.ts +50 -0
  40. package/dist/page-tools/utils/dom.d.ts +6 -0
  41. package/dist/page-tools/utils/scroll.d.ts +15 -0
  42. package/dist/runtime.d.ts +7 -0
  43. package/dist/runtime.js +732 -0
  44. package/dist/utils/builtinProxy.d.ts +1 -1
  45. package/dist/vitest.config.d.ts +2 -0
  46. package/index.ts +35 -5
  47. package/package.json +23 -29
  48. package/page-tools/a11y/build.ts +74 -0
  49. package/page-tools/a11y/config.ts +465 -0
  50. package/page-tools/a11y/constants.ts +131 -0
  51. package/page-tools/a11y/search.ts +127 -0
  52. package/page-tools/a11y/types.ts +105 -0
  53. package/page-tools/a11y/utils.ts +239 -0
  54. package/page-tools/a11y/vnode.ts +439 -0
  55. package/page-tools/a11y-tree.ts +9 -527
  56. package/page-tools/bridge.ts +23 -3
  57. package/page-tools/configs/console-cloud.ts +172 -0
  58. package/page-tools/constants.ts +12 -0
  59. package/page-tools/context.ts +50 -0
  60. package/page-tools/handlers/browserState.ts +12 -0
  61. package/page-tools/handlers/click.ts +30 -0
  62. package/page-tools/handlers/executeJavascript.ts +22 -0
  63. package/page-tools/handlers/fill.ts +65 -0
  64. package/page-tools/handlers/scroll.ts +66 -0
  65. package/page-tools/handlers/searchTree.ts +27 -0
  66. package/page-tools/handlers/select.ts +39 -0
  67. package/page-tools/page-agent-highlight/index.ts +245 -0
  68. package/page-tools/page-agent-mask/SimulatorMask.module.css +14 -0
  69. package/page-tools/page-agent-mask/SimulatorMask.ts +299 -0
  70. package/page-tools/page-agent-mask/checkDarkMode.ts +181 -0
  71. package/page-tools/page-agent-mask/cursor-border.svg +3 -0
  72. package/page-tools/page-agent-mask/cursor-fill.svg +5 -0
  73. package/page-tools/page-agent-mask/cursor.module.css +70 -0
  74. package/page-tools/page-agent-mask/hauwei.svg +25 -0
  75. package/page-tools/page-agent-prompt.md +34 -18
  76. package/page-tools/page-agent-tool-event.ts +113 -0
  77. package/page-tools/page-agent-tool.ts +146 -162
  78. package/page-tools/schema.ts +52 -0
  79. package/page-tools/tool-config.ts +100 -0
  80. package/page-tools/utils/dom.ts +158 -0
  81. package/page-tools/utils/scroll.ts +58 -0
  82. package/runtime.ts +44 -0
  83. package/test/page-tools/a11y/build.test.ts +638 -0
  84. package/test/page-tools/a11y/config.test.ts +370 -0
  85. package/test/page-tools/configs/console-cloud.test.ts +168 -0
  86. package/test/page-tools/page-agent-highlight.test.ts +110 -0
  87. package/test/page-tools/page-agent-tool-dispatch.test.ts +208 -0
  88. package/test/page-tools/page-agent-tool.test.ts +102 -0
  89. package/test/page-tools/tool-config.test.ts +112 -0
  90. package/test/page-tools/utils/dom.test.ts +122 -0
  91. package/utils/builtinProxy.ts +45 -13
  92. package/vite.config.runtime.ts +22 -0
  93. package/vite.config.ts +52 -8
  94. package/vitest.config.ts +10 -0
  95. package/McpSdk.ts +0 -14
  96. package/WebAgent.ts +0 -5
  97. package/WebMcp.ts +0 -26
  98. package/Zod.ts +0 -1
  99. package/dist/McpSdk.d.ts +0 -14
  100. package/dist/SimulatorMask-BHVXyogh-BFEGpD5S.js +0 -1048
  101. package/dist/SimulatorMask-BHVXyogh-CCYbrb84.js +0 -801
  102. package/dist/WebAgent.d.ts +0 -5
  103. package/dist/WebMcp.d.ts +0 -23
  104. package/dist/Zod.d.ts +0 -1
  105. package/dist/index.es.dev.js +0 -59017
  106. package/dist/index.es.js +0 -46795
  107. package/dist/index.umd.dev.js +0 -60355
  108. package/dist/index.umd.js +0 -1248
  109. package/dist/mcpsdk@1.25.3.dev.js +0 -22780
  110. package/dist/mcpsdk@1.25.3.es.dev.js +0 -22778
  111. package/dist/mcpsdk@1.25.3.es.js +0 -16960
  112. package/dist/mcpsdk@1.25.3.js +0 -48
  113. package/dist/transport/ExtensionClientTransport.d.ts +0 -24
  114. package/dist/transport/ExtensionContentServerTransport.d.ts +0 -39
  115. package/dist/transport/ExtensionPageServerTransport.d.ts +0 -36
  116. package/dist/transport/messages.d.ts +0 -9
  117. package/dist/vite.config.mcpSdk.d.ts +0 -2
  118. package/dist/vite.config.webAgent.d.ts +0 -2
  119. package/dist/vite.config.webMcp.d.ts +0 -2
  120. package/dist/vite.config.webMcpFull.d.ts +0 -2
  121. package/dist/vite.config.zod.d.ts +0 -2
  122. package/dist/webagent.dev.js +0 -49360
  123. package/dist/webagent.es.dev.js +0 -49071
  124. package/dist/webagent.es.js +0 -39219
  125. package/dist/webagent.js +0 -642
  126. package/dist/webmcp-full.dev.js +0 -31336
  127. package/dist/webmcp-full.es.dev.js +0 -30283
  128. package/dist/webmcp-full.es.js +0 -22889
  129. package/dist/webmcp-full.js +0 -645
  130. package/dist/webmcp.dev.js +0 -9572
  131. package/dist/webmcp.es.dev.js +0 -8518
  132. package/dist/webmcp.es.js +0 -6727
  133. package/dist/webmcp.js +0 -602
  134. package/dist/zod@3.25.76.dev.js +0 -4037
  135. package/dist/zod@3.25.76.es.dev.js +0 -4033
  136. package/dist/zod@3.25.76.es.js +0 -2945
  137. package/dist/zod@3.25.76.js +0 -1
  138. package/transport/ExtensionClientTransport.ts +0 -100
  139. package/transport/ExtensionContentServerTransport.ts +0 -162
  140. package/transport/ExtensionPageServerTransport.ts +0 -149
  141. package/transport/messages.ts +0 -63
  142. package/vite-build-tsc.ts +0 -63
  143. package/vite-env.d.ts +0 -10
  144. package/vite.config.mcpSdk.ts +0 -28
  145. package/vite.config.webAgent.ts +0 -19
  146. package/vite.config.webMcp.ts +0 -40
  147. package/vite.config.webMcpFull.ts +0 -19
  148. package/vite.config.zod.ts +0 -23
  149. /package/dist/{vite-build-tsc.d.ts → vite.config.runtime.d.ts} +0 -0
@@ -1,29 +1,46 @@
1
1
  import { initializeBuiltinWebMCP } from './initialize-builtin-WebMCP'
2
- import { z } from 'zod'
3
2
  import { zodToJsonSchema } from 'zod-to-json-schema'
4
3
  import pageAgentPrompt from './page-agent-prompt.md?raw'
5
- import { PageController, clickElement, inputTextElement, selectOptionElement } from '@page-agent/page-controller'
6
- import { buildA11yTree, searchA11yTree, type RefMap } from './a11y-tree'
4
+ import { PageController } from '@page-agent/page-controller'
5
+ import { buildA11yTree, type RefMap } from './a11y-tree'
7
6
  import { PageStateCache } from './page-state-cache'
8
-
9
- declare global {
10
- interface Window {
11
- __webmcpcli_interactiveWhitelist?: Element[]
12
- __webmcpcli_interactiveBlacklist?: Element[]
13
- __webmcpcli_beforeGetBrowserState?: (() => void) | null
14
- }
15
- }
7
+ import { SimulatorMask } from './page-agent-mask/SimulatorMask'
8
+ import { highlight, unhighlight, globalRemoveListener } from './page-agent-highlight'
9
+ import { setupPageAgentToolEventBridge } from './page-agent-tool-event'
10
+
11
+ import type { PageAgentToolOptions } from './constants'
12
+ import { inputSchema, type PageAgentToolInput } from './schema'
13
+ import { createActionErrorResult, type ActionContext } from './context'
14
+ import { detectPageDialog, detectValidationErrors } from './utils/dom'
15
+ import { getPageAgentToolConfig, setPageAgentToolConfig } from './tool-config'
16
+
17
+ import { handleBrowserState } from './handlers/browserState'
18
+ import { handleClick } from './handlers/click'
19
+ import { handleFill } from './handlers/fill'
20
+ import { handleSelect } from './handlers/select'
21
+ import { handleScroll } from './handlers/scroll'
22
+ import { handleExecuteJavascript } from './handlers/executeJavascript'
23
+ import { handleSearchTree } from './handlers/searchTree'
16
24
 
17
25
  /** 在浏览器页面中注册 page-agent-tool, 用于页面的内容获取和操作,页面的动效 */
18
- export function registerPageAgentTool() {
26
+ export function registerPageAgentTool(options: PageAgentToolOptions = {}) {
19
27
  initializeBuiltinWebMCP()
20
28
 
21
- window.__webmcpcli_interactiveWhitelist = [] // 白名单元素列表,存在则识别为交互元素
22
- window.__webmcpcli_interactiveBlacklist = [] // 黑名单,反之
23
- window.__webmcpcli_beforeGetBrowserState = null // 指定网站覆盖该函数,用于设置当前网站的黑白名单
29
+ // 完整工具配置(顶层选项 enableHighlight + 统一无障碍配置 a11yConfig):与默认配置合并后
30
+ // 得到运行期唯一生效的配置(存于 window.__webmcpcli_toolConfig),后续可通过
31
+ // setPageAgentToolConfig 在运行期继续修改(追加式合并/函数式过滤/整体替换)
32
+ setPageAgentToolConfig(options, { mode: 'replace' })
24
33
 
25
- // 保留 PageController 仅用于 showMask/hideMask(UX 遮罩层)
26
- const pageController = new PageController({ enableMask: true })
34
+ window.__webmcpcli_beforeGetBrowserState = window.__webmcpcli_beforeGetBrowserState || null // 指定网站覆盖该函数,可在其中调用 setPageAgentToolConfig 动态调整当前页面的配置
35
+
36
+ // 保留 PageController ,先关闭内置mask, 再手工绑定当前项目的mask类
37
+ const pageController = new PageController({ enableMask: false })
38
+
39
+ // @ts-ignore
40
+ pageController.maskReady = (async () => {
41
+ // @ts-ignore
42
+ pageController.mask = new SimulatorMask()
43
+ })()
27
44
 
28
45
  // ─── 状态 Diff 缓存
29
46
  const stateCache = new PageStateCache()
@@ -31,60 +48,51 @@ export function registerPageAgentTool() {
31
48
  // 当前 ref 索引 → HTMLElement 映射(每次 buildA11yTree 后更新)
32
49
  let currentRefMap: RefMap = new Map()
33
50
 
34
- // ─── inputSchema 与原版完全一致(对外 API 不变)──────────────────────────
35
- const inputSchema = z.object({
36
- action: z.enum(['browserState', 'click', 'fill', 'select', 'scroll', 'executeJavascript', 'searchTree'] as const)
37
- .describe(`执行的动作名称, 每一次执行 'click', 'fill', 'select'动作之前,**必须**要先调用 'browserState' 去获取页面的最新状态。
38
- browserState: '查询整个页面的浏览器状态;返回页面的标题、URL、YAML格式的语义化页面树',
39
- click: '根据元素索引点击;',
40
- fill: '根据元素索引填写文本;';
41
- select: '根据元素索引选择下拉框选项;';
42
- scroll: '滚动页面的动作,可以指定水平滚动还是上下滚动; 不带元素索引时:滚动整个文档。带元素索引时:滚动该索引处的容器(或其最近的可滚动祖先元素)'
43
- executeJavascript: '执行javascript代码'
44
- searchTree: '按关键词搜索无障碍树,返回带行号的匹配节点及上下文,无需获取全量树。适合快速定位特定元素(如所有按钮、特定名称的链接等),显著减少上下文消耗。必须提供 query 参数。'
45
- `),
46
- index: z.number().min(0).optional().describe('执行动作 of the element index, 动作为 click,fill,select时,必须提供元素索引'),
47
- text: z.string().optional().describe('执行动作的文本内容, 动作为 fill,select 时,必须提供文本内容'),
48
- down: z.boolean().optional().describe('执行上下滚动时,必须提供down参数'),
49
- right: z.boolean().optional().describe('执行水平滚动方向, 必须提供right参数'),
50
- numPages: z
51
- .number()
52
- .optional()
53
- .describe('执行动作的滚动页数, 动作为 scroll时,可以提供滚动页数,建议每次滚动0.1页,该值不要大于5.'),
54
- pixels: z.number().int().min(0).optional().describe('执行动作的滚动像素数,动作为 scroll时,可以提供滚动像素数'),
55
- script: z.string().optional().describe('执行的javascript代码,动作为 executeJavascript时,必须提供script参数'),
56
- query: z.string().optional().describe('搜索关键词,动作为 searchTree 时必须提供。支持按 role(如 button、link)、节点名称、状态(如 checked)或 ref 索引(如 #3)搜索'),
57
- contextLines: z.number().int().min(0).max(10).default(2).describe('searchTree 时每个命中行前后保留的上下文行数,默认 2'),
58
- maxMatches: z.number().int().min(1).max(50).default(20).describe('searchTree 时最大返回分组数,默认 20'),
59
- responseMode: z.enum(['full', 'diff', 'both'] as const)
60
- .optional()
61
- .default('diff')
62
- .describe('返回浏览器状态的模式。full: 仅返回当前全量 A11y 树;diff: 仅返回与上一次状态的增量差异;both: 同时返回全量 A11y 树与增量差异。默认值为 diff。')
63
- })
64
-
65
51
  // ─── 辅助:构建错误响应 ───────────────────────────────────────────────────
66
52
  async function errContent(msg: string) {
67
- await pageController.hideMask()
68
- return { content: [{ type: 'text', text: msg }] }
53
+ return { content: [{ type: 'text' as const, text: msg }] }
54
+ }
55
+
56
+ async function actionError(msg: string) {
57
+ const result = await createActionErrorResult(msg, buildBrowserStateResponse)
58
+ options.removeMaskAfterToolCall && (await pageController.hideMask())
59
+ return result
60
+ }
61
+
62
+ // AI 使用过期 ref 时,自动重建 A11y 树并返回全量状态,
63
+ // 避免 AI 额外往返调用 browserState,减少操作轮次
64
+ async function refreshOnStaleRef(action: string, index: number) {
65
+ const refreshResult = await buildBrowserStateResponse('full')
66
+ const warning = `⚠️ ${action}失败: ref 索引 ${index} 已失效(页面可能已刷新),已自动重新加载页面状态,请使用新的 ref 索引重试。\n`
67
+ return {
68
+ content: [{ type: 'text' as const, text: warning + refreshResult.content[0].text }]
69
+ }
69
70
  }
70
71
 
71
72
  // ─── 核心:构建 browserState 响应(全量 or 增量 Diff)────────────────────
72
- async function buildBrowserStateResponse(responseMode: 'full' | 'diff' | 'both' = 'diff'): Promise<{ content: Array<{ type: string; text: string }> }> {
73
+ async function buildBrowserStateResponse(
74
+ responseMode: 'full' | 'diff' | 'both' = 'diff'
75
+ ): Promise<{ content: Array<{ type: 'text'; text: string }> }> {
73
76
  const url = window.location.href
74
77
  const title = document.title
75
78
 
76
- // 获取用户自定义黑名单与白名单
77
- const blacklist = (window.__webmcpcli_interactiveBlacklist ?? []) as Element[]
78
- const whitelist = (window.__webmcpcli_interactiveWhitelist ?? []) as Element[]
79
-
80
- // 生成语义化 ARIA YAML 树 + 刷新 refMap
81
- const { yaml, refMap } = buildA11yTree(document.body, blacklist, whitelist)
79
+ // 生成语义化 ARIA YAML 树 + 刷新 refMap(统一读取运行期生效的配置,支持 setPageAgentToolConfig 动态修改)
80
+ const { yaml, refMap } = buildA11yTree(document.body, getPageAgentToolConfig().a11yConfig)
82
81
  currentRefMap = refMap
83
82
 
83
+ // 高亮交互元素,且增加全局移除高亮的监听(读取运行期生效的配置,支持 setPageAgentToolConfig 动态修改)
84
+ if (getPageAgentToolConfig().enableHighlight) {
85
+ highlight(refMap)
86
+ globalRemoveListener()
87
+ }
88
+
84
89
  // 计算 Diff
85
90
  const diff = stateCache.update(url, yaml)
86
91
 
87
- await pageController.hideMask()
92
+ // 检测页面弹窗/遮罩层(确认框、提示框等),让 AI 优先处理
93
+ const dialogAlert = detectPageDialog()
94
+ // 检测表单校验错误,提醒 AI 优先修复
95
+ const validationErrors = detectValidationErrors()
88
96
 
89
97
  // 根据 responseMode 组装 content
90
98
  let displayContent = ''
@@ -102,122 +110,98 @@ export function registerPageAgentTool() {
102
110
  title,
103
111
  content: displayContent
104
112
  }
105
- const text = `浏览器状态: ${JSON.stringify(stateObj)}`
113
+ const text = `浏览器状态: ${JSON.stringify(stateObj)}${dialogAlert}${validationErrors}`
106
114
  return { content: [{ type: 'text', text }] }
107
115
  }
108
116
 
117
+ // 组装上下文传递给 handlers
118
+ const actionContext: ActionContext = {
119
+ pageController,
120
+ stateCache,
121
+ getRefMap: () => currentRefMap,
122
+ setRefMap: (map: RefMap) => {
123
+ currentRefMap = map
124
+ },
125
+ buildBrowserStateResponse,
126
+ refreshOnStaleRef,
127
+ errContent,
128
+ actionError
129
+ }
130
+
131
+ async function executePageAgentTool(args: PageAgentToolInput) {
132
+ try {
133
+ let ret: any
134
+
135
+ switch (args.action) {
136
+ case 'browserState':
137
+ ret = await handleBrowserState(args, actionContext)
138
+ break
139
+ case 'click':
140
+ await pageController.showMask()
141
+ pageController.mask.borderElement(currentRefMap.get(args.index))
142
+ ret = await handleClick(args, actionContext)
143
+ pageController.mask.removeBorderElement()
144
+ options.removeMaskAfterToolCall && (await pageController.hideMask())
145
+ break
146
+ case 'fill':
147
+ await pageController.showMask()
148
+ pageController.mask.borderElement(currentRefMap.get(args.index))
149
+ ret = await handleFill(args, actionContext)
150
+ pageController.mask.removeBorderElement()
151
+ options.removeMaskAfterToolCall && (await pageController.hideMask())
152
+ break
153
+ case 'select':
154
+ await pageController.showMask()
155
+ pageController.mask.borderElement(currentRefMap.get(args.index))
156
+ ret = await handleSelect(args, actionContext)
157
+ pageController.mask.removeBorderElement()
158
+ options.removeMaskAfterToolCall && (await pageController.hideMask())
159
+ break
160
+ case 'scroll':
161
+ await pageController.showMask()
162
+ ret = await handleScroll(args, actionContext)
163
+ options.removeMaskAfterToolCall && (await pageController.hideMask())
164
+ break
165
+ case 'executeJavascript':
166
+ ret = await handleExecuteJavascript(args, actionContext)
167
+ break
168
+ case 'searchTree':
169
+ ret = await handleSearchTree(args, actionContext)
170
+ break
171
+ default:
172
+ ret = { content: [{ type: 'text' as const, text: `未知操作: ${args.action}` }] }
173
+ }
174
+ return ret
175
+ } catch (error) {
176
+ const actionNames = {
177
+ click: '点击',
178
+ fill: '填写',
179
+ select: '选择'
180
+ } as const
181
+ if (args.action in actionNames) {
182
+ return actionError(
183
+ `${actionNames[args.action as keyof typeof actionNames]}执行异常: ${error instanceof Error ? error.message : String(error)}`
184
+ )
185
+ }
186
+ options.removeMaskAfterToolCall && (await pageController.hideMask())
187
+ throw error
188
+ }
189
+ }
190
+
109
191
  // ─── 工具注册(名称与 inputSchema 与原版完全一致)────────────────────────
110
192
  ;(document as any).modelContext.registerTool({
111
193
  name: 'page-agent-tool',
112
194
  description: pageAgentPrompt,
113
195
  // @ts-ignore
114
196
  inputSchema: zodToJsonSchema(inputSchema) as any,
115
- async execute(args: any) {
116
- await pageController.showMask()
117
- const mode = args.responseMode ?? 'diff'
197
+ async execute(args: PageAgentToolInput) {
118
198
  try {
119
- // ── browserState:生成语义化 YAML + Diff ──────────────────────────
120
- if (args.action === 'browserState') {
121
- if (window.__webmcpcli_beforeGetBrowserState) {
122
- window.__webmcpcli_beforeGetBrowserState()
123
- }
124
- return buildBrowserStateResponse(mode)
125
-
126
- // ── click:用底层 clickElement(el) 操作,操作后自动返回 diff ───────
127
- } else if (args.action === 'click') {
128
- if (args.index === undefined) return errContent('点击结果: 缺少元素索引')
129
- const el = currentRefMap.get(args.index)
130
- if (!el) return errContent(`点击结果: 无效的 ref 索引 ${args.index},请先调用 browserState 刷新页面状态`)
131
- await clickElement(el)
132
- // 操作成功后自动返回 diff/both/full
133
- return buildBrowserStateResponse(mode)
134
-
135
- // ── fill:输入文本,操作后自动返回 diff ───────────────────────────
136
- } else if (args.action === 'fill') {
137
- if (args.index === undefined || !args.text) return errContent('填写结果: 缺少元素索引或文本内容')
138
- const el = currentRefMap.get(args.index)
139
- if (!el) return errContent(`填写结果: 无效的 ref 索引 ${args.index},请先调用 browserState 刷新页面状态`)
140
-
141
- let targetEl = el
142
- if (!(targetEl instanceof HTMLInputElement) && !(targetEl instanceof HTMLTextAreaElement)) {
143
- const innerInput = el.querySelector('input, textarea')
144
- if (innerInput) {
145
- targetEl = innerInput as HTMLElement
146
- }
147
- }
148
-
149
- if (targetEl instanceof HTMLInputElement && targetEl.readOnly) {
150
- targetEl.value = args.text
151
- targetEl.dispatchEvent(new Event('input', { bubbles: true }))
152
- targetEl.dispatchEvent(new Event('change', { bubbles: true }))
153
- targetEl.dispatchEvent(new Event('blur', { bubbles: true }))
154
- } else {
155
- await inputTextElement(targetEl, args.text)
156
- }
157
- return buildBrowserStateResponse(mode)
158
-
159
- // ── select:选择下拉框,操作后自动返回 diff ───────────────────────
160
- } else if (args.action === 'select') {
161
- if (args.index === undefined || !args.text) return errContent('选择结果: 缺少元素索引或文本内容')
162
- const el = currentRefMap.get(args.index) as HTMLSelectElement
163
- if (!el) return errContent(`选择结果: 无效的 ref 索引 ${args.index},请先调用 browserState 刷新页面状态`)
164
- await selectOptionElement(el, args.text)
165
- return buildBrowserStateResponse(mode)
166
-
167
- // ── scroll:滚动页面,操作后自动返回 diff ─────────────────────────
168
- } else if (args.action === 'scroll') {
169
- if (!args.down && !args.right) return errContent('滚动结果: 缺少滚动方向参数')
170
-
171
- // 确定滚动目标(有 index 时滚动该元素容器,否则滚动整个文档)
172
- const scrollTarget = args.index !== undefined
173
- ? (currentRefMap.get(args.index) ?? window)
174
- : window
175
-
176
- if (args.right) {
177
- const pixels = args.pixels ?? 300
178
- scrollTarget.scrollBy({ left: args.right ? pixels : -pixels, behavior: 'smooth' })
179
- } else {
180
- const pixels = args.pixels ?? Math.round((args.numPages ?? 1) * window.innerHeight)
181
- scrollTarget.scrollBy({ top: args.down ? pixels : -pixels, behavior: 'smooth' })
182
- }
183
-
184
- // 等待滚动动画完成后再采集状态
185
- await new Promise(r => setTimeout(r, 400))
186
- return buildBrowserStateResponse(mode)
187
- // ── executeJavascript:执行任意 JS ────────────────────────────────
188
- } else if (args.action === 'executeJavascript') {
189
- if (!args.script) return errContent('脚本执行异常: 缺少javascript代码')
190
- // eslint-disable-next-line no-new-func
191
- const result = await new Function(`return (async () => { ${args.script} })()`)()
192
- await pageController.hideMask()
193
- return {
194
- content: [{ type: 'text', text: `脚本执行结果: ${JSON.stringify(result)}` }]
195
- }
196
-
197
- // ── searchTree:关键词搜索无障碍树,返回带行号的精准结果 ──────────
198
- } else if (args.action === 'searchTree') {
199
- if (!args.query) return errContent('搜索失败: 缺少 query 参数')
200
- const blacklist = (window.__webmcpcli_interactiveBlacklist ?? []) as Element[]
201
- const whitelist = (window.__webmcpcli_interactiveWhitelist ?? []) as Element[]
202
- const { text } = searchA11yTree(
203
- args.query,
204
- document.body,
205
- blacklist,
206
- whitelist,
207
- {
208
- contextLines: args.contextLines,
209
- maxMatches: args.maxMatches,
210
- },
211
- )
212
- await pageController.hideMask()
213
- return {
214
- content: [{ type: 'text', text }]
215
- }
216
- }
199
+ return executePageAgentTool(args)
217
200
  } catch (error) {
218
- await pageController.hideMask()
219
- return { content: [{ type: 'text', text: `异常: ${String(error)}` }] }
201
+ return { content: [{ type: 'text' as const, text: `异常: ${String(error)}` }] }
220
202
  }
221
203
  }
222
204
  })
205
+
206
+ setupPageAgentToolEventBridge(executePageAgentTool, pageController)
223
207
  }
@@ -0,0 +1,52 @@
1
+ import { z } from 'zod'
2
+
3
+ export const inputSchema = z.object({
4
+ action: z.enum(['browserState', 'click', 'fill', 'select', 'scroll', 'executeJavascript', 'searchTree'] as const)
5
+ .describe(`执行的动作名称, 每一次执行 'click', 'fill', 'select'动作之前,**必须**要先调用 'browserState' 去获取页面的最新状态。
6
+ browserState: '查询整个页面的浏览器状态;返回页面的标题、URL、YAML格式的语义化页面树',
7
+ click: '根据元素索引点击;',
8
+ fill: '根据元素索引填写文本;';
9
+ select: '根据元素索引选择下拉框选项;';
10
+ scroll: '滚动页面的动作,可以指定水平滚动还是上下滚动; 不带元素索引时:滚动整个文档。带元素索引时:滚动该索引处的容器(或其最近的可滚动祖先元素)'
11
+ executeJavascript: '执行javascript代码'
12
+ searchTree: '按关键词搜索无障碍树,返回带行号的匹配节点及上下文,无需获取全量树。适合快速定位特定元素(如所有按钮、特定名称的链接等),显著减少上下文消耗。必须提供 query 参数。'
13
+ `),
14
+ index: z
15
+ .number()
16
+ .min(0)
17
+ .optional()
18
+ .describe('执行动作 of the element index, 动作为 click,fill,select时,必须提供元素索引'),
19
+ text: z.string().optional().describe('执行动作的文本内容, 动作为 fill,select 时,必须提供文本内容'),
20
+ down: z.boolean().optional().describe('执行上下滚动时,必须提供down参数'),
21
+ right: z.boolean().optional().describe('执行水平滚动方向, 必须提供right参数'),
22
+ numPages: z
23
+ .number()
24
+ .optional()
25
+ .describe('执行动作的滚动页数, 动作为 scroll时,可以提供滚动页数,建议每次滚动0.1页,该值不要大于5.'),
26
+ pixels: z.number().int().min(0).optional().describe('执行动作的滚动像素数,动作为 scroll时,可以提供滚动像素数'),
27
+ script: z.string().optional().describe('执行的javascript代码,动作为 executeJavascript时,必须提供script参数'),
28
+ query: z
29
+ .string()
30
+ .optional()
31
+ .describe(
32
+ '搜索关键词,动作为 searchTree 时必须提供。支持按 role(如 button、link)、节点名称、状态(如 checked)或 ref 索引(如 #3)搜索'
33
+ ),
34
+ contextLines: z
35
+ .number()
36
+ .int()
37
+ .min(0)
38
+ .max(10)
39
+ .default(2)
40
+ .describe('searchTree 时每个命中行前后保留的上下文行数,默认 2'),
41
+ maxMatches: z.number().int().min(1).max(50).default(20).describe('searchTree 时最大返回分组数,默认 20'),
42
+ responseMode: z
43
+ .enum(['full', 'diff', 'both'] as const)
44
+ .optional()
45
+ .default('diff')
46
+ .describe(
47
+ '返回浏览器状态的模式。full: 仅返回当前全量 A11y 树;diff: 仅返回与上一次状态的增量差异;both: 同时返回全量 A11y 树与增量差异。默认值为 diff。'
48
+ )
49
+ })
50
+
51
+ export type PageAgentToolInput = z.infer<typeof inputSchema>
52
+ export type PageAgentToolRawInput = z.input<typeof inputSchema>
@@ -0,0 +1,100 @@
1
+ /**
2
+ * tool-config.ts
3
+ *
4
+ * registerPageAgentTool 的完整运行期配置:顶层选项(enableHighlight 等)+ 统一无障碍配置(a11yConfig)。
5
+ * 两者合并到同一个配置对象里,提供唯一一套运行期读写 API(getPageAgentToolConfig/setPageAgentToolConfig):
6
+ * 注册时初始化一次,之后可随时读取/修改,无需分别记住两套 get/set 函数。
7
+ */
8
+
9
+ import type { A11yConfig, ResolvedA11yConfig } from './a11y/config'
10
+ import { DEFAULT_A11Y_CONFIG, mergeA11yConfig, mergeA11yConfigs } from './a11y/config'
11
+
12
+ export interface PageAgentToolConfig {
13
+ /** 是否启用元素高亮 */
14
+ enableHighlight: boolean
15
+ /** 是否在工具调用后移除遮罩, 默认值为 true 移除 */
16
+ removeMaskAfterToolCall?: boolean
17
+ /**
18
+ * 统一无障碍配置:按角色(roles)、状态(states:selected/disabled/error/warning 等)自定义规则,
19
+ * 以及白名单/黑名单/自定义暴露属性/弹窗选择器。已与默认配置合并(数组类字段是拼接结果)。
20
+ */
21
+ a11yConfig: ResolvedA11yConfig
22
+ }
23
+
24
+ /**
25
+ * 用于初始化(registerPageAgentTool 的入参)或运行期 setPageAgentToolConfig 的补丁对象:
26
+ * enableHighlight 为覆盖式更新;a11yConfig 为拼接式合并(数组类字段追加,不丢已有规则,语义见 a11y/config.ts)。
27
+ */
28
+ export interface PageAgentToolConfigPatch {
29
+ /** 是否启用元素高亮 */
30
+ enableHighlight?: boolean
31
+ /** 是否在工具调用后移除遮罩, 默认值为 true 移除 */
32
+ removeMaskAfterToolCall?: boolean
33
+ /** 是否在工具调用后移除遮罩 */
34
+ /** 统一无障碍配置,会与当前生效的 a11yConfig 按数组拼接合并 */
35
+ a11yConfig?: A11yConfig
36
+ }
37
+
38
+ /** registerPageAgentTool(options) 的入参类型,与运行期补丁 {@link PageAgentToolConfigPatch} 完全一致 */
39
+ export type PageAgentToolOptions = PageAgentToolConfigPatch
40
+
41
+ /** 默认生效的完整工具配置 */
42
+ export const DEFAULT_PAGE_AGENT_TOOL_CONFIG: PageAgentToolConfig = {
43
+ enableHighlight: false,
44
+ removeMaskAfterToolCall: true,
45
+ a11yConfig: DEFAULT_A11Y_CONFIG
46
+ }
47
+
48
+ declare global {
49
+ interface Window {
50
+ /** 运行期唯一生效的 page-agent-tool 完整配置(已与默认值合并),可通过 getPageAgentToolConfig/setPageAgentToolConfig 读写 */
51
+ __webmcpcli_toolConfig?: PageAgentToolConfig
52
+ }
53
+ }
54
+
55
+ /** 读取当前生效的完整工具配置(已与默认值合并)。未初始化时返回默认配置 */
56
+ export function getPageAgentToolConfig(): PageAgentToolConfig {
57
+ if (typeof window !== 'undefined' && window.__webmcpcli_toolConfig) {
58
+ return window.__webmcpcli_toolConfig
59
+ }
60
+ return {
61
+ enableHighlight: DEFAULT_PAGE_AGENT_TOOL_CONFIG.enableHighlight,
62
+ removeMaskAfterToolCall: DEFAULT_PAGE_AGENT_TOOL_CONFIG.removeMaskAfterToolCall,
63
+ a11yConfig: mergeA11yConfig()
64
+ }
65
+ }
66
+
67
+ function resolvePatch(patch: PageAgentToolConfigPatch, base: PageAgentToolConfig): PageAgentToolConfig {
68
+ return {
69
+ enableHighlight: patch.enableHighlight ?? base.enableHighlight,
70
+ removeMaskAfterToolCall: patch.removeMaskAfterToolCall ?? base.removeMaskAfterToolCall,
71
+ a11yConfig: mergeA11yConfigs(base.a11yConfig, patch.a11yConfig ?? {})
72
+ }
73
+ }
74
+
75
+ /**
76
+ * 更新当前生效配置。
77
+ * - patch 为对象时:enableHighlight 覆盖式更新;a11yConfig 与当前配置按数组拼接合并(不丢已有规则)
78
+ * - patch 为函数时:入参为当前生效配置,返回值直接与默认配置合并(而不是再与 current 相加)。
79
+ * 这样函数体内可以对 current.a11yConfig 做任意过滤/裁剪(如按条件"移除"某条旧规则),
80
+ * 返回值即为最终生效的配置,不会因为再与 current 相加而让被过滤掉的旧规则"复活"
81
+ * - options.mode = 'replace' 时(仅影响对象类型的 patch):不与"当前生效配置"合并,而是与默认配置重新合并
82
+ * 返回合并后的最新完整配置,并写回运行期存储供 buildBrowserStateResponse/buildA11yTree 等读取。
83
+ */
84
+ export function setPageAgentToolConfig(
85
+ patch: PageAgentToolConfigPatch | ((current: PageAgentToolConfig) => PageAgentToolConfigPatch),
86
+ options?: { mode?: 'merge' | 'replace' }
87
+ ): PageAgentToolConfig {
88
+ const mode = options?.mode ?? 'merge'
89
+ const current = getPageAgentToolConfig()
90
+
91
+ const next =
92
+ typeof patch === 'function'
93
+ ? resolvePatch(patch(current), DEFAULT_PAGE_AGENT_TOOL_CONFIG)
94
+ : resolvePatch(patch, mode === 'replace' ? DEFAULT_PAGE_AGENT_TOOL_CONFIG : current)
95
+
96
+ if (typeof window !== 'undefined') {
97
+ window.__webmcpcli_toolConfig = next
98
+ }
99
+ return next
100
+ }