@opentiny/next-sdk 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/agent/AgentModelProvider.ts +78 -81
  2. package/agent/type.ts +6 -9
  3. package/agent/utils/getAISDKTools.ts +0 -1
  4. package/agent/utils/getBuiltinMcpTools.ts +7 -7
  5. package/core.ts +0 -3
  6. package/dist/SimulatorMask-BHVXyogh-CARX3Rff.js +361 -0
  7. package/dist/agent/type.d.ts +4 -12
  8. package/dist/agent/utils/getBuiltinMcpTools.d.ts +3 -3
  9. package/dist/core.d.ts +0 -1
  10. package/dist/core.js +16 -17
  11. package/dist/index-R_HIbfUX.js +6604 -0
  12. package/dist/index.d.ts +11 -3
  13. package/dist/index.js +76 -4969
  14. package/dist/{initialize-builtin-WebMCP-HgObT902.js → initialize-builtin-WebMCP-JaoKwVlm.js} +1156 -1037
  15. package/dist/page-tools/a11y/build.d.ts +10 -0
  16. package/dist/page-tools/a11y/config.d.ts +96 -0
  17. package/dist/page-tools/a11y/constants.d.ts +11 -0
  18. package/dist/page-tools/a11y/search.d.ts +17 -0
  19. package/dist/page-tools/a11y/types.d.ts +95 -0
  20. package/dist/page-tools/a11y/utils.d.ts +55 -0
  21. package/dist/page-tools/a11y/vnode.d.ts +40 -0
  22. package/dist/page-tools/a11y-tree.d.ts +9 -99
  23. package/dist/page-tools/configs/console-cloud.d.ts +6 -0
  24. package/dist/page-tools/constants.d.ts +10 -0
  25. package/dist/page-tools/context.d.ts +40 -0
  26. package/dist/page-tools/handlers/browserState.d.ts +8 -0
  27. package/dist/page-tools/handlers/click.d.ts +8 -0
  28. package/dist/page-tools/handlers/executeJavascript.d.ts +8 -0
  29. package/dist/page-tools/handlers/fill.d.ts +8 -0
  30. package/dist/page-tools/handlers/scroll.d.ts +8 -0
  31. package/dist/page-tools/handlers/searchTree.d.ts +9 -0
  32. package/dist/page-tools/handlers/select.d.ts +8 -0
  33. package/dist/page-tools/page-agent-highlight/index.d.ts +21 -0
  34. package/dist/page-tools/page-agent-mask/SimulatorMask.d.ts +16 -0
  35. package/dist/page-tools/page-agent-mask/checkDarkMode.d.ts +5 -0
  36. package/dist/page-tools/page-agent-tool-event.d.ts +26 -0
  37. package/dist/page-tools/page-agent-tool.d.ts +3 -8
  38. package/dist/page-tools/schema.d.ts +44 -0
  39. package/dist/page-tools/tool-config.d.ts +50 -0
  40. package/dist/page-tools/utils/dom.d.ts +6 -0
  41. package/dist/page-tools/utils/scroll.d.ts +15 -0
  42. package/dist/runtime.d.ts +7 -0
  43. package/dist/runtime.js +732 -0
  44. package/dist/utils/builtinProxy.d.ts +1 -1
  45. package/dist/vitest.config.d.ts +2 -0
  46. package/index.ts +35 -5
  47. package/package.json +23 -29
  48. package/page-tools/a11y/build.ts +74 -0
  49. package/page-tools/a11y/config.ts +465 -0
  50. package/page-tools/a11y/constants.ts +131 -0
  51. package/page-tools/a11y/search.ts +127 -0
  52. package/page-tools/a11y/types.ts +105 -0
  53. package/page-tools/a11y/utils.ts +239 -0
  54. package/page-tools/a11y/vnode.ts +439 -0
  55. package/page-tools/a11y-tree.ts +9 -527
  56. package/page-tools/bridge.ts +23 -3
  57. package/page-tools/configs/console-cloud.ts +172 -0
  58. package/page-tools/constants.ts +12 -0
  59. package/page-tools/context.ts +50 -0
  60. package/page-tools/handlers/browserState.ts +12 -0
  61. package/page-tools/handlers/click.ts +30 -0
  62. package/page-tools/handlers/executeJavascript.ts +22 -0
  63. package/page-tools/handlers/fill.ts +65 -0
  64. package/page-tools/handlers/scroll.ts +66 -0
  65. package/page-tools/handlers/searchTree.ts +27 -0
  66. package/page-tools/handlers/select.ts +39 -0
  67. package/page-tools/page-agent-highlight/index.ts +245 -0
  68. package/page-tools/page-agent-mask/SimulatorMask.module.css +14 -0
  69. package/page-tools/page-agent-mask/SimulatorMask.ts +299 -0
  70. package/page-tools/page-agent-mask/checkDarkMode.ts +181 -0
  71. package/page-tools/page-agent-mask/cursor-border.svg +3 -0
  72. package/page-tools/page-agent-mask/cursor-fill.svg +5 -0
  73. package/page-tools/page-agent-mask/cursor.module.css +70 -0
  74. package/page-tools/page-agent-mask/hauwei.svg +25 -0
  75. package/page-tools/page-agent-prompt.md +34 -18
  76. package/page-tools/page-agent-tool-event.ts +113 -0
  77. package/page-tools/page-agent-tool.ts +146 -162
  78. package/page-tools/schema.ts +52 -0
  79. package/page-tools/tool-config.ts +100 -0
  80. package/page-tools/utils/dom.ts +158 -0
  81. package/page-tools/utils/scroll.ts +58 -0
  82. package/runtime.ts +44 -0
  83. package/test/page-tools/a11y/build.test.ts +638 -0
  84. package/test/page-tools/a11y/config.test.ts +370 -0
  85. package/test/page-tools/configs/console-cloud.test.ts +168 -0
  86. package/test/page-tools/page-agent-highlight.test.ts +110 -0
  87. package/test/page-tools/page-agent-tool-dispatch.test.ts +208 -0
  88. package/test/page-tools/page-agent-tool.test.ts +102 -0
  89. package/test/page-tools/tool-config.test.ts +112 -0
  90. package/test/page-tools/utils/dom.test.ts +122 -0
  91. package/utils/builtinProxy.ts +45 -13
  92. package/vite.config.runtime.ts +22 -0
  93. package/vite.config.ts +52 -8
  94. package/vitest.config.ts +10 -0
  95. package/McpSdk.ts +0 -14
  96. package/WebAgent.ts +0 -5
  97. package/WebMcp.ts +0 -26
  98. package/Zod.ts +0 -1
  99. package/dist/McpSdk.d.ts +0 -14
  100. package/dist/SimulatorMask-BHVXyogh-BFEGpD5S.js +0 -1048
  101. package/dist/SimulatorMask-BHVXyogh-CCYbrb84.js +0 -801
  102. package/dist/WebAgent.d.ts +0 -5
  103. package/dist/WebMcp.d.ts +0 -23
  104. package/dist/Zod.d.ts +0 -1
  105. package/dist/index.es.dev.js +0 -59017
  106. package/dist/index.es.js +0 -46795
  107. package/dist/index.umd.dev.js +0 -60355
  108. package/dist/index.umd.js +0 -1248
  109. package/dist/mcpsdk@1.25.3.dev.js +0 -22780
  110. package/dist/mcpsdk@1.25.3.es.dev.js +0 -22778
  111. package/dist/mcpsdk@1.25.3.es.js +0 -16960
  112. package/dist/mcpsdk@1.25.3.js +0 -48
  113. package/dist/transport/ExtensionClientTransport.d.ts +0 -24
  114. package/dist/transport/ExtensionContentServerTransport.d.ts +0 -39
  115. package/dist/transport/ExtensionPageServerTransport.d.ts +0 -36
  116. package/dist/transport/messages.d.ts +0 -9
  117. package/dist/vite.config.mcpSdk.d.ts +0 -2
  118. package/dist/vite.config.webAgent.d.ts +0 -2
  119. package/dist/vite.config.webMcp.d.ts +0 -2
  120. package/dist/vite.config.webMcpFull.d.ts +0 -2
  121. package/dist/vite.config.zod.d.ts +0 -2
  122. package/dist/webagent.dev.js +0 -49360
  123. package/dist/webagent.es.dev.js +0 -49071
  124. package/dist/webagent.es.js +0 -39219
  125. package/dist/webagent.js +0 -642
  126. package/dist/webmcp-full.dev.js +0 -31336
  127. package/dist/webmcp-full.es.dev.js +0 -30283
  128. package/dist/webmcp-full.es.js +0 -22889
  129. package/dist/webmcp-full.js +0 -645
  130. package/dist/webmcp.dev.js +0 -9572
  131. package/dist/webmcp.es.dev.js +0 -8518
  132. package/dist/webmcp.es.js +0 -6727
  133. package/dist/webmcp.js +0 -602
  134. package/dist/zod@3.25.76.dev.js +0 -4037
  135. package/dist/zod@3.25.76.es.dev.js +0 -4033
  136. package/dist/zod@3.25.76.es.js +0 -2945
  137. package/dist/zod@3.25.76.js +0 -1
  138. package/transport/ExtensionClientTransport.ts +0 -100
  139. package/transport/ExtensionContentServerTransport.ts +0 -162
  140. package/transport/ExtensionPageServerTransport.ts +0 -149
  141. package/transport/messages.ts +0 -63
  142. package/vite-build-tsc.ts +0 -63
  143. package/vite-env.d.ts +0 -10
  144. package/vite.config.mcpSdk.ts +0 -28
  145. package/vite.config.webAgent.ts +0 -19
  146. package/vite.config.webMcp.ts +0 -40
  147. package/vite.config.webMcpFull.ts +0 -19
  148. package/vite.config.zod.ts +0 -23
  149. /package/dist/{vite-build-tsc.d.ts → vite.config.runtime.d.ts} +0 -0
package/page-tools/page-agent-mask/checkDarkMode.ts ADDED
@@ -0,0 +1,181 @@
1
+ /**
2
+ * A comprehensive function to determine if the page is currently in a dark theme.
3
+ * Heuristic check. Only work for common patterns. Return false by default.
4
+ */
5
+ export function isPageDark() {
6
+ try {
7
+ if (hasDarkModeClass()) return true
8
+ if (hasDarkModeDataAttribute()) return true
9
+ if (isColorSchemeDark()) return true
10
+ if (isBackgroundDark()) return true
11
+ if (isMainContentBackgroundDark()) return true
12
+ if (isTextColorLight()) return true
13
+
14
+ return false
15
+ } catch (error) {
16
+ console.warn('Error determining if page is dark:', error)
17
+ return false
18
+ }
19
+ }
20
+
21
+ /**
22
+ * Checks for common dark mode CSS classes on the html or body elements.
23
+ */
24
+ function hasDarkModeClass() {
25
+ const DEFAULT_DARK_MODE_CLASSES = ['dark', 'dark-mode', 'theme-dark', 'night', 'night-mode']
26
+
27
+ const htmlElement = document.documentElement
28
+ const bodyElement = document.body || document.documentElement // can be null in some cases
29
+
30
+ // Check class names on <html> and <body>
31
+ for (const className of DEFAULT_DARK_MODE_CLASSES) {
32
+ if (htmlElement.classList.contains(className) || bodyElement?.classList.contains(className)) {
33
+ return true
34
+ }
35
+ }
36
+
37
+ return false
38
+ }
39
+
40
+ /**
41
+ * Some UI frameworks use data attributes to indicate theme
42
+ */
43
+ function hasDarkModeDataAttribute() {
44
+ const htmlElement = document.documentElement
45
+ const bodyElement = document.body || document.documentElement // can be null in some cases
46
+
47
+ const dataAttrs = ['data-theme', 'data-color-mode', 'data-bs-theme', 'data-mui-color-scheme']
48
+ for (const attr of dataAttrs) {
49
+ const bodyValue = bodyElement?.getAttribute(attr)
50
+ const htmlValue = htmlElement.getAttribute(attr)
51
+
52
+ if (bodyValue?.toLowerCase() === 'dark' || htmlValue?.toLowerCase() === 'dark') {
53
+ return true
54
+ }
55
+ }
56
+
57
+ return false
58
+ }
59
+
60
+ /**
61
+ * Checks the CSS `color-scheme` property and `<meta name="color-scheme">` tag.
62
+ * Only "dark"/"only dark" counts as dark; "light dark" is ambiguous and ignored.
63
+ */
64
+ function isColorSchemeDark() {
65
+ // Check <meta name="color-scheme" content="dark">
66
+ const meta = document.querySelector<HTMLMetaElement>('meta[name="color-scheme"]')
67
+ const metaContent = meta?.content.toLowerCase()
68
+ if (metaContent === 'dark' || metaContent === 'only dark') return true
69
+
70
+ // Check the computed color-scheme CSS property on :root
71
+ const rootStyle = window.getComputedStyle(document.documentElement)
72
+ const colorScheme = rootStyle.getPropertyValue('color-scheme').trim().toLowerCase()
73
+ return colorScheme === 'dark' || colorScheme === 'only dark'
74
+ }
75
+
76
+ /**
77
+ * Checks the background color of the body element to determine if the page is dark.
78
+ */
79
+ function isBackgroundDark() {
80
+ // We check both <html> and <body> because some pages set the color on <html>
81
+ const htmlStyle = window.getComputedStyle(document.documentElement)
82
+ const bodyStyle = window.getComputedStyle(document.body || document.documentElement)
83
+
84
+ // Get background colors
85
+ const htmlBgColor = htmlStyle.backgroundColor
86
+ const bodyBgColor = bodyStyle.backgroundColor
87
+
88
+ // The body's background might be transparent, in which case we should
89
+ // fall back to the html element's background.
90
+ if (isColorDark(bodyBgColor)) {
91
+ return true
92
+ } else if (bodyBgColor === 'transparent' || bodyBgColor.startsWith('rgba(0, 0, 0, 0)')) {
93
+ return isColorDark(htmlBgColor)
94
+ }
95
+
96
+ return false
97
+ }
98
+
99
+ /**
100
+ * Checks if the text color on the body is light, which implies a dark background.
101
+ */
102
+ function isTextColorLight() {
103
+ /** Luminance (0-255) above which body text is considered light */
104
+ const LIGHT_TEXT_LUMINANCE = 200
105
+
106
+ const bodyStyle = window.getComputedStyle(document.body || document.documentElement)
107
+ const luminance = getLuminance(bodyStyle.color)
108
+
109
+ // Light text has high luminance (e.g. white text on dark bg)
110
+ return luminance !== null && luminance > LIGHT_TEXT_LUMINANCE
111
+ }
112
+
113
+ /**
114
+ * Checks the background color of major layout elements (#app, #root, etc.).
115
+ * Many SPAs render into a container that may have its own dark background while
116
+ * <body> remains transparent.
117
+ */
118
+ function isMainContentBackgroundDark() {
119
+ const { innerWidth: vw, innerHeight: vh } = window
120
+ const minArea = vw * vh * 0.5
121
+
122
+ const selectors = ['#app', '#root', '#__next']
123
+ for (const selector of selectors) {
124
+ const el = document.querySelector(selector)
125
+ if (!el) continue
126
+
127
+ const rect = el.getBoundingClientRect()
128
+ if (rect.width * rect.height < minArea) continue
129
+
130
+ if (isColorDark(window.getComputedStyle(el).backgroundColor)) return true
131
+ }
132
+ return false
133
+ }
134
+
135
+ // --- utils ---
136
+
137
+ /**
138
+ * Parses an RGB or RGBA color string and returns an object with r, g, b properties.
139
+ * @param {string} colorString - e.g., "rgb(34, 34, 34)" or "rgba(0, 0, 0, 0.5)"
140
+ * @returns {{r: number, g: number, b: number}|null}
141
+ */
142
+ function parseRgbColor(colorString: string) {
143
+ const rgbMatch = /rgba?\((\d+),\s*(\d+),\s*(\d+)/.exec(colorString)
144
+ if (!rgbMatch) {
145
+ return null // Not a valid rgb/rgba string
146
+ }
147
+ return {
148
+ r: parseInt(rgbMatch[1]),
149
+ g: parseInt(rgbMatch[2]),
150
+ b: parseInt(rgbMatch[3]),
151
+ }
152
+ }
153
+
154
+ /**
155
+ * Calculates the perceived luminance (0-255) of a CSS color string.
156
+ * @param {string} colorString - e.g., "rgb(50, 50, 50)" or "rgba(0, 0, 0, 0.5)"
157
+ * @returns {number|null} - The luminance, or null if the color is transparent or unparseable.
158
+ */
159
+ function getLuminance(colorString: string): number | null {
160
+ if (!colorString || colorString === 'transparent' || colorString.startsWith('rgba(0, 0, 0, 0)')) {
161
+ return null // Transparent has no meaningful luminance
162
+ }
163
+
164
+ const rgb = parseRgbColor(colorString)
165
+ if (!rgb) {
166
+ return null // Could not parse color
167
+ }
168
+
169
+ // Standard perceived luminance formula
170
+ return 0.299 * rgb.r + 0.587 * rgb.g + 0.114 * rgb.b
171
+ }
172
+
173
+ /**
174
+ * Determines if a color is "dark" based on its calculated luminance.
175
+ * @param {string} colorString - The CSS color string (e.g., "rgb(50, 50, 50)").
176
+ * @param {number} threshold - A value between 0 and 255. Colors with luminance below this will be considered dark. Default is 128.
177
+ */
178
+ function isColorDark(colorString: string, threshold = 128) {
179
+ const luminance = getLuminance(colorString)
180
+ return luminance !== null && luminance < threshold
181
+ }
package/page-tools/page-agent-mask/cursor-border.svg ADDED
@@ -0,0 +1,3 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" fill="none">
2
+ <g><path d="M 15 42 L 15 36.99 Q 15 31.99 23.7 31.99 L 28.05 31.99 Q 32.41 31.99 32.41 21.99 L 32.41 17 Q 32.41 12 41.09 16.95 L 76.31 37.05 Q 85 42 76.31 46.95 L 41.09 67.05 Q 32.41 72 32.41 62.01 L 32.41 57.01 Q 32.41 52.01 23.7 52.01 L 19.35 52.01 Q 15 52.01 15 47.01 Z" fill="none" stroke="#000000" stroke-width="6" stroke-miterlimit="10" style="stroke: light-dark(rgb(0, 0, 0), rgb(255, 255, 255));"/></g>
3
+ </svg>
package/page-tools/page-agent-mask/cursor-fill.svg ADDED
@@ -0,0 +1,5 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
2
+ <defs>
3
+ </defs>
4
+ <g xmlns="http://www.w3.org/2000/svg" style="filter: drop-shadow(light-dark(rgba(0, 0, 0, 0.4), rgba(237, 237, 237, 0.4)) 3px 4px 4px);"><path d="M 15 42 L 15 36.99 Q 15 31.99 23.7 31.99 L 28.05 31.99 Q 32.41 31.99 32.41 21.99 L 32.41 17 Q 32.41 12 41.09 16.95 L 76.31 37.05 Q 85 42 76.31 46.95 L 41.09 67.05 Q 32.41 72 32.41 62.01 L 32.41 57.01 Q 32.41 52.01 23.7 52.01 L 19.35 52.01 Q 15 52.01 15 47.01 Z" fill="#ffffff" stroke="none" style="fill: #ffffff;"/></g>
5
+ </svg>
package/page-tools/page-agent-mask/cursor.module.css ADDED
@@ -0,0 +1,70 @@
1
+ .webmcp-page-agent-cursor {
2
+ position: absolute;
3
+ width: var(--cursor-size, 75px);
4
+ height: var(--cursor-size, 75px);
5
+ pointer-events: none;
6
+ z-index: 10000;
7
+ }
8
+
9
+ .webmcp-page-agent-cursorBorder {
10
+ position: absolute;
11
+ width: 100%;
12
+ height: 100%;
13
+ background: linear-gradient(45deg, rgb(57, 182, 255), rgb(189, 69, 251));
14
+ mask-image: url("data:image/svg+xml,%3csvg%20xmlns='http://www.w3.org/2000/svg'%20viewBox='0%200%20100%20100'%20fill='none'%3e%3cg%3e%3cpath%20d='M%2015%2042%20L%2015%2036.99%20Q%2015%2031.99%2023.7%2031.99%20L%2028.05%2031.99%20Q%2032.41%2031.99%2032.41%2021.99%20L%2032.41%2017%20Q%2032.41%2012%2041.09%2016.95%20L%2076.31%2037.05%20Q%2085%2042%2076.31%2046.95%20L%2041.09%2067.05%20Q%2032.41%2072%2032.41%2062.01%20L%2032.41%2057.01%20Q%2032.41%2052.01%2023.7%2052.01%20L%2019.35%2052.01%20Q%2015%2052.01%2015%2047.01%20Z'%20fill='none'%20stroke='%23000000'%20stroke-width='6'%20stroke-miterlimit='10'%20style='stroke:%20light-dark(rgb(0,%200,%200),%20rgb(255,%20255,%20255));'/%3e%3c/g%3e%3c/svg%3e");
15
+ mask-size: 100% 100%;
16
+ mask-repeat: no-repeat;
17
+
18
+ transform-origin: center;
19
+ transform: rotate(-90deg) scale(0.8);
20
+ margin-left: -10px;
21
+ margin-top: -18px;
22
+ }
23
+
24
+ .webmcp-page-agent-cursorFilling {
25
+ position: absolute;
26
+ width: 100%;
27
+ height: 100%;
28
+ background: url("data:image/svg+xml,%3csvg%20xmlns='http://www.w3.org/2000/svg'%20viewBox='0%200%20100%20100'%3e%3cdefs%3e%3c/defs%3e%3cg%20xmlns='http://www.w3.org/2000/svg'%20style='filter:%20drop-shadow(light-dark(rgba(0,%200,%200,%200.4),%20rgba(237,%20237,%20237,%200.4))%203px%204px%204px);'%3e%3cpath%20d='M%2015%2042%20L%2015%2036.99%20Q%2015%2031.99%2023.7%2031.99%20L%2028.05%2031.99%20Q%2032.41%2031.99%2032.41%2021.99%20L%2032.41%2017%20Q%2032.41%2012%2041.09%2016.95%20L%2076.31%2037.05%20Q%2085%2042%2076.31%2046.95%20L%2041.09%2067.05%20Q%2032.41%2072%2032.41%2062.01%20L%2032.41%2057.01%20Q%2032.41%2052.01%2023.7%2052.01%20L%2019.35%2052.01%20Q%2015%2052.01%2015%2047.01%20Z'%20fill='%23ffffff'%20stroke='none'%20style='fill:%20%23ffffff;'/%3e%3c/g%3e%3c/svg%3e");
29
+ background-size: 100% 100%;
30
+ background-repeat: no-repeat;
31
+
32
+ transform-origin: center;
33
+ transform: rotate(-90deg) scale(0.8);
34
+ margin-left: -10px;
35
+ margin-top: -18px;
36
+ }
37
+
38
+ .webmcp-page-agent-cursorRipple {
39
+ position: absolute;
40
+ width: 100%;
41
+ height: 100%;
42
+ pointer-events: none;
43
+ margin-left: -50%;
44
+ margin-top: -50%;
45
+
46
+ &::after {
47
+ content: '';
48
+ opacity: 0;
49
+ position: absolute;
50
+ inset: 0;
51
+ border: 4px solid rgba(57, 182, 255, 1);
52
+ border-radius: 50%;
53
+ }
54
+ }
55
+
56
+ .webmcp-page-agent-cursor.clicking .webmcp-page-agent-cursorRipple::after {
57
+ animation: webmcp-page-agent-cursor-ripple 300ms ease-out forwards;
58
+ }
59
+
60
+ @keyframes webmcp-page-agent-cursor-ripple {
61
+ 0% {
62
+ transform: scale(0);
63
+ opacity: 1;
64
+ }
65
+
66
+ 100% {
67
+ transform: scale(2);
68
+ opacity: 0;
69
+ }
70
+ }
package/page-tools/page-agent-mask/hauwei.svg ADDED
@@ -0,0 +1,25 @@
1
+ <svg viewBox="0 0 79.0834 79.4553" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="79.083374" height="79.455322" fill="none" customFrame="#000000">
2
+ --
3
+ <defs>
4
+ <g id="pixso_custom_effect_0">
5
+ <effect x="6.000000" y="6.000000" visibility="visible" fill="rgb(0,0,0)" fill-opacity="0.180000007" effectType="dropShadow" stdDeviation="20" radius="0" />
6
+ </g>
7
+ <filter id="filter_0" width="79.083374" height="79.455322" x="0.000000" y="0.000000" filterUnits="userSpaceOnUse" customEffect="url(#pixso_custom_effect_0)" color-interpolation-filters="sRGB">
8
+ <feFlood flood-opacity="0" result="BackgroundImageFix" />
9
+ <feOffset dx="6.000000" dy="6.000000" in="SourceAlpha" />
10
+ <feGaussianBlur stdDeviation="6.66666651" />
11
+ <feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.18 0 " />
12
+ <feBlend result="effect_dropShadow_1" in2="BackgroundImageFix" mode="normal" />
13
+ <feBlend result="shape" in="SourceGraphic" in2="effect_dropShadow_1" mode="normal" />
14
+ </filter>
15
+ <linearGradient id="paint_linear_0" x1="18.4743214" x2="18.4213886" y1="-1.86264515e-09" y2="34.9221306" gradientUnits="userSpaceOnUse">
16
+ <stop stop-color="rgb(255,181,96)" offset="0" stop-opacity="1" />
17
+ <stop stop-color="rgb(255,120,114)" offset="0.5" stop-opacity="1" />
18
+ <stop stop-color="rgb(241,109,237)" offset="1" stop-opacity="1" />
19
+ </linearGradient>
20
+ </defs>
21
+ <g filter="url(#filter_0)">
22
+ <path id="多边形 11" d="M20.2632 3.57771L34.6257 32.3027C35.4444 33.9402 33.8152 35.7258 32.1097 35.0603L19.2014 30.0229C18.7339 29.8404 18.2148 29.8404 17.7472 30.0229L4.83891 35.0603C3.13343 35.7258 1.50423 33.9402 2.32297 32.3027L16.6855 3.57771C17.4225 2.10361 19.5261 2.10361 20.2632 3.57771Z" fill="url(#paint_linear_0)" fill-rule="evenodd" transform="matrix(0.719908,-0.69407,0.69407,0.719908,2.00256,28.0095)" />
23
+ <path id="多边形 11" d="M37.2169 32.6848C37.1578 32.189 37.0096 31.7039 36.7723 31.2294L22.4098 2.5044C22.1973 2.07946 21.9328 1.70554 21.6163 1.38261C21.3524 1.11342 21.0524 0.879671 20.7162 0.681366C20.3925 0.490407 20.0558 0.34434 19.7061 0.243163C19.312 0.129145 18.9014 0.0721366 18.4743 0.0721366C18.0472 0.0721366 17.6367 0.129145 17.2426 0.243163C16.8929 0.34434 16.5562 0.490407 16.2324 0.681366C15.8962 0.879671 15.5962 1.11342 15.3323 1.38261C15.0158 1.70553 14.7513 2.07946 14.5388 2.5044L0.176342 31.2294C-0.0609121 31.7039 -0.209128 32.189 -0.268305 32.6848C-0.316251 33.0865 -0.305747 33.4951 -0.236792 33.9108C-0.171059 34.307 -0.0570567 34.6809 0.105216 35.0326C0.292784 35.4391 0.544845 35.8158 0.861397 36.1627C1.17794 36.5097 1.53001 36.7951 1.91761 37.0191C2.25297 37.2128 2.61492 37.3606 3.00348 37.4623C3.41106 37.5689 3.81704 37.6168 4.22142 37.6058C4.72053 37.5922 5.21719 37.4889 5.71141 37.2961L18.4743 32.3154L31.2372 37.2961C31.7315 37.4889 32.2281 37.5922 32.7272 37.6058C33.1316 37.6168 33.5376 37.5689 33.9452 37.4623C34.3337 37.3606 34.6957 37.2128 35.0311 37.0191C35.4187 36.7951 35.7707 36.5097 36.0873 36.1627C36.4038 35.8158 36.6558 35.4391 36.8434 35.0327C37.0057 34.681 37.1197 34.307 37.1854 33.9108C37.2544 33.4951 37.2649 33.0865 37.2169 32.6848ZM34.6257 32.3027L20.2632 3.57771C19.5261 2.10361 17.4225 2.10361 16.6855 3.57771L2.32297 32.3027C1.50423 33.9402 3.13343 35.7258 4.83891 35.0603L17.7472 30.0229C18.2148 29.8404 18.7339 29.8404 19.2014 30.0229L32.1097 35.0603C33.8152 35.7258 35.4444 33.9402 34.6257 32.3027Z" fill="rgb(255,255,255)" fill-rule="evenodd" transform="matrix(0.719908,-0.69407,0.69407,0.719908,2.00256,28.0095)" />
24
+ </g>
25
+ </svg>
package/page-tools/page-agent-prompt.md CHANGED
@@ -26,35 +26,48 @@
26
26
 
27
27
  - url: 当前查看页面的 URL。
28
28
  - title: 当前查看页面的标题。
29
- - 可交互元素总数：带有 `[ref=N]` 标记的元素数量。
29
+ - 可交互元素总数：带有 `#N` 标记的元素数量。
30
30
  - YAML 树：页面的语义化结构，格式如下：
31
31
 
32
32
  ```yaml
33
33
  - region:
34
- - main:
35
- - button [cursor=pointer] [ref=9]: 产品文档
36
- - button [selected] [cursor=pointer] [ref=47]: 40元/月 2核CPU 2GB内存
37
- - radio [checked] [cursor=pointer] [ref=53]: 自动生成密码
38
- - button [cursor=pointer] [ref=74]: 立即购买
34
+ - main:
35
+ - button #9 [cursor=pointer] "产品文档"
36
+ - button #47 [selected] [cursor=pointer] "40元/月 2核CPU 2GB内存"
37
+ - radio #53 [checked] [cursor=pointer] "自动生成密码"
38
+ - button #74 [cursor=pointer] "立即购买"
39
+ - generic #6 [cf-uba="serviceList..Flexus云服务"] "Flexus云服务"
39
40
  ```
40
41
 
41
42
  节点格式说明：
42
- - `- role [state1] [state2] [ref=N]: accessible name`
43
- - `role`：ARIA 语义角色（如 button/link/radio/heading/list/listitem 等）
44
- - `[state]`：可选状态标记，如 `[checked]` `[selected]` `[disabled]` `[expanded]` `[cursor=pointer]`
45
- - `[ref=N]`：可交互元素的唯一操作索引，**只有带 ref 的节点才能被操作**
46
- - `accessible name`：元素的语义化名称（通过 aria-label/aria-labelledby/innerText 等计算得出）
43
+ - `- role #N [token1] [token2] "accessible name"`
44
+ - `role`：ARIA 语义角色（如 button/link/radio/heading/listitem/generic 等）
45
+ - `#N`：可交互元素的唯一操作索引，**只有带 `#N` 的节点才能被操作**，操作时将 N 作为 `index` 参数传入
46
+ - `[token]`：可选 token，包括状态标记（如 `[checked]` `[selected]` `[pressed]` `[current]` `[expanded]` `[disabled]` `[readonly]` `[required]` `[invalid]` `[busy]` `[error]` `[warning]` `[cursor=pointer]`）和定制属性（如 `[cf-uba="..."]`）
47
+ - `"accessible name"`：元素的语义化名称，**用双引号包裹**（通过 aria-label/aria-labelledby/innerText 等计算得出）；无名称的节点此字段省略
47
48
  - 缩进表示父子关系
48
49
 
49
50
  </browser_state>
50
51
 
52
+ <responseMode>
53
+
54
+ `page-agent-tool` 支持 `responseMode` 参数，用于控制操作后返回的页面状态形式：
55
+
56
+ - **`diff`**（默认）：仅返回自上一次状态以来的增量 DOM 差异，极大节省 Token。
57
+ - **`full`**：返回当前视口中完整的语义化 ARIA YAML 树。
58
+ - **`both`**：同时返回全量树和增量差异。
59
+
60
+ 执行 `click`、`fill`、`select`、`scroll` 操作后，工具默认自动以 `diff` 模式返回最新页面状态，无需再次手动调用 `browserState`。
61
+
62
+ </responseMode>
63
+
51
64
  <browser_state_diff>
52
65
 
53
66
  在交互过程中，为了提高效率和减少上下文占用，工具会生成页面的增量差异（Diff）信息。
54
67
  Diff 格式示例如下：
55
68
  ```diff
56
- - button [ref=9]: 产品文档
57
- + button [ref=9]: 产品文档 (已点击)
69
+ - button #9: 产品文档
70
+ + button #9: 产品文档 (已点击)
58
71
  ```
59
72
  或者，当页面结构发生改变时，会展示新增或移除的节点差异。
60
73
 
@@ -62,14 +75,14 @@ Diff 格式示例如下：
62
75
 
63
76
  > 与业界 AI 编辑器（Cursor / Windsurf）按需读取文件而非全量加载的理念完全一致——**先精准搜索，再按需拉取全量**，将发送给大模型的 token 降至最低。
64
77
 
65
- 1. **按关键词精准搜索（最高优先级）**：当你已知要找什么类型的元素（如按钮、输入框、特定名称的链接）时，**优先使用 `searchTree` 动作**，传入 role 名称、元素文本或状态关键词搜索。它只返回命中行和上下文，token 消耗比全量树减少 80%+ 。
78
+ 1. **按关键词精准搜索（最高优先级）**：当你已知要找什么类型的元素（如按钮、输入框、特定名称的链接）时，**优先使用 `searchTree` 动作**，传入 role 名称、元素文本或状态关键词搜索。它只返回命中行和上下文，token 消耗比全量树减少 80%+。
66
79
  - 示例：查找所有按钮 → `{"action": "searchTree", "query": "button"}`
67
80
  - 示例：查找含"提交"文本的节点 → `{"action": "searchTree", "query": "提交"}`
68
81
  - 示例：查找已勾选的复选框 → `{"action": "searchTree", "query": "checked"}`
69
82
  - 示例：精确定位某个 ref → `{"action": "searchTree", "query": "#5"}`
70
83
  2. **首次获取全量**：首次进入页面、页面发生重大刷新、或 `searchTree` 无法找到所需信息时，调用 `browserState` 并指定 `responseMode` 为 `full` 或 `both` 获取完整 A11y 树。
71
84
  3. **增量优先**：执行 `click`、`fill`、`select`、`scroll` 操作后，工具默认自动返回 `diff` 增量信息。优先阅读这些 Diff，以便快速确认操作是否生效。
72
- 4. **按需拉取全量**：如果增量 Diff 不足以支持下一步操作，或需要寻找不在 Diff 中的 `[ref=N]` 节点，且 `searchTree` 也无法定位时，再显式调用 `browserState` 拉取完整树。
85
+ 4. **按需拉取全量**：如果增量 Diff 不足以支持下一步操作，或需要寻找不在 Diff 中的 `#N` 节点，且 `searchTree` 也无法定位时，再显式调用 `browserState` 拉取完整树。
73
86
 
74
87
  </browser_state_diff>
75
88
 
@@ -78,9 +91,9 @@ Diff 格式示例如下：
78
91
  <browser_rules>
79
92
  在使用浏览器和浏览网页时严格遵守以下规则：
80
93
 
81
- - 仅与分配有 `[ref=N]` 的元素进行交互。
82
- - 仅使用明确提供的 ref 索引。
83
- - 每次操作后 ref 索引会重新分配，不要使用旧的 ref 索引。
94
+ - 仅与分配有 `#N` 索引的元素进行交互，将 N 作为 `index` 参数传入。
95
+ - 仅使用明确出现在树中的 `#N` 索引，切勿猜测或沿用过期索引。
96
+ - 每次操作后 `#N` 索引会重新分配，不要使用旧的索引。
84
97
  - 如果页面在执行操作后发生变化（例如输入文本操作），分析是否需要与新元素进行交互，例如从列表中选择正确的选项。
85
98
  - 默认情况下，仅列出可见视口中的元素。如果你怀疑需要交互的相关内容在屏幕外，请使用滚动操作。仅在页面下方或上方还有更多像素时才滚动。
86
99
  - 你可以使用 num_pages 参数滚动特定页数（例如，0.5 表示半页，2.0 表示两页）。
@@ -91,6 +104,9 @@ Diff 格式示例如下：
91
104
  - 如果 <user_request> 包含特定的页面信息，如产品类型、评分、价格、位置等，尝试应用过滤器以提高效率。
92
105
  - <user_request> 是最终目标。如果用户指定了明确的步骤，它们始终具有最高优先级。
93
106
  - 如果你向字段输入文本，可能需要按回车键、点击搜索按钮或从下拉列表中选择以完成操作。
107
+ - **不要通过 executeJavascript 向页面插入 DOM 元素**（如创建 div 显示调试信息）。这会污染页面结构并影响后续操作。如需读取数据，使用 return 语句返回结果即可。
108
+ - 如果操作返回中出现 `[校验提示]`，说明页面有表单校验错误未通过，请先根据提示修复错误后再继续操作。
109
+ - 如果操作后页面没有发生预期变化，优先检查是否有 `[校验提示]` 或 `[error]` token 的错误信息，而非重复操作。
94
110
  - 如果不需要，不要登录页面。如果没有凭据，不要登录。
95
111
  - 有两种类型的任务，首先思考你正在处理哪种类型的请求：
96
112
 
package/page-tools/page-agent-tool-event.ts ADDED
@@ -0,0 +1,113 @@
1
+ import { inputSchema, type PageAgentToolInput, type PageAgentToolRawInput } from './schema'
2
+ import { PageController } from '@page-agent/page-controller'
3
+
4
+ export const PAGE_AGENT_TOOL_CALL_EVENT = 'page-agent-tool-call'
5
+ export const PAGE_AGENT_TOOL_RESULT_EVENT = 'page-agent-tool-result'
6
+ export const PAGE_AGENT_CHAT_END_EVENT = 'page-agent-chat-end'
7
+
8
+ export type PageAgentToolCallEventDetail = {
9
+ data?: PageAgentToolRawInput
10
+ requestId: string
11
+ }
12
+
13
+ export type PageAgentToolResultEventDetail = {
14
+ data: {
15
+ success: boolean
16
+ result?: unknown
17
+ error?: string
18
+ }
19
+ requestId: string
20
+ }
21
+
22
+ type ExecutePageAgentTool = (data: PageAgentToolInput) => Promise<unknown>
23
+
24
+ function isPageAgentToolErrorResult(result: unknown): result is { isError: true; error: string } {
25
+ return (
26
+ typeof result === 'object' &&
27
+ result !== null &&
28
+ 'isError' in result &&
29
+ result.isError === true &&
30
+ 'error' in result &&
31
+ typeof result.error === 'string'
32
+ )
33
+ }
34
+
35
+ declare global {
36
+ interface Window {
37
+ __nextSdkPageAgentToolEventCleanup?: () => void
38
+ }
39
+ }
40
+
41
+ function dispatchPageAgentToolResult(detail: PageAgentToolResultEventDetail) {
42
+ window.dispatchEvent(new CustomEvent(PAGE_AGENT_TOOL_RESULT_EVENT, { detail }))
43
+ }
44
+
45
+ export function setupPageAgentToolEventBridge(
46
+ executePageAgentTool: ExecutePageAgentTool,
47
+ pageController: PageController
48
+ ) {
49
+ window.__nextSdkPageAgentToolEventCleanup?.()
50
+
51
+ const handleToolCall = async (event: Event) => {
52
+ const detail = (event as CustomEvent<PageAgentToolCallEventDetail>).detail
53
+ const requestId = detail?.requestId
54
+
55
+ if (!requestId) {
56
+ dispatchPageAgentToolResult({
57
+ requestId: '',
58
+ data: {
59
+ success: false,
60
+ error: '缺少 requestId'
61
+ }
62
+ })
63
+ return
64
+ }
65
+
66
+ try {
67
+ const data = inputSchema.parse(detail.data)
68
+ const result = await executePageAgentTool(data)
69
+ if (isPageAgentToolErrorResult(result)) {
70
+ dispatchPageAgentToolResult({
71
+ requestId,
72
+ data: {
73
+ success: false,
74
+ error: result.error,
75
+ result
76
+ }
77
+ })
78
+ return
79
+ }
80
+ dispatchPageAgentToolResult({
81
+ requestId,
82
+ data: {
83
+ success: true,
84
+ result
85
+ }
86
+ })
87
+ } catch (error) {
88
+ dispatchPageAgentToolResult({
89
+ requestId,
90
+ data: {
91
+ success: false,
92
+ error: error instanceof Error ? error.message : String(error)
93
+ }
94
+ })
95
+ }
96
+ }
97
+
98
+ // 聊天结束事件，用于在聊天结束时移除遮罩
99
+ const handleChatEnd = async () => {
100
+ try {
101
+ await pageController.hideMask()
102
+ } catch (error) {}
103
+ }
104
+
105
+ window.addEventListener(PAGE_AGENT_TOOL_CALL_EVENT, handleToolCall)
106
+ window.addEventListener(PAGE_AGENT_CHAT_END_EVENT, handleChatEnd)
107
+
108
+ window.__nextSdkPageAgentToolEventCleanup = () => {
109
+ window.removeEventListener(PAGE_AGENT_TOOL_CALL_EVENT, handleToolCall)
110
+ window.removeEventListener(PAGE_AGENT_CHAT_END_EVENT, handleChatEnd)
111
+ window.__nextSdkPageAgentToolEventCleanup = undefined
112
+ }
113
+ }