@weavetab/mcp 2.5.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (281) hide show
  1. package/AGENTS.md +48 -0
  2. package/ARCHITECTURE.md +590 -0
  3. package/LICENSE +631 -0
  4. package/README.md +909 -0
  5. package/SECURITY.md +196 -0
  6. package/TROUBLESHOOTING.md +365 -0
  7. package/dist/anticaptcha/detector.d.ts +49 -0
  8. package/dist/anticaptcha/detector.js +465 -0
  9. package/dist/anticaptcha/integration.d.ts +36 -0
  10. package/dist/anticaptcha/integration.js +123 -0
  11. package/dist/anticaptcha/solver.d.ts +27 -0
  12. package/dist/anticaptcha/solver.js +552 -0
  13. package/dist/audit/logger.d.ts +13 -0
  14. package/dist/audit/logger.js +177 -0
  15. package/dist/audit/telemetry.d.ts +36 -0
  16. package/dist/audit/telemetry.js +162 -0
  17. package/dist/cdp/bridge.d.ts +16 -0
  18. package/dist/cdp/bridge.js +826 -0
  19. package/dist/cdp/confirm.d.ts +26 -0
  20. package/dist/cdp/confirm.js +79 -0
  21. package/dist/cdp/connector.d.ts +37 -0
  22. package/dist/cdp/connector.js +661 -0
  23. package/dist/cdp/extension.d.ts +36 -0
  24. package/dist/cdp/extension.js +118 -0
  25. package/dist/cdp/finder.d.ts +1 -0
  26. package/dist/cdp/finder.js +2 -0
  27. package/dist/cdp/helpers.d.ts +11 -0
  28. package/dist/cdp/helpers.js +195 -0
  29. package/dist/cdp/label_cache.d.ts +12 -0
  30. package/dist/cdp/label_cache.js +51 -0
  31. package/dist/cdp/lock.d.ts +33 -0
  32. package/dist/cdp/lock.js +206 -0
  33. package/dist/cdp/omni_broker.d.ts +17 -0
  34. package/dist/cdp/omni_broker.js +119 -0
  35. package/dist/cdp/profiles.d.ts +1 -0
  36. package/dist/cdp/profiles.js +87 -0
  37. package/dist/cdp/pruner.d.ts +41 -0
  38. package/dist/cdp/pruner.js +263 -0
  39. package/dist/cdp/ref_cache.d.ts +101 -0
  40. package/dist/cdp/ref_cache.js +234 -0
  41. package/dist/cdp/retry.d.ts +5 -0
  42. package/dist/cdp/retry.js +39 -0
  43. package/dist/cdp/retry_enhanced.d.ts +44 -0
  44. package/dist/cdp/retry_enhanced.js +102 -0
  45. package/dist/cdp/security.d.ts +50 -0
  46. package/dist/cdp/security.js +237 -0
  47. package/dist/cdp/verify.d.ts +39 -0
  48. package/dist/cdp/verify.js +143 -0
  49. package/dist/cdp/walker.d.ts +91 -0
  50. package/dist/cdp/walker.js +896 -0
  51. package/dist/cdp/walker.test.d.ts +2 -0
  52. package/dist/cdp/walker.test.js +172 -0
  53. package/dist/cdp/websocket_transport.d.ts +15 -0
  54. package/dist/cdp/websocket_transport.js +118 -0
  55. package/dist/cli/plugin.d.ts +9 -0
  56. package/dist/cli/plugin.js +1134 -0
  57. package/dist/cli/updater.d.ts +1 -0
  58. package/dist/cli/updater.js +72 -0
  59. package/dist/config/loader.d.ts +120 -0
  60. package/dist/config/loader.js +704 -0
  61. package/dist/index.d.ts +2 -0
  62. package/dist/index.js +1248 -0
  63. package/dist/intelligence/hints.d.ts +11 -0
  64. package/dist/intelligence/hints.js +39 -0
  65. package/dist/intelligence/intent.d.ts +33 -0
  66. package/dist/intelligence/intent.js +171 -0
  67. package/dist/intelligence/sessionMemory.d.ts +66 -0
  68. package/dist/intelligence/sessionMemory.js +284 -0
  69. package/dist/intelligence/sessionMemory.test.d.ts +2 -0
  70. package/dist/intelligence/sessionMemory.test.js +62 -0
  71. package/dist/intelligence/trail.d.ts +64 -0
  72. package/dist/intelligence/trail.js +233 -0
  73. package/dist/intelligence/trail.test.d.ts +2 -0
  74. package/dist/intelligence/trail.test.js +110 -0
  75. package/dist/intelligence/vision.d.ts +24 -0
  76. package/dist/intelligence/vision.js +151 -0
  77. package/dist/lifecycle/mission_manager.d.ts +16 -0
  78. package/dist/lifecycle/mission_manager.js +44 -0
  79. package/dist/overlay/bubble.d.ts +16 -0
  80. package/dist/overlay/bubble.js +37 -0
  81. package/dist/overlay/ghost.d.ts +35 -0
  82. package/dist/overlay/ghost.js +508 -0
  83. package/dist/overlay/injector.d.ts +22 -0
  84. package/dist/overlay/injector.js +198 -0
  85. package/dist/plugin/interface.d.ts +134 -0
  86. package/dist/plugin/interface.js +1 -0
  87. package/dist/plugin/loader.d.ts +18 -0
  88. package/dist/plugin/loader.js +155 -0
  89. package/dist/plugin/registry.d.ts +50 -0
  90. package/dist/plugin/registry.js +234 -0
  91. package/dist/security/blacklist.d.ts +24 -0
  92. package/dist/security/blacklist.js +80 -0
  93. package/dist/security/filter.d.ts +2 -0
  94. package/dist/security/filter.js +20 -0
  95. package/dist/security/guard.d.ts +8 -0
  96. package/dist/security/guard.js +35 -0
  97. package/dist/security/ratelimit.d.ts +11 -0
  98. package/dist/security/ratelimit.js +57 -0
  99. package/dist/security/secrets.d.ts +75 -0
  100. package/dist/security/secrets.js +365 -0
  101. package/dist/sensors/actionability.d.ts +81 -0
  102. package/dist/sensors/actionability.js +303 -0
  103. package/dist/sensors/guard.d.ts +25 -0
  104. package/dist/sensors/guard.js +168 -0
  105. package/dist/sensors/mutations.d.ts +23 -0
  106. package/dist/sensors/mutations.js +93 -0
  107. package/dist/sensors/network.d.ts +67 -0
  108. package/dist/sensors/network.js +193 -0
  109. package/dist/sensors/operation_monitor.d.ts +80 -0
  110. package/dist/sensors/operation_monitor.js +203 -0
  111. package/dist/sensors/pacing.d.ts +44 -0
  112. package/dist/sensors/pacing.js +111 -0
  113. package/dist/sensors/thoughts.d.ts +2 -0
  114. package/dist/sensors/thoughts.js +59 -0
  115. package/dist/sensors/wait.d.ts +12 -0
  116. package/dist/sensors/wait.js +44 -0
  117. package/dist/server.d.ts +7 -0
  118. package/dist/server.js +1687 -0
  119. package/dist/state/agent.d.ts +36 -0
  120. package/dist/state/agent.js +101 -0
  121. package/dist/state/cache.d.ts +16 -0
  122. package/dist/state/cache.js +92 -0
  123. package/dist/state/cache.test.d.ts +2 -0
  124. package/dist/state/cache.test.js +76 -0
  125. package/dist/state/delta.d.ts +71 -0
  126. package/dist/state/delta.js +234 -0
  127. package/dist/state/delta.test.d.ts +2 -0
  128. package/dist/state/delta.test.js +279 -0
  129. package/dist/state/groups.d.ts +15 -0
  130. package/dist/state/groups.js +80 -0
  131. package/dist/state/session.d.ts +30 -0
  132. package/dist/state/session.js +155 -0
  133. package/dist/tools/accessibility.d.ts +14 -0
  134. package/dist/tools/accessibility.js +32 -0
  135. package/dist/tools/ask.d.ts +14 -0
  136. package/dist/tools/ask.js +49 -0
  137. package/dist/tools/automation.d.ts +36 -0
  138. package/dist/tools/automation.js +218 -0
  139. package/dist/tools/browser_detect.d.ts +13 -0
  140. package/dist/tools/browser_detect.js +73 -0
  141. package/dist/tools/burst.d.ts +35 -0
  142. package/dist/tools/burst.js +344 -0
  143. package/dist/tools/canvas.d.ts +25 -0
  144. package/dist/tools/canvas.js +132 -0
  145. package/dist/tools/captcha.d.ts +38 -0
  146. package/dist/tools/captcha.js +209 -0
  147. package/dist/tools/click.d.ts +36 -0
  148. package/dist/tools/click.js +561 -0
  149. package/dist/tools/click_and_wait.d.ts +33 -0
  150. package/dist/tools/click_and_wait.js +149 -0
  151. package/dist/tools/clipboard.d.ts +29 -0
  152. package/dist/tools/clipboard.js +279 -0
  153. package/dist/tools/console.d.ts +21 -0
  154. package/dist/tools/console.js +107 -0
  155. package/dist/tools/cookies.d.ts +67 -0
  156. package/dist/tools/cookies.js +121 -0
  157. package/dist/tools/design_extract.d.ts +109 -0
  158. package/dist/tools/design_extract.js +641 -0
  159. package/dist/tools/dialog.d.ts +24 -0
  160. package/dist/tools/dialog.js +109 -0
  161. package/dist/tools/drag.d.ts +15 -0
  162. package/dist/tools/drag.js +29 -0
  163. package/dist/tools/emulation.d.ts +33 -0
  164. package/dist/tools/emulation.js +69 -0
  165. package/dist/tools/eval.d.ts +20 -0
  166. package/dist/tools/eval.js +92 -0
  167. package/dist/tools/fill.d.ts +41 -0
  168. package/dist/tools/fill.js +667 -0
  169. package/dist/tools/find.d.ts +30 -0
  170. package/dist/tools/find.js +60 -0
  171. package/dist/tools/geo.d.ts +19 -0
  172. package/dist/tools/geo.js +29 -0
  173. package/dist/tools/github/analyze.d.ts +4 -0
  174. package/dist/tools/github/analyze.js +113 -0
  175. package/dist/tools/github/cache.d.ts +21 -0
  176. package/dist/tools/github/cache.js +28 -0
  177. package/dist/tools/github/index.d.ts +4 -0
  178. package/dist/tools/github/index.js +4 -0
  179. package/dist/tools/github/issues.d.ts +9 -0
  180. package/dist/tools/github/issues.js +88 -0
  181. package/dist/tools/github/pr.d.ts +5 -0
  182. package/dist/tools/github/pr.js +88 -0
  183. package/dist/tools/github/rateLimiter.d.ts +4 -0
  184. package/dist/tools/github/rateLimiter.js +46 -0
  185. package/dist/tools/github/read.d.ts +34 -0
  186. package/dist/tools/github/read.js +74 -0
  187. package/dist/tools/github/repoContext.d.ts +20 -0
  188. package/dist/tools/github/repoContext.js +129 -0
  189. package/dist/tools/highlight.d.ts +20 -0
  190. package/dist/tools/highlight.js +140 -0
  191. package/dist/tools/hover.d.ts +19 -0
  192. package/dist/tools/hover.js +57 -0
  193. package/dist/tools/inspect.d.ts +17 -0
  194. package/dist/tools/inspect.js +125 -0
  195. package/dist/tools/integration.test.d.ts +2 -0
  196. package/dist/tools/integration.test.js +96 -0
  197. package/dist/tools/key.d.ts +25 -0
  198. package/dist/tools/key.js +173 -0
  199. package/dist/tools/macro_compiler.d.ts +7 -0
  200. package/dist/tools/macro_compiler.js +82 -0
  201. package/dist/tools/mouse.d.ts +21 -0
  202. package/dist/tools/mouse.js +62 -0
  203. package/dist/tools/navigate.d.ts +16 -0
  204. package/dist/tools/navigate.js +188 -0
  205. package/dist/tools/network_intercept.d.ts +17 -0
  206. package/dist/tools/network_intercept.js +151 -0
  207. package/dist/tools/office/index.d.ts +9 -0
  208. package/dist/tools/office/index.js +7 -0
  209. package/dist/tools/office/list.d.ts +15 -0
  210. package/dist/tools/office/list.js +41 -0
  211. package/dist/tools/office/parse.d.ts +20 -0
  212. package/dist/tools/office/parse.js +172 -0
  213. package/dist/tools/pdf.d.ts +22 -0
  214. package/dist/tools/pdf.js +42 -0
  215. package/dist/tools/performance.d.ts +12 -0
  216. package/dist/tools/performance.js +47 -0
  217. package/dist/tools/performance_test.d.ts +70 -0
  218. package/dist/tools/performance_test.js +128 -0
  219. package/dist/tools/plan.d.ts +39 -0
  220. package/dist/tools/plan.js +161 -0
  221. package/dist/tools/pointer.d.ts +35 -0
  222. package/dist/tools/pointer.js +522 -0
  223. package/dist/tools/read.d.ts +30 -0
  224. package/dist/tools/read.js +488 -0
  225. package/dist/tools/recording.d.ts +45 -0
  226. package/dist/tools/recording.js +124 -0
  227. package/dist/tools/response.d.ts +129 -0
  228. package/dist/tools/response.js +5 -0
  229. package/dist/tools/scrape.d.ts +24 -0
  230. package/dist/tools/scrape.js +140 -0
  231. package/dist/tools/screenshot.d.ts +40 -0
  232. package/dist/tools/screenshot.js +79 -0
  233. package/dist/tools/scroll.d.ts +23 -0
  234. package/dist/tools/scroll.js +196 -0
  235. package/dist/tools/select.d.ts +13 -0
  236. package/dist/tools/select.js +217 -0
  237. package/dist/tools/snapshot.d.ts +27 -0
  238. package/dist/tools/snapshot.js +99 -0
  239. package/dist/tools/storage.d.ts +18 -0
  240. package/dist/tools/storage.js +29 -0
  241. package/dist/tools/tabs.d.ts +77 -0
  242. package/dist/tools/tabs.js +342 -0
  243. package/dist/tools/thought.d.ts +16 -0
  244. package/dist/tools/thought.js +79 -0
  245. package/dist/tools/type.d.ts +65 -0
  246. package/dist/tools/type.js +1061 -0
  247. package/dist/tools/type_secret.d.ts +23 -0
  248. package/dist/tools/type_secret.js +229 -0
  249. package/dist/tools/upload.d.ts +19 -0
  250. package/dist/tools/upload.js +103 -0
  251. package/dist/tools/video_compiler.d.ts +24 -0
  252. package/dist/tools/video_compiler.js +199 -0
  253. package/dist/tools/viewport.d.ts +44 -0
  254. package/dist/tools/viewport.js +230 -0
  255. package/dist/tools/wait.d.ts +19 -0
  256. package/dist/tools/wait.js +202 -0
  257. package/dist/tools/wait_for.d.ts +23 -0
  258. package/dist/tools/wait_for.js +30 -0
  259. package/dist/tools/windows.d.ts +22 -0
  260. package/dist/tools/windows.js +57 -0
  261. package/dist/ui/index.d.ts +1 -0
  262. package/dist/ui/index.js +12 -0
  263. package/docs/DEV_MODE.md +68 -0
  264. package/docs/DOCUMENTATION.md +532 -0
  265. package/docs/PLUGINS.md +538 -0
  266. package/docs/TOOLS.md +104 -0
  267. package/docs/dev_plugin.md +310 -0
  268. package/ext-dist/background.js +467 -0
  269. package/ext-dist/content.js +3089 -0
  270. package/ext-dist/dashboard.html +583 -0
  271. package/ext-dist/hud/hud.js +73 -0
  272. package/ext-dist/hud/index.html +20 -0
  273. package/ext-dist/manifest.json +77 -0
  274. package/ext-dist/sidepanel/sidepanel.js +25 -0
  275. package/ext-dist/sidepanel.html +547 -0
  276. package/ext-dist/styles.css +218 -0
  277. package/icon.png +0 -0
  278. package/mcp.json +796 -0
  279. package/package.json +113 -0
  280. package/server.json +23 -0
  281. package/smithery.yaml +11 -0
@@ -0,0 +1,522 @@
1
+ /**
2
+ * Weavetab — tools/pointer.ts
3
+ * Unified tool for all mouse/pointer interactions: click, double_click, hover, drag_and_drop.
4
+ * Supports targeting by ref ID (e.g. w:42) or absolute coordinates (x, y).
5
+ * Replaces mouse.ts, hover.ts, and drag.ts.
6
+ * Weavetab by fy2ne
7
+ */
8
+ import { logAction } from "../audit/logger.js";
9
+ import { recordAction } from "../intelligence/trail.js";
10
+ import { ghostMove, ghostClick } from "../overlay/ghost.js";
11
+ import { waitForDOMStable } from "../sensors/guard.js";
12
+ import { buildActionMap, findAllMatches, resolveByAria, safeSelector } from "../cdp/walker.js";
13
+ import { getCachedActionMap, cacheActionMap } from "../state/cache.js";
14
+ import { computeDelta, storeSnapshot } from "../state/delta.js";
15
+ import { sessionRegistry } from "../state/session.js";
16
+ import { getConfigAgentId } from "../config/loader.js";
17
+ import { browserWait } from "./wait.js";
18
+ function withTimeout(promise, ms, label) {
19
+ return Promise.race([
20
+ promise,
21
+ new Promise((_, reject) => setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms)),
22
+ ]);
23
+ }
24
+ async function resolveRefToCoords(session, targetId, ref, currentUrl) {
25
+ let map = getCachedActionMap(targetId, currentUrl);
26
+ if (!map) {
27
+ map = await buildActionMap(session, { lite: true }, targetId);
28
+ cacheActionMap(targetId, map);
29
+ }
30
+ let element = map.elements.find((el) => el.id === ref || el.id.endsWith(ref.startsWith("w:") ? ref.slice(2) : ref));
31
+ if (!element && ref && !ref.startsWith("w:")) {
32
+ element = {
33
+ id: ref,
34
+ label: ref,
35
+ selector: ref,
36
+ role: "element",
37
+ group: "utility",
38
+ depth: 0,
39
+ fingerprint: "selector:" + ref,
40
+ };
41
+ }
42
+ if (!element)
43
+ return null;
44
+ const selector = safeSelector(element);
45
+ if (!selector)
46
+ return null;
47
+ // Scroll element into view before resolving coordinates
48
+ await session.Runtime.evaluate({
49
+ expression: `(function(sel) {
50
+ const el = document.querySelector(sel);
51
+ if (el) {
52
+ if (el.scrollIntoViewIfNeeded) el.scrollIntoViewIfNeeded();
53
+ else el.scrollIntoView({behavior:'instant',block:'center'});
54
+ }
55
+ })(${JSON.stringify(selector)})`,
56
+ returnByValue: true
57
+ }).catch(() => { });
58
+ await new Promise(r => setTimeout(r, 100));
59
+ // Try the live index coords first
60
+ const coordsRes = await session.Runtime.evaluate({
61
+ expression: `(window.__wt_getCoords || window.__Weavetab_getCoords || function(){return null;})(${JSON.stringify(selector)})`,
62
+ returnByValue: true
63
+ }).catch(() => null);
64
+ if (coordsRes?.result?.value) {
65
+ const v = coordsRes.result.value;
66
+ if (v.x !== 0 || v.y !== 0)
67
+ return { x: v.x, y: v.y, selector };
68
+ }
69
+ // Fallback 1: use element bounding rect directly via CDP evaluate
70
+ const bboxRes = await session.Runtime.evaluate({
71
+ expression: `(function(sel) {
72
+ const el = document.querySelector(sel);
73
+ if (!el) return null;
74
+ // Primary: getBoundingClientRect
75
+ const r = el.getBoundingClientRect();
76
+ if (r.width > 0 || r.height > 0) {
77
+ return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) };
78
+ }
79
+ // Fallback for inline text elements (e.g. Wikipedia lang links).
80
+ // Inline elements can have zero-size bounding rects but non-zero client rects
81
+ // for their first text-box fragment.
82
+ const rects = el.getClientRects();
83
+ if (rects && rects.length > 0) {
84
+ const fr = rects[0];
85
+ if (fr.width > 0 || fr.height > 0) {
86
+ return { x: Math.round(fr.left + fr.width / 2), y: Math.round(fr.top + fr.height / 2) };
87
+ }
88
+ }
89
+ return null;
90
+ })(${JSON.stringify(selector)})`,
91
+ returnByValue: true
92
+ }).catch(() => null);
93
+ if (bboxRes?.result?.value) {
94
+ const v = bboxRes.result.value;
95
+ return { x: v.x, y: v.y, selector };
96
+ }
97
+ // Fallback 2: DOM.getBoxModel via CDP directly
98
+ if (element.backendNodeId) {
99
+ try {
100
+ await session.DOM.getDocument({ depth: 0 }); // ensure DOM is enabled/active
101
+ const box = await session.DOM.getBoxModel({ backendNodeId: element.backendNodeId });
102
+ if (box && box.model && box.model.content && box.model.content.length === 8) {
103
+ const c = box.model.content;
104
+ const x = Math.round((c[0] + c[2]) / 2);
105
+ const y = Math.round((c[1] + c[5]) / 2);
106
+ if (x !== 0 || y !== 0)
107
+ return { x, y, selector };
108
+ }
109
+ }
110
+ catch { /* ignore box model errors */ }
111
+ }
112
+ // Fallback 3: Map coords (AX or live index from map time).
113
+ // Note: elements genuinely positioned at (0,0) are valid targets and accepted.
114
+ if (element.x !== undefined && element.y !== undefined) {
115
+ return { x: element.x, y: element.y, selector };
116
+ }
117
+ return null;
118
+ }
119
+ export async function browserPointer(session, args, config) {
120
+ let { action, ref, selector, label, x, y, end_ref, end_x, end_y, waitForChild, holdMs = 300 } = args;
121
+ if (selector && !ref) {
122
+ ref = selector;
123
+ }
124
+ else if (args.target && !ref) {
125
+ ref = args.target;
126
+ }
127
+ const startMs = Date.now();
128
+ const { Page, Input, Runtime } = session;
129
+ const timeoutMs = action === "drag_and_drop" ? 25000 : 15000;
130
+ const { frameTree } = await withTimeout(Page.getFrameTree(), 5000, "browser_pointer getFrameTree");
131
+ const initialUrl = frameTree.frame.url;
132
+ let targetId = "default";
133
+ const active = sessionRegistry.getActive(getConfigAgentId(config));
134
+ if (active)
135
+ targetId = active.targetId;
136
+ // Resolve label to a ref ID before coordinate resolution,
137
+ // mirroring the logic in browser_click.
138
+ if (!ref && label && !x && !y) {
139
+ let map = getCachedActionMap(targetId, initialUrl);
140
+ if (!map) {
141
+ map = await withTimeout(buildActionMap(session, { lite: true }, targetId), 6000, "buildActionMap label");
142
+ cacheActionMap(targetId, map);
143
+ }
144
+ const matches = findAllMatches(map.elements, label);
145
+ if (matches.length > 0) {
146
+ ref = matches[0].id;
147
+ }
148
+ else {
149
+ const ariaEl = await resolveByAria(session, label).catch(() => null);
150
+ if (ariaEl) {
151
+ ref = ariaEl.id;
152
+ }
153
+ }
154
+ if (!ref) {
155
+ throw new Error(`browser_pointer: Could not find element matching label: '${label}'`);
156
+ }
157
+ }
158
+ // Resolve start coordinates
159
+ let startX = x, startY = y, startSelector = "";
160
+ if (ref) {
161
+ let coords = await withTimeout(resolveRefToCoords(session, targetId, ref, initialUrl), 4000, "resolveRefToCoords");
162
+ if (!coords) {
163
+ await buildActionMap(session, { lite: true }, targetId).then(m => cacheActionMap(targetId, m));
164
+ coords = await withTimeout(resolveRefToCoords(session, targetId, ref, initialUrl), 4000, "resolveRefToCoords retry");
165
+ }
166
+ if (!coords) {
167
+ throw new Error(`Could not resolve coordinates for ref: ${ref}`);
168
+ }
169
+ startX = coords.x;
170
+ startY = coords.y;
171
+ startSelector = coords.selector;
172
+ }
173
+ if (startX === undefined || startY === undefined) {
174
+ throw new Error(`browser_pointer requires either ref, label, or explicit (x, y) coordinates.`);
175
+ }
176
+ // Record DOM state before interaction
177
+ let map = getCachedActionMap(targetId, initialUrl);
178
+ if (!map) {
179
+ map = await withTimeout(buildActionMap(session, { lite: true }, targetId), 6000, "buildActionMap");
180
+ cacheActionMap(targetId, map);
181
+ }
182
+ storeSnapshot(targetId, map.url, map.elements);
183
+ // Move to target using human-like motion
184
+ await withTimeout((async () => {
185
+ if (action === "hover") {
186
+ await Promise.race([
187
+ ghostMove(session, startX, startY, config),
188
+ new Promise(r => setTimeout(r, 3000)).then(() => ghostMove(session, startX, startY, config, undefined, "instant"))
189
+ ]);
190
+ }
191
+ else {
192
+ await ghostMove(session, startX, startY, config);
193
+ }
194
+ })(), 10000, "ghostMove");
195
+ let preBox = null;
196
+ let isCanvasTarget = false;
197
+ if (action === "hover") {
198
+ // Dispatch both mouseenter AND mouseover (some sites only listen to one)
199
+ if (startSelector) {
200
+ await session.Runtime.evaluate({
201
+ expression: `(function(sel) {
202
+ const el = document.querySelector(sel) || (window.__Weavetab_shadowQuery ? window.__Weavetab_shadowQuery(sel) : null);
203
+ if (!el) return;
204
+ el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: false, cancelable: true }));
205
+ el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true }));
206
+ })(${JSON.stringify(startSelector)})`,
207
+ returnByValue: true
208
+ }).catch(() => { });
209
+ }
210
+ await new Promise(r => setTimeout(r, holdMs));
211
+ if (waitForChild) {
212
+ await browserWait(session, { for: "element", value: waitForChild, timeout: 5000 }, config);
213
+ }
214
+ }
215
+ else if (action === "click") {
216
+ await ghostClick(session, startX, startY);
217
+ await Input.dispatchMouseEvent({ type: "mouseReleased", x: startX, y: startY, button: "left", clickCount: 1 });
218
+ }
219
+ else if (action === "double_click") {
220
+ await ghostClick(session, startX, startY);
221
+ await Input.dispatchMouseEvent({ type: "mouseReleased", x: startX, y: startY, button: "left", clickCount: 1 });
222
+ await new Promise(r => setTimeout(r, 100));
223
+ await ghostClick(session, startX, startY, { clickCount: 2 });
224
+ await Input.dispatchMouseEvent({ type: "mouseReleased", x: startX, y: startY, button: "left", clickCount: 2 });
225
+ }
226
+ else if (action === "drag_and_drop") {
227
+ let destX = end_x, destY = end_y;
228
+ if (end_ref) {
229
+ const destCoords = await withTimeout(resolveRefToCoords(session, targetId, end_ref), 4000, "resolveRefToCoords end_ref").catch(() => null);
230
+ if (!destCoords) {
231
+ throw new Error(`Could not resolve coordinates for end_ref: ${end_ref}`);
232
+ }
233
+ destX = destCoords.x;
234
+ destY = destCoords.y;
235
+ }
236
+ if (destX === undefined || destY === undefined) {
237
+ throw new Error("drag_and_drop requires either end_ref or end_x/end_y");
238
+ }
239
+ // ── Pre-drag: snapshot the element bounding box for verification ──────
240
+ const preDragBBox = await session.Runtime.evaluate({
241
+ expression: `(function(sx, sy, sel) {
242
+ var el = sel ? document.querySelector(sel) : document.elementFromPoint(sx, sy);
243
+ if (!el) return null;
244
+ var r = el.getBoundingClientRect();
245
+ return { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), tag: el.tagName };
246
+ })(${startX}, ${startY}, ${startSelector ? JSON.stringify(startSelector) : 'null'})`,
247
+ returnByValue: true,
248
+ }).catch(() => null);
249
+ preBox = preDragBBox?.result?.value;
250
+ isCanvasTarget = preBox?.tag === 'CANVAS';
251
+ // ── Strategy 1: CDP-native Input.dispatchDragEvent ────────────────────
252
+ // Most reliable path for HTML5 DnD — Chrome handles the full
253
+ // dragstart→dragover→drop→dragend lifecycle internally.
254
+ // Only used for non-canvas targets (canvas apps don't use HTML5 DnD).
255
+ let cdpDragWorked = false;
256
+ if (!isCanvasTarget) {
257
+ try {
258
+ const dragData = {
259
+ items: [],
260
+ dragOperationsMask: 35, // copy(1) | link(2) | move(32)
261
+ };
262
+ await Input.dispatchDragEvent({
263
+ type: "dragEnter",
264
+ x: startX,
265
+ y: startY,
266
+ data: dragData,
267
+ });
268
+ await new Promise(r => setTimeout(r, 50));
269
+ const cdpDragSteps = 10;
270
+ for (let i = 1; i <= cdpDragSteps; i++) {
271
+ const t = i / cdpDragSteps;
272
+ const mx = Math.round(startX + (destX - startX) * t);
273
+ const my = Math.round(startY + (destY - startY) * t);
274
+ await Input.dispatchDragEvent({
275
+ type: "dragOver",
276
+ x: mx,
277
+ y: my,
278
+ data: dragData,
279
+ });
280
+ await new Promise(r => setTimeout(r, 30));
281
+ }
282
+ await Input.dispatchDragEvent({
283
+ type: "drop",
284
+ x: destX,
285
+ y: destY,
286
+ data: dragData,
287
+ });
288
+ await new Promise(r => setTimeout(r, 50));
289
+ await Input.dispatchDragEvent({
290
+ type: "dragEnd",
291
+ x: destX,
292
+ y: destY,
293
+ data: dragData,
294
+ });
295
+ cdpDragWorked = true;
296
+ }
297
+ catch {
298
+ // Input.dispatchDragEvent may not be supported; fall through.
299
+ }
300
+ }
301
+ // ── Strategy 2: Full HTML5 DnD event sequence via JS injection ────────
302
+ // For frameworks (React DnD, SortableJS, native drop zones).
303
+ // Skipped for canvas targets — they don't use HTML5 DnD.
304
+ if (!cdpDragWorked && !isCanvasTarget) {
305
+ await session.Runtime.evaluate({
306
+ expression: `(function(sx, sy, dx, dy, srcSel) {
307
+ var src = srcSel ? document.querySelector(srcSel) : document.elementFromPoint(sx, sy);
308
+ var dest = document.elementFromPoint(dx, dy);
309
+ if (!src) return;
310
+
311
+ var dt;
312
+ try { dt = new DataTransfer(); } catch(e) {
313
+ dt = { items: [], types: [], files: [],
314
+ setData: function(k,v){ this.types.push(k); this._data = this._data||{}; this._data[k]=v; },
315
+ getData: function(k){ return (this._data||{})[k]||''; },
316
+ clearData: function(){}, dropEffect:'move', effectAllowed:'all' };
317
+ }
318
+ dt.effectAllowed = 'all';
319
+
320
+ function fire(el, type, related) {
321
+ if (!el) return true;
322
+ var evt = new DragEvent(type, {
323
+ bubbles: true, cancelable: true, view: window,
324
+ clientX: type.startsWith('drag') && (type==='dragstart'||type==='drag'||type==='dragend') ? sx : dx,
325
+ clientY: type.startsWith('drag') && (type==='dragstart'||type==='drag'||type==='dragend') ? sy : dy,
326
+ dataTransfer: dt,
327
+ relatedTarget: related || null
328
+ });
329
+ return el.dispatchEvent(evt);
330
+ }
331
+
332
+ fire(src, 'dragstart', null);
333
+ fire(dest, 'dragenter', src);
334
+ for (var i = 0; i < 3; i++) { fire(dest, 'dragover', src); }
335
+ fire(dest, 'drop', src);
336
+ fire(src, 'dragend', dest);
337
+ })(${startX}, ${startY}, ${destX}, ${destY}, ${startSelector ? JSON.stringify(startSelector) : 'null'})`,
338
+ returnByValue: false,
339
+ }).catch(() => { });
340
+ }
341
+ // ── Strategy 3: CDP pointer/mouse sequence (primary for canvas apps) ──
342
+ // For canvas apps (Excalidraw, Figma, tldraw, etc.) that use
343
+ // PointerEvents + setPointerCapture. The REAL CDP mousePressed creates
344
+ // the active pointer that the app's pointerdown handler can capture.
345
+ // We then augment each CDP mouseMoved with a synthetic PointerEvent
346
+ // dispatched on the CAPTURED element so pointermove reaches it even
347
+ // though the cursor has left its bounds.
348
+ //
349
+ // The old code dispatched synthetic pointerdown BEFORE CDP mousePressed,
350
+ // which meant setPointerCapture(pointerId) threw NotFoundError because no
351
+ // real pointer existed yet. The CDP press comes first, establishing the
352
+ // pointer, and synthetic events only augment the move/release phases.
353
+ // Step 1: CDP mousePressed — creates the real active pointer
354
+ await Input.dispatchMouseEvent({ type: "mousePressed", x: startX, y: startY, button: "left", clickCount: 1, pointerType: "mouse" });
355
+ await new Promise(r => setTimeout(r, 120));
356
+ // Step 2: Interpolated drag — CDP mouseMoved + synthetic PointerEvent
357
+ const dragSteps = 16;
358
+ for (let i = 1; i <= dragSteps; i++) {
359
+ const t = i / dragSteps;
360
+ // Ease-in-out for natural motion
361
+ const eased = t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
362
+ const mx = Math.round(startX + (destX - startX) * eased);
363
+ const my = Math.round(startY + (destY - startY) * eased);
364
+ // CDP mouse move (generates a real pointer event internally)
365
+ await Input.dispatchMouseEvent({ type: "mouseMoved", x: mx, y: my, button: "left", clickCount: 1, pointerType: "mouse" });
366
+ // Augment with synthetic pointermove on the source element.
367
+ // If the app called setPointerCapture, the browser routes real
368
+ // pointer events to the capturing element. But some apps also
369
+ // listen on document/window, so we fire on the original element
370
+ // for belt-and-suspenders coverage.
371
+ if (isCanvasTarget) {
372
+ await session.Runtime.evaluate({
373
+ expression: `(function(sx, sy, mx, my) {
374
+ var el = document.elementFromPoint(sx, sy) || document.elementFromPoint(mx, my);
375
+ if (!el) return;
376
+ el.dispatchEvent(new PointerEvent('pointermove', {
377
+ bubbles: true, cancelable: true, view: window,
378
+ pointerId: 1, pointerType: 'mouse', isPrimary: true,
379
+ width: 1, height: 1, pressure: 0.5,
380
+ button: -1, buttons: 1,
381
+ clientX: ${mx}, clientY: ${my}
382
+ }));
383
+ })(${startX}, ${startY}, ${mx}, ${my})`,
384
+ returnByValue: false,
385
+ }).catch(() => { });
386
+ }
387
+ await new Promise(r => setTimeout(r, 18));
388
+ }
389
+ // Small settle before release
390
+ await new Promise(r => setTimeout(r, 80));
391
+ // Step 3: Release at destination
392
+ await Input.dispatchMouseEvent({ type: "mouseReleased", x: destX, y: destY, button: "left", clickCount: 1, pointerType: "mouse" });
393
+ // Augment with synthetic pointerup for canvas apps
394
+ if (isCanvasTarget) {
395
+ await session.Runtime.evaluate({
396
+ expression: `(function(sx, sy, dx, dy) {
397
+ var el = document.elementFromPoint(sx, sy) || document.elementFromPoint(dx, dy);
398
+ if (!el) return;
399
+ el.dispatchEvent(new PointerEvent('pointerup', {
400
+ bubbles: true, cancelable: true, view: window,
401
+ pointerId: 1, pointerType: 'mouse', isPrimary: true,
402
+ width: 1, height: 1, pressure: 0, button: 0, buttons: 0,
403
+ clientX: ${destX}, clientY: ${destY}
404
+ }));
405
+ })(${startX}, ${startY}, ${destX}, ${destY})`,
406
+ returnByValue: false,
407
+ }).catch(() => { });
408
+ }
409
+ // Clean up stuck drag CSS classes
410
+ await session.Runtime.evaluate({
411
+ expression: `(function() {
412
+ const classes = ['dragging', 'drag-active', 'ui-draggable-dragging', 'is-dragging', 'being-dragged'];
413
+ const attrs = ['data-dragging'];
414
+ classes.forEach(cls => {
415
+ document.querySelectorAll('.' + cls).forEach(el => el.classList.remove(cls));
416
+ });
417
+ attrs.forEach(attr => {
418
+ document.querySelectorAll('[' + attr + ']').forEach(el => el.removeAttribute(attr));
419
+ });
420
+ })()`,
421
+ returnByValue: false,
422
+ }).catch(() => { });
423
+ }
424
+ // Settle
425
+ await waitForDOMStable(session, 50, 1500).catch(() => { });
426
+ const { frameTree: postFrameTree } = await withTimeout(Page.getFrameTree(), 5000, "post-action getFrameTree");
427
+ const finalUrl = postFrameTree.frame.url;
428
+ // Calculate DOM changes (timeout-safe)
429
+ const newMap = await buildActionMap(session, { lite: true }, targetId).catch(() => null) || map;
430
+ cacheActionMap(targetId, newMap);
431
+ const diff = computeDelta(targetId, newMap.url, newMap.elements);
432
+ const changesCount = diff ? diff.added.length + diff.removed.length + diff.changed.length : 0;
433
+ const domChanged = changesCount > 0 || finalUrl !== initialUrl;
434
+ // ── Post-drag verification for drag_and_drop ───────────────────────────
435
+ // Check whether the element actually moved by comparing its bounding box
436
+ // before and after. For canvas apps we check if the source element's box
437
+ // shifted by at least 60% of the drag vector. For DOM-based drops we
438
+ // probe for common drop-indicator patterns.
439
+ let dragVerified = undefined;
440
+ if (action === "drag_and_drop" && startX !== undefined && startY !== undefined && end_x !== undefined && end_y !== undefined) {
441
+ await new Promise(r => setTimeout(r, 200)); // let animations settle
442
+ // Check 1: bounding-box delta on the originally-clicked element
443
+ const postDragBBox = await session.Runtime.evaluate({
444
+ expression: `(function(dx, dy, sel) {
445
+ var el = sel ? document.querySelector(sel) : document.elementFromPoint(dx, dy);
446
+ if (!el) return null;
447
+ var r = el.getBoundingClientRect();
448
+ return { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height) };
449
+ })(${end_x}, ${end_y}, ${startSelector ? JSON.stringify(startSelector) : 'null'})`,
450
+ returnByValue: true,
451
+ }).catch(() => null);
452
+ const postBox = postDragBBox?.result?.value;
453
+ // Check 2: probe for drop-zone indicators (text change, class change)
454
+ const dropProbe = await session.Runtime.evaluate({
455
+ expression: `(function(dx, dy) {
456
+ var el = document.elementFromPoint(dx, dy);
457
+ if (!el) return { found: false };
458
+ var text = (el.textContent || '').trim().substring(0, 100);
459
+ var classes = el.className || '';
460
+ return { found: true, text: text, classes: typeof classes === 'string' ? classes : '' };
461
+ })(${end_x}, ${end_y})`,
462
+ returnByValue: true,
463
+ }).catch(() => null);
464
+ const probe = dropProbe?.result?.value;
465
+ if (postBox && startSelector) {
466
+ // DOM element with a selector — check if it moved
467
+ const curBBoxRes = await session.Runtime.evaluate({
468
+ expression: `(function(sel) {
469
+ var el = document.querySelector(sel);
470
+ if (!el) return null;
471
+ var r = el.getBoundingClientRect();
472
+ return { x: Math.round(r.left), y: Math.round(r.top) };
473
+ })(${JSON.stringify(startSelector)})`,
474
+ returnByValue: true,
475
+ }).catch(() => null);
476
+ const curBox = curBBoxRes?.result?.value;
477
+ if (curBox && preBox) {
478
+ const movedX = Math.abs(curBox.x - preBox.x);
479
+ const movedY = Math.abs(curBox.y - preBox.y);
480
+ const dragDist = Math.sqrt(Math.pow((end_x || 0) - startX, 2) + Math.pow((end_y || 0) - startY, 2));
481
+ dragVerified = (movedX + movedY) > Math.max(5, dragDist * 0.3);
482
+ }
483
+ }
484
+ else if (domChanged) {
485
+ // DOM changed after drag — likely a sortable/drop zone interaction
486
+ dragVerified = true;
487
+ }
488
+ else if (probe?.found) {
489
+ // Check for common drop indicator patterns
490
+ const dropIndicators = ['dropped', 'drop-success', 'drag-complete', 'moved'];
491
+ const probeText = (probe.text || '').toLowerCase();
492
+ const probeClasses = (probe.classes || '').toLowerCase();
493
+ dragVerified = dropIndicators.some(ind => probeText.includes(ind) || probeClasses.includes(ind));
494
+ }
495
+ // For canvas targets where we can't easily verify, don't claim failure —
496
+ // leave dragVerified as undefined (unknown) rather than false
497
+ if (dragVerified === undefined && !domChanged) {
498
+ dragVerified = false;
499
+ }
500
+ }
501
+ const executionMs = Date.now() - startMs;
502
+ logAction("browser_pointer", action, `ref:${ref || 'none'} (${startX}, ${startY}) (exec: ${executionMs}ms)${dragVerified !== undefined ? ` verified:${dragVerified}` : ''}`);
503
+ recordAction({
504
+ timestamp: new Date().toISOString(),
505
+ tool: "browser_pointer",
506
+ input: args,
507
+ result: { success: true, drag_verified: dragVerified },
508
+ });
509
+ return {
510
+ success: true,
511
+ action,
512
+ x: startX,
513
+ y: startY,
514
+ end_x,
515
+ end_y,
516
+ execution_ms: executionMs,
517
+ tab_id: targetId,
518
+ url: finalUrl,
519
+ dom_changed: domChanged,
520
+ ...(dragVerified !== undefined ? { drag_verified: dragVerified } : {}),
521
+ };
522
+ }
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Weavetab — tools/read.ts
3
+ * browser_map: ultra-lite page reader with delta mode and visual fallback.
4
+ * Weavetab by fy2ne
5
+ */
6
+ import CDP from "chrome-remote-interface";
7
+ import type { Config } from "../config/loader.js";
8
+ export interface ReadOptions {
9
+ lite?: boolean;
10
+ full?: boolean;
11
+ scope?: string;
12
+ query?: string;
13
+ limit?: number;
14
+ delta?: boolean;
15
+ intent?: string;
16
+ ax_tree?: boolean;
17
+ /** Debug: draws colored element borders via extension. The screen glow (blue edge fade) is the standard visual indicator, shown automatically during reading states. */
18
+ show_overlays?: boolean;
19
+ /** Force deep shadow DOM traversal (crosses all shadow boundaries to find elements). Auto-enabled when flat query returns sparse results. */
20
+ deep?: boolean;
21
+ /**
22
+ * Run one-shot semantic DOM pruning before returning elements.
23
+ * Drops hidden elements with no interactive descendants and no known toggle-trigger.
24
+ * Replaces SVG icons with [SVG Icon: label] tokens.
25
+ * NOTE: distinct from delta — delta diffs across time, prune compresses the current snapshot once.
26
+ */
27
+ prune?: boolean;
28
+ }
29
+ export declare function browserMap(session: CDP.Client, options: ReadOptions | undefined, targetId: string | undefined, config: Config): Promise<string>;
30
+ export { browserMap as weaveRead };