@weavetab/mcp 2.5.0-beta.5 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +49 -49
- package/ARCHITECTURE.md +590 -590
- package/README.md +216 -100
- package/SECURITY.md +207 -207
- package/TROUBLESHOOTING.md +373 -365
- package/dist/anticaptcha/detector.js +395 -395
- package/dist/anticaptcha/solver.js +130 -130
- package/dist/cdp/bridge.js +73 -73
- package/dist/cdp/connector.js +46 -22
- package/dist/cdp/extension.js +34 -34
- package/dist/cdp/helpers.js +174 -174
- package/dist/cdp/pruner.js +195 -195
- package/dist/cdp/ref_cache.js +66 -66
- package/dist/cdp/security.js +10 -10
- package/dist/cdp/verify.js +57 -57
- package/dist/cdp/walker.js +563 -525
- package/dist/cli/plugin.js +26 -3
- package/dist/config/loader.d.ts +0 -3
- package/dist/config/loader.js +0 -9
- package/dist/index.js +0 -4
- package/dist/intelligence/intent.js +81 -81
- package/dist/intelligence/sessionMemory.js +2 -2
- package/dist/overlay/ghost.js +22 -22
- package/dist/overlay/injector.js +89 -89
- package/dist/plugin/extension_bridge.js +54 -54
- package/dist/plugin/interface.d.ts +7 -0
- package/dist/plugin/loader.js +3 -4
- package/dist/plugin/schema.d.ts +11 -0
- package/dist/plugin/schema.js +59 -0
- package/dist/security/guard.js +18 -18
- package/dist/sensors/actionability.js +167 -167
- package/dist/sensors/guard.js +29 -29
- package/dist/sensors/network.js +15 -17
- package/dist/server.d.ts +1 -1
- package/dist/server.js +173 -144
- package/dist/state/delta.js +6 -7
- package/dist/tools/automation.js +85 -85
- package/dist/tools/canvas.js +48 -25
- package/dist/tools/checkpoint.js +94 -35
- package/dist/tools/click.js +52 -41
- package/dist/tools/clipboard.js +80 -74
- package/dist/tools/design_extract.js +441 -441
- package/dist/tools/dialog.js +13 -13
- package/dist/tools/emulation.d.ts +2 -0
- package/dist/tools/emulation.js +7 -0
- package/dist/tools/fill.js +344 -347
- package/dist/tools/find.js +78 -52
- package/dist/tools/highlight.js +67 -67
- package/dist/tools/inspect.js +54 -25
- package/dist/tools/key.js +42 -35
- package/dist/tools/navigate.d.ts +1 -1
- package/dist/tools/navigate.js +35 -35
- package/dist/tools/office/parse.js +62 -18
- package/dist/tools/pdf.d.ts +9 -0
- package/dist/tools/pdf.js +7 -4
- package/dist/tools/performance.d.ts +11 -4
- package/dist/tools/performance.js +94 -42
- package/dist/tools/performance_test.d.ts +2 -2
- package/dist/tools/performance_test.js +69 -69
- package/dist/tools/pointer.js +116 -116
- package/dist/tools/read.js +99 -67
- package/dist/tools/recording.js +9 -0
- package/dist/tools/scrape.js +146 -146
- package/dist/tools/screenshot.d.ts +2 -0
- package/dist/tools/screenshot.js +46 -4
- package/dist/tools/scroll.d.ts +3 -1
- package/dist/tools/scroll.js +13 -15
- package/dist/tools/select.d.ts +3 -0
- package/dist/tools/select.js +131 -124
- package/dist/tools/snapshot.js +8 -6
- package/dist/tools/storage.d.ts +5 -1
- package/dist/tools/storage.js +16 -0
- package/dist/tools/tabs.d.ts +7 -5
- package/dist/tools/tabs.js +79 -26
- package/dist/tools/thought.js +8 -8
- package/dist/tools/type.js +274 -274
- package/dist/tools/type_secret.js +23 -23
- package/dist/tools/upload.js +3 -3
- package/dist/tools/video_compiler.js +9 -0
- package/dist/tools/wait.js +35 -35
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/DOCUMENTATION.md +583 -585
- package/docs/PLUGINS.md +139 -121
- package/docs/TOOLS.md +3 -4
- package/docs/architecture/blind-injection.md +83 -83
- package/docs/architecture/overview.md +49 -49
- package/docs/configuration/configuration.md +39 -27
- package/docs/configuration/folders.md +62 -55
- package/docs/desktop-automation/electron-apps.md +225 -214
- package/docs/get-started/installation.md +1 -1
- package/docs/get-started/introduction.md +60 -60
- package/docs/get-started/quickstart.md +2 -2
- package/docs/get-started/setup-wizard.md +1 -1
- package/docs/intelligence/site-profiles.md +151 -151
- package/docs/tools/tools.md +582 -631
- package/docs/troubleshooting/trouble-loop.md +31 -31
- package/ext-dist/background.js +415 -415
- package/ext-dist/content.js +2527 -2527
- package/ext-dist/dashboard.html +583 -583
- package/ext-dist/hud/hud.js +3 -3
- package/ext-dist/hud/index.html +20 -20
- package/ext-dist/manifest.json +73 -73
- package/ext-dist/styles.css +223 -223
- package/mcp.json +796 -805
- package/package.json +8 -8
- package/server.json +23 -23
- package/smithery.yaml +7 -11
- package/dist/cdp/walker.test.d.ts +0 -2
- package/dist/cdp/walker.test.js +0 -172
- package/dist/intelligence/sessionMemory.test.d.ts +0 -2
- package/dist/intelligence/sessionMemory.test.js +0 -62
- package/dist/intelligence/trail.test.d.ts +0 -2
- package/dist/intelligence/trail.test.js +0 -110
- package/dist/state/cache.test.d.ts +0 -2
- package/dist/state/cache.test.js +0 -76
- package/dist/state/delta.test.d.ts +0 -2
- package/dist/state/delta.test.js +0 -279
- package/dist/state/groups.d.ts +0 -15
- package/dist/state/groups.js +0 -80
- package/dist/tools/accessibility.d.ts +0 -14
- package/dist/tools/accessibility.js +0 -32
- package/dist/tools/ask.d.ts +0 -14
- package/dist/tools/ask.js +0 -49
- package/dist/tools/click_and_wait.d.ts +0 -33
- package/dist/tools/click_and_wait.js +0 -149
- package/dist/tools/drag.d.ts +0 -15
- package/dist/tools/drag.js +0 -29
- package/dist/tools/geo.d.ts +0 -19
- package/dist/tools/geo.js +0 -29
- package/dist/tools/github/analyze.d.ts +0 -4
- package/dist/tools/github/analyze.js +0 -113
- package/dist/tools/github/cache.d.ts +0 -21
- package/dist/tools/github/cache.js +0 -28
- package/dist/tools/github/index.d.ts +0 -19
- package/dist/tools/github/index.js +0 -40
- package/dist/tools/github/issues.d.ts +0 -9
- package/dist/tools/github/issues.js +0 -88
- package/dist/tools/github/pr.d.ts +0 -5
- package/dist/tools/github/pr.js +0 -88
- package/dist/tools/github/rateLimiter.d.ts +0 -4
- package/dist/tools/github/rateLimiter.js +0 -46
- package/dist/tools/github/read.d.ts +0 -34
- package/dist/tools/github/read.js +0 -74
- package/dist/tools/github/repoContext.d.ts +0 -20
- package/dist/tools/github/repoContext.js +0 -129
- package/dist/tools/hover.d.ts +0 -19
- package/dist/tools/hover.js +0 -57
- package/dist/tools/integration.test.d.ts +0 -2
- package/dist/tools/integration.test.js +0 -96
- package/dist/tools/macro_compiler.d.ts +0 -7
- package/dist/tools/macro_compiler.js +0 -82
- package/dist/tools/mouse.d.ts +0 -21
- package/dist/tools/mouse.js +0 -62
- package/dist/tools/response.d.ts +0 -129
- package/dist/tools/response.js +0 -5
- package/dist/tools/wait_for.d.ts +0 -23
- package/dist/tools/wait_for.js +0 -30
- package/dist/tools/windows.d.ts +0 -22
- package/dist/tools/windows.js +0 -57
package/dist/tools/pointer.js
CHANGED
|
@@ -46,12 +46,12 @@ async function resolveRefToCoords(session, targetId, ref, currentUrl) {
|
|
|
46
46
|
return null;
|
|
47
47
|
// Scroll element into view before resolving coordinates
|
|
48
48
|
await session.Runtime.evaluate({
|
|
49
|
-
expression: `(function(sel) {
|
|
50
|
-
const el = document.querySelector(sel);
|
|
51
|
-
if (el) {
|
|
52
|
-
if (el.scrollIntoViewIfNeeded) el.scrollIntoViewIfNeeded();
|
|
53
|
-
else el.scrollIntoView({behavior:'instant',block:'center'});
|
|
54
|
-
}
|
|
49
|
+
expression: `(function(sel) {
|
|
50
|
+
const el = document.querySelector(sel);
|
|
51
|
+
if (el) {
|
|
52
|
+
if (el.scrollIntoViewIfNeeded) el.scrollIntoViewIfNeeded();
|
|
53
|
+
else el.scrollIntoView({behavior:'instant',block:'center'});
|
|
54
|
+
}
|
|
55
55
|
})(${JSON.stringify(selector)})`,
|
|
56
56
|
returnByValue: true
|
|
57
57
|
}).catch(() => { });
|
|
@@ -68,25 +68,25 @@ async function resolveRefToCoords(session, targetId, ref, currentUrl) {
|
|
|
68
68
|
}
|
|
69
69
|
// Fallback 1: use element bounding rect directly via CDP evaluate
|
|
70
70
|
const bboxRes = await session.Runtime.evaluate({
|
|
71
|
-
expression: `(function(sel) {
|
|
72
|
-
const el = document.querySelector(sel);
|
|
73
|
-
if (!el) return null;
|
|
74
|
-
// Primary: getBoundingClientRect
|
|
75
|
-
const r = el.getBoundingClientRect();
|
|
76
|
-
if (r.width > 0 || r.height > 0) {
|
|
77
|
-
return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) };
|
|
78
|
-
}
|
|
79
|
-
// Fallback for inline text elements (e.g. Wikipedia lang links).
|
|
80
|
-
// Inline elements can have zero-size bounding rects but non-zero client rects
|
|
81
|
-
// for their first text-box fragment.
|
|
82
|
-
const rects = el.getClientRects();
|
|
83
|
-
if (rects && rects.length > 0) {
|
|
84
|
-
const fr = rects[0];
|
|
85
|
-
if (fr.width > 0 || fr.height > 0) {
|
|
86
|
-
return { x: Math.round(fr.left + fr.width / 2), y: Math.round(fr.top + fr.height / 2) };
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return null;
|
|
71
|
+
expression: `(function(sel) {
|
|
72
|
+
const el = document.querySelector(sel);
|
|
73
|
+
if (!el) return null;
|
|
74
|
+
// Primary: getBoundingClientRect
|
|
75
|
+
const r = el.getBoundingClientRect();
|
|
76
|
+
if (r.width > 0 || r.height > 0) {
|
|
77
|
+
return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) };
|
|
78
|
+
}
|
|
79
|
+
// Fallback for inline text elements (e.g. Wikipedia lang links).
|
|
80
|
+
// Inline elements can have zero-size bounding rects but non-zero client rects
|
|
81
|
+
// for their first text-box fragment.
|
|
82
|
+
const rects = el.getClientRects();
|
|
83
|
+
if (rects && rects.length > 0) {
|
|
84
|
+
const fr = rects[0];
|
|
85
|
+
if (fr.width > 0 || fr.height > 0) {
|
|
86
|
+
return { x: Math.round(fr.left + fr.width / 2), y: Math.round(fr.top + fr.height / 2) };
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
90
|
})(${JSON.stringify(selector)})`,
|
|
91
91
|
returnByValue: true
|
|
92
92
|
}).catch(() => null);
|
|
@@ -167,12 +167,12 @@ export async function browserPointer(session, args, config) {
|
|
|
167
167
|
// Dispatch mouseenter/mouseover events for dynamic flyout/hover menus
|
|
168
168
|
if (stepCoords.selector) {
|
|
169
169
|
await session.Runtime.evaluate({
|
|
170
|
-
expression: `(function(sel, cx, cy) {
|
|
171
|
-
const el = document.querySelector(sel) || (window.__Weavetab_shadowQuery ? window.__Weavetab_shadowQuery(sel) : null);
|
|
172
|
-
if (!el) return;
|
|
173
|
-
el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: false, cancelable: true, clientX: cx, clientY: cy }));
|
|
174
|
-
el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true, clientX: cx, clientY: cy }));
|
|
175
|
-
el.dispatchEvent(new MouseEvent('mousemove', { bubbles: true, cancelable: true, clientX: cx, clientY: cy }));
|
|
170
|
+
expression: `(function(sel, cx, cy) {
|
|
171
|
+
const el = document.querySelector(sel) || (window.__Weavetab_shadowQuery ? window.__Weavetab_shadowQuery(sel) : null);
|
|
172
|
+
if (!el) return;
|
|
173
|
+
el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: false, cancelable: true, clientX: cx, clientY: cy }));
|
|
174
|
+
el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true, clientX: cx, clientY: cy }));
|
|
175
|
+
el.dispatchEvent(new MouseEvent('mousemove', { bubbles: true, cancelable: true, clientX: cx, clientY: cy }));
|
|
176
176
|
})(${JSON.stringify(stepCoords.selector)}, ${stepCoords.x}, ${stepCoords.y})`,
|
|
177
177
|
returnByValue: false
|
|
178
178
|
}).catch(() => { });
|
|
@@ -286,11 +286,11 @@ export async function browserPointer(session, args, config) {
|
|
|
286
286
|
// Dispatch both mouseenter AND mouseover (some sites only listen to one)
|
|
287
287
|
if (startSelector) {
|
|
288
288
|
await session.Runtime.evaluate({
|
|
289
|
-
expression: `(function(sel) {
|
|
290
|
-
const el = document.querySelector(sel) || (window.__Weavetab_shadowQuery ? window.__Weavetab_shadowQuery(sel) : null);
|
|
291
|
-
if (!el) return;
|
|
292
|
-
el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: false, cancelable: true }));
|
|
293
|
-
el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true }));
|
|
289
|
+
expression: `(function(sel) {
|
|
290
|
+
const el = document.querySelector(sel) || (window.__Weavetab_shadowQuery ? window.__Weavetab_shadowQuery(sel) : null);
|
|
291
|
+
if (!el) return;
|
|
292
|
+
el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: false, cancelable: true }));
|
|
293
|
+
el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true }));
|
|
294
294
|
})(${JSON.stringify(startSelector)})`,
|
|
295
295
|
returnByValue: true
|
|
296
296
|
}).catch(() => { });
|
|
@@ -326,11 +326,11 @@ export async function browserPointer(session, args, config) {
|
|
|
326
326
|
}
|
|
327
327
|
// ── Pre-drag: snapshot the element bounding box for verification ──────
|
|
328
328
|
const preDragBBox = await session.Runtime.evaluate({
|
|
329
|
-
expression: `(function(sx, sy, sel) {
|
|
330
|
-
var el = sel ? document.querySelector(sel) : document.elementFromPoint(sx, sy);
|
|
331
|
-
if (!el) return null;
|
|
332
|
-
var r = el.getBoundingClientRect();
|
|
333
|
-
return { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), tag: el.tagName };
|
|
329
|
+
expression: `(function(sx, sy, sel) {
|
|
330
|
+
var el = sel ? document.querySelector(sel) : document.elementFromPoint(sx, sy);
|
|
331
|
+
if (!el) return null;
|
|
332
|
+
var r = el.getBoundingClientRect();
|
|
333
|
+
return { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), tag: el.tagName };
|
|
334
334
|
})(${startX}, ${startY}, ${startSelector ? JSON.stringify(startSelector) : 'null'})`,
|
|
335
335
|
returnByValue: true,
|
|
336
336
|
}).catch(() => null);
|
|
@@ -391,37 +391,37 @@ export async function browserPointer(session, args, config) {
|
|
|
391
391
|
// Skipped for canvas targets — they don't use HTML5 DnD.
|
|
392
392
|
if (!cdpDragWorked && !isCanvasTarget) {
|
|
393
393
|
await session.Runtime.evaluate({
|
|
394
|
-
expression: `(function(sx, sy, dx, dy, srcSel) {
|
|
395
|
-
var src = srcSel ? document.querySelector(srcSel) : document.elementFromPoint(sx, sy);
|
|
396
|
-
var dest = document.elementFromPoint(dx, dy);
|
|
397
|
-
if (!src) return;
|
|
398
|
-
|
|
399
|
-
var dt;
|
|
400
|
-
try { dt = new DataTransfer(); } catch(e) {
|
|
401
|
-
dt = { items: [], types: [], files: [],
|
|
402
|
-
setData: function(k,v){ this.types.push(k); this._data = this._data||{}; this._data[k]=v; },
|
|
403
|
-
getData: function(k){ return (this._data||{})[k]||''; },
|
|
404
|
-
clearData: function(){}, dropEffect:'move', effectAllowed:'all' };
|
|
405
|
-
}
|
|
406
|
-
dt.effectAllowed = 'all';
|
|
407
|
-
|
|
408
|
-
function fire(el, type, related) {
|
|
409
|
-
if (!el) return true;
|
|
410
|
-
var evt = new DragEvent(type, {
|
|
411
|
-
bubbles: true, cancelable: true, view: window,
|
|
412
|
-
clientX: type.startsWith('drag') && (type==='dragstart'||type==='drag'||type==='dragend') ? sx : dx,
|
|
413
|
-
clientY: type.startsWith('drag') && (type==='dragstart'||type==='drag'||type==='dragend') ? sy : dy,
|
|
414
|
-
dataTransfer: dt,
|
|
415
|
-
relatedTarget: related || null
|
|
416
|
-
});
|
|
417
|
-
return el.dispatchEvent(evt);
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
fire(src, 'dragstart', null);
|
|
421
|
-
fire(dest, 'dragenter', src);
|
|
422
|
-
for (var i = 0; i < 3; i++) { fire(dest, 'dragover', src); }
|
|
423
|
-
fire(dest, 'drop', src);
|
|
424
|
-
fire(src, 'dragend', dest);
|
|
394
|
+
expression: `(function(sx, sy, dx, dy, srcSel) {
|
|
395
|
+
var src = srcSel ? document.querySelector(srcSel) : document.elementFromPoint(sx, sy);
|
|
396
|
+
var dest = document.elementFromPoint(dx, dy);
|
|
397
|
+
if (!src) return;
|
|
398
|
+
|
|
399
|
+
var dt;
|
|
400
|
+
try { dt = new DataTransfer(); } catch(e) {
|
|
401
|
+
dt = { items: [], types: [], files: [],
|
|
402
|
+
setData: function(k,v){ this.types.push(k); this._data = this._data||{}; this._data[k]=v; },
|
|
403
|
+
getData: function(k){ return (this._data||{})[k]||''; },
|
|
404
|
+
clearData: function(){}, dropEffect:'move', effectAllowed:'all' };
|
|
405
|
+
}
|
|
406
|
+
dt.effectAllowed = 'all';
|
|
407
|
+
|
|
408
|
+
function fire(el, type, related) {
|
|
409
|
+
if (!el) return true;
|
|
410
|
+
var evt = new DragEvent(type, {
|
|
411
|
+
bubbles: true, cancelable: true, view: window,
|
|
412
|
+
clientX: type.startsWith('drag') && (type==='dragstart'||type==='drag'||type==='dragend') ? sx : dx,
|
|
413
|
+
clientY: type.startsWith('drag') && (type==='dragstart'||type==='drag'||type==='dragend') ? sy : dy,
|
|
414
|
+
dataTransfer: dt,
|
|
415
|
+
relatedTarget: related || null
|
|
416
|
+
});
|
|
417
|
+
return el.dispatchEvent(evt);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
fire(src, 'dragstart', null);
|
|
421
|
+
fire(dest, 'dragenter', src);
|
|
422
|
+
for (var i = 0; i < 3; i++) { fire(dest, 'dragover', src); }
|
|
423
|
+
fire(dest, 'drop', src);
|
|
424
|
+
fire(src, 'dragend', dest);
|
|
425
425
|
})(${startX}, ${startY}, ${destX}, ${destY}, ${startSelector ? JSON.stringify(startSelector) : 'null'})`,
|
|
426
426
|
returnByValue: false,
|
|
427
427
|
}).catch(() => { });
|
|
@@ -458,16 +458,16 @@ export async function browserPointer(session, args, config) {
|
|
|
458
458
|
// for belt-and-suspenders coverage.
|
|
459
459
|
if (isCanvasTarget) {
|
|
460
460
|
await session.Runtime.evaluate({
|
|
461
|
-
expression: `(function(sx, sy, mx, my) {
|
|
462
|
-
var el = document.elementFromPoint(sx, sy) || document.elementFromPoint(mx, my);
|
|
463
|
-
if (!el) return;
|
|
464
|
-
el.dispatchEvent(new PointerEvent('pointermove', {
|
|
465
|
-
bubbles: true, cancelable: true, view: window,
|
|
466
|
-
pointerId: 1, pointerType: 'mouse', isPrimary: true,
|
|
467
|
-
width: 1, height: 1, pressure: 0.5,
|
|
468
|
-
button: -1, buttons: 1,
|
|
469
|
-
clientX: ${mx}, clientY: ${my}
|
|
470
|
-
}));
|
|
461
|
+
expression: `(function(sx, sy, mx, my) {
|
|
462
|
+
var el = document.elementFromPoint(sx, sy) || document.elementFromPoint(mx, my);
|
|
463
|
+
if (!el) return;
|
|
464
|
+
el.dispatchEvent(new PointerEvent('pointermove', {
|
|
465
|
+
bubbles: true, cancelable: true, view: window,
|
|
466
|
+
pointerId: 1, pointerType: 'mouse', isPrimary: true,
|
|
467
|
+
width: 1, height: 1, pressure: 0.5,
|
|
468
|
+
button: -1, buttons: 1,
|
|
469
|
+
clientX: ${mx}, clientY: ${my}
|
|
470
|
+
}));
|
|
471
471
|
})(${startX}, ${startY}, ${mx}, ${my})`,
|
|
472
472
|
returnByValue: false,
|
|
473
473
|
}).catch(() => { });
|
|
@@ -481,30 +481,30 @@ export async function browserPointer(session, args, config) {
|
|
|
481
481
|
// Augment with synthetic pointerup for canvas apps
|
|
482
482
|
if (isCanvasTarget) {
|
|
483
483
|
await session.Runtime.evaluate({
|
|
484
|
-
expression: `(function(sx, sy, dx, dy) {
|
|
485
|
-
var el = document.elementFromPoint(sx, sy) || document.elementFromPoint(dx, dy);
|
|
486
|
-
if (!el) return;
|
|
487
|
-
el.dispatchEvent(new PointerEvent('pointerup', {
|
|
488
|
-
bubbles: true, cancelable: true, view: window,
|
|
489
|
-
pointerId: 1, pointerType: 'mouse', isPrimary: true,
|
|
490
|
-
width: 1, height: 1, pressure: 0, button: 0, buttons: 0,
|
|
491
|
-
clientX: ${destX}, clientY: ${destY}
|
|
492
|
-
}));
|
|
484
|
+
expression: `(function(sx, sy, dx, dy) {
|
|
485
|
+
var el = document.elementFromPoint(sx, sy) || document.elementFromPoint(dx, dy);
|
|
486
|
+
if (!el) return;
|
|
487
|
+
el.dispatchEvent(new PointerEvent('pointerup', {
|
|
488
|
+
bubbles: true, cancelable: true, view: window,
|
|
489
|
+
pointerId: 1, pointerType: 'mouse', isPrimary: true,
|
|
490
|
+
width: 1, height: 1, pressure: 0, button: 0, buttons: 0,
|
|
491
|
+
clientX: ${destX}, clientY: ${destY}
|
|
492
|
+
}));
|
|
493
493
|
})(${startX}, ${startY}, ${destX}, ${destY})`,
|
|
494
494
|
returnByValue: false,
|
|
495
495
|
}).catch(() => { });
|
|
496
496
|
}
|
|
497
497
|
// Clean up stuck drag CSS classes
|
|
498
498
|
await session.Runtime.evaluate({
|
|
499
|
-
expression: `(function() {
|
|
500
|
-
const classes = ['dragging', 'drag-active', 'ui-draggable-dragging', 'is-dragging', 'being-dragged'];
|
|
501
|
-
const attrs = ['data-dragging'];
|
|
502
|
-
classes.forEach(cls => {
|
|
503
|
-
document.querySelectorAll('.' + cls).forEach(el => el.classList.remove(cls));
|
|
504
|
-
});
|
|
505
|
-
attrs.forEach(attr => {
|
|
506
|
-
document.querySelectorAll('[' + attr + ']').forEach(el => el.removeAttribute(attr));
|
|
507
|
-
});
|
|
499
|
+
expression: `(function() {
|
|
500
|
+
const classes = ['dragging', 'drag-active', 'ui-draggable-dragging', 'is-dragging', 'being-dragged'];
|
|
501
|
+
const attrs = ['data-dragging'];
|
|
502
|
+
classes.forEach(cls => {
|
|
503
|
+
document.querySelectorAll('.' + cls).forEach(el => el.classList.remove(cls));
|
|
504
|
+
});
|
|
505
|
+
attrs.forEach(attr => {
|
|
506
|
+
document.querySelectorAll('[' + attr + ']').forEach(el => el.removeAttribute(attr));
|
|
507
|
+
});
|
|
508
508
|
})()`,
|
|
509
509
|
returnByValue: false,
|
|
510
510
|
}).catch(() => { });
|
|
@@ -529,23 +529,23 @@ export async function browserPointer(session, args, config) {
|
|
|
529
529
|
await new Promise(r => setTimeout(r, 200)); // let animations settle
|
|
530
530
|
// Check 1: bounding-box delta on the originally-clicked element
|
|
531
531
|
const postDragBBox = await session.Runtime.evaluate({
|
|
532
|
-
expression: `(function(dx, dy, sel) {
|
|
533
|
-
var el = sel ? document.querySelector(sel) : document.elementFromPoint(dx, dy);
|
|
534
|
-
if (!el) return null;
|
|
535
|
-
var r = el.getBoundingClientRect();
|
|
536
|
-
return { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height) };
|
|
532
|
+
expression: `(function(dx, dy, sel) {
|
|
533
|
+
var el = sel ? document.querySelector(sel) : document.elementFromPoint(dx, dy);
|
|
534
|
+
if (!el) return null;
|
|
535
|
+
var r = el.getBoundingClientRect();
|
|
536
|
+
return { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height) };
|
|
537
537
|
})(${end_x}, ${end_y}, ${startSelector ? JSON.stringify(startSelector) : 'null'})`,
|
|
538
538
|
returnByValue: true,
|
|
539
539
|
}).catch(() => null);
|
|
540
540
|
const postBox = postDragBBox?.result?.value;
|
|
541
541
|
// Check 2: probe for drop-zone indicators (text change, class change)
|
|
542
542
|
const dropProbe = await session.Runtime.evaluate({
|
|
543
|
-
expression: `(function(dx, dy) {
|
|
544
|
-
var el = document.elementFromPoint(dx, dy);
|
|
545
|
-
if (!el) return { found: false };
|
|
546
|
-
var text = (el.textContent || '').trim().substring(0, 100);
|
|
547
|
-
var classes = el.className || '';
|
|
548
|
-
return { found: true, text: text, classes: typeof classes === 'string' ? classes : '' };
|
|
543
|
+
expression: `(function(dx, dy) {
|
|
544
|
+
var el = document.elementFromPoint(dx, dy);
|
|
545
|
+
if (!el) return { found: false };
|
|
546
|
+
var text = (el.textContent || '').trim().substring(0, 100);
|
|
547
|
+
var classes = el.className || '';
|
|
548
|
+
return { found: true, text: text, classes: typeof classes === 'string' ? classes : '' };
|
|
549
549
|
})(${end_x}, ${end_y})`,
|
|
550
550
|
returnByValue: true,
|
|
551
551
|
}).catch(() => null);
|
|
@@ -553,11 +553,11 @@ export async function browserPointer(session, args, config) {
|
|
|
553
553
|
if (postBox && startSelector) {
|
|
554
554
|
// DOM element with a selector — check if it moved
|
|
555
555
|
const curBBoxRes = await session.Runtime.evaluate({
|
|
556
|
-
expression: `(function(sel) {
|
|
557
|
-
var el = document.querySelector(sel);
|
|
558
|
-
if (!el) return null;
|
|
559
|
-
var r = el.getBoundingClientRect();
|
|
560
|
-
return { x: Math.round(r.left), y: Math.round(r.top) };
|
|
556
|
+
expression: `(function(sel) {
|
|
557
|
+
var el = document.querySelector(sel);
|
|
558
|
+
if (!el) return null;
|
|
559
|
+
var r = el.getBoundingClientRect();
|
|
560
|
+
return { x: Math.round(r.left), y: Math.round(r.top) };
|
|
561
561
|
})(${JSON.stringify(startSelector)})`,
|
|
562
562
|
returnByValue: true,
|
|
563
563
|
}).catch(() => null);
|
package/dist/tools/read.js
CHANGED
|
@@ -205,15 +205,15 @@ async function buildVisualGrounding(session, targetId, elements) {
|
|
|
205
205
|
}
|
|
206
206
|
// Also discover canvas elements that may not be in the DOM map
|
|
207
207
|
const { result: canvasRes } = await session.Runtime.evaluate({
|
|
208
|
-
expression: `(function(){
|
|
209
|
-
var canvases = [...document.querySelectorAll('canvas')];
|
|
210
|
-
return JSON.stringify(canvases.map(function(c){
|
|
211
|
-
var r = c.getBoundingClientRect();
|
|
212
|
-
return r.width > 0 && r.height > 0 ? {
|
|
213
|
-
x: Math.round(r.left + r.width/2), y: Math.round(r.top + r.height/2),
|
|
214
|
-
w: Math.round(r.width), h: Math.round(r.height)
|
|
215
|
-
} : null;
|
|
216
|
-
}).filter(Boolean));
|
|
208
|
+
expression: `(function(){
|
|
209
|
+
var canvases = [...document.querySelectorAll('canvas')];
|
|
210
|
+
return JSON.stringify(canvases.map(function(c){
|
|
211
|
+
var r = c.getBoundingClientRect();
|
|
212
|
+
return r.width > 0 && r.height > 0 ? {
|
|
213
|
+
x: Math.round(r.left + r.width/2), y: Math.round(r.top + r.height/2),
|
|
214
|
+
w: Math.round(r.width), h: Math.round(r.height)
|
|
215
|
+
} : null;
|
|
216
|
+
}).filter(Boolean));
|
|
217
217
|
})()`,
|
|
218
218
|
returnByValue: true,
|
|
219
219
|
}).catch(() => ({ result: { value: '[]' } }));
|
|
@@ -228,19 +228,19 @@ async function buildVisualGrounding(session, targetId, elements) {
|
|
|
228
228
|
if (domMarks.length === 0)
|
|
229
229
|
return null;
|
|
230
230
|
// Inject badges — fixed-position, shadow-safe, above everything
|
|
231
|
-
const injectExpr = `(function(marks){
|
|
232
|
-
var BADGE_KEY = '__wt_som_badges__';
|
|
233
|
-
if (window[BADGE_KEY]) window[BADGE_KEY].forEach(function(d){ d.parentNode && d.parentNode.removeChild(d); });
|
|
234
|
-
window[BADGE_KEY] = [];
|
|
235
|
-
marks.forEach(function(m){
|
|
236
|
-
var d = document.createElement('div');
|
|
237
|
-
d.setAttribute('data-wt-som', String(m.mark));
|
|
238
|
-
d.style.cssText = 'position:fixed;left:'+(m.x - 10)+'px;top:'+(m.y - 10)+'px;width:20px;height:20px;z-index:2147483647;pointer-events:none;background:#7c3aed;color:#fff;font:bold 11px/20px monospace;text-align:center;border-radius:50%;box-shadow:0 1px 4px rgba(0,0,0,.5);';
|
|
239
|
-
d.textContent = String(m.mark);
|
|
240
|
-
document.documentElement.appendChild(d);
|
|
241
|
-
window[BADGE_KEY].push(d);
|
|
242
|
-
});
|
|
243
|
-
return true;
|
|
231
|
+
const injectExpr = `(function(marks){
|
|
232
|
+
var BADGE_KEY = '__wt_som_badges__';
|
|
233
|
+
if (window[BADGE_KEY]) window[BADGE_KEY].forEach(function(d){ d.parentNode && d.parentNode.removeChild(d); });
|
|
234
|
+
window[BADGE_KEY] = [];
|
|
235
|
+
marks.forEach(function(m){
|
|
236
|
+
var d = document.createElement('div');
|
|
237
|
+
d.setAttribute('data-wt-som', String(m.mark));
|
|
238
|
+
d.style.cssText = 'position:fixed;left:'+(m.x - 10)+'px;top:'+(m.y - 10)+'px;width:20px;height:20px;z-index:2147483647;pointer-events:none;background:#7c3aed;color:#fff;font:bold 11px/20px monospace;text-align:center;border-radius:50%;box-shadow:0 1px 4px rgba(0,0,0,.5);';
|
|
239
|
+
d.textContent = String(m.mark);
|
|
240
|
+
document.documentElement.appendChild(d);
|
|
241
|
+
window[BADGE_KEY].push(d);
|
|
242
|
+
});
|
|
243
|
+
return true;
|
|
244
244
|
})(${JSON.stringify(domMarks)})`;
|
|
245
245
|
await session.Runtime.evaluate({ expression: injectExpr, returnByValue: true }).catch(() => { });
|
|
246
246
|
// Hide Weavetab overlay so badges are the only overlay in the shot
|
|
@@ -272,14 +272,14 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
272
272
|
// ── Wait for DOM and Visual Stability ──────────────────────────────────────
|
|
273
273
|
try {
|
|
274
274
|
await session.Runtime.evaluate({
|
|
275
|
-
expression: `
|
|
276
|
-
new Promise(resolve => {
|
|
277
|
-
if (document.readyState === 'complete') {
|
|
278
|
-
setTimeout(resolve, 300); // 300ms visual stability delay
|
|
279
|
-
} else {
|
|
280
|
-
window.addEventListener('load', () => setTimeout(resolve, 300));
|
|
281
|
-
}
|
|
282
|
-
})
|
|
275
|
+
expression: `
|
|
276
|
+
new Promise(resolve => {
|
|
277
|
+
if (document.readyState === 'complete') {
|
|
278
|
+
setTimeout(resolve, 300); // 300ms visual stability delay
|
|
279
|
+
} else {
|
|
280
|
+
window.addEventListener('load', () => setTimeout(resolve, 300));
|
|
281
|
+
}
|
|
282
|
+
})
|
|
283
283
|
`,
|
|
284
284
|
awaitPromise: true
|
|
285
285
|
});
|
|
@@ -324,18 +324,21 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
324
324
|
catch (e) {
|
|
325
325
|
logDebug(`[browserMap] Failed to compute DOM hash: ${e}`);
|
|
326
326
|
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
327
|
+
const hasFilterOptions = Boolean(options.scope || options.query || options.limit);
|
|
328
|
+
if (!hasFilterOptions) {
|
|
329
|
+
// Try strictly validated cache first
|
|
330
|
+
const cachedMap = getCachedActionMap(targetId, currentUrl);
|
|
331
|
+
if (cachedMap) {
|
|
332
|
+
logDebug(`[browserMap] Cache Hit (strict)! Returning cached action map with ${cachedMap.elements.length} elements.`);
|
|
333
|
+
map = cachedMap;
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
// Fallback: check raw cache with domHash
|
|
337
|
+
const rawMap = getRawCachedActionMap(targetId, currentUrl);
|
|
338
|
+
if (rawMap && rawMap.domHash === domHash) {
|
|
339
|
+
logDebug(`[browserMap] Cache Hit (domHash=${domHash})! MutationObserver invalidated but DOM content length unchanged.`);
|
|
340
|
+
map = rawMap;
|
|
341
|
+
}
|
|
339
342
|
}
|
|
340
343
|
}
|
|
341
344
|
}
|
|
@@ -403,11 +406,11 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
403
406
|
map.domHash = result.value ?? 0;
|
|
404
407
|
}
|
|
405
408
|
catch (e) { }
|
|
406
|
-
// Never write an AX-tree result into the primary DOM map cache.
|
|
409
|
+
// Never write an AX-tree result or a filtered map into the primary DOM map cache.
|
|
407
410
|
// AX tree uses sequential w:1…w:N IDs; DOM walker uses stable content-addressable hashes.
|
|
408
|
-
// Caching
|
|
409
|
-
|
|
410
|
-
if (!usedAXTree) {
|
|
411
|
+
// Caching a filtered or AX result would make subsequent plain browser_map() calls return incomplete refs.
|
|
412
|
+
const hasFilterOptions = Boolean(options.scope || options.query || options.limit);
|
|
413
|
+
if (!usedAXTree && !hasFilterOptions) {
|
|
411
414
|
cacheActionMap(targetId, map);
|
|
412
415
|
}
|
|
413
416
|
logAction("browser_map", map.url, `${map.elements.length} elements`);
|
|
@@ -461,21 +464,22 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
461
464
|
const activeTabId = sessionRegistry.getActive(getConfigAgentId(config))?.targetId ?? targetId;
|
|
462
465
|
// ── Visual fallback when AX tree is empty ──────────────────────────────────
|
|
463
466
|
let visualFallback = null;
|
|
464
|
-
|
|
467
|
+
const hasFilter = Boolean(options.query || options.scope);
|
|
468
|
+
if (map.elements.length === 0 && !hasFilter) {
|
|
465
469
|
try {
|
|
466
470
|
// Canvas detection — check before full screenshot analysis
|
|
467
471
|
const { result: canvasCheck } = await session.Runtime.evaluate({
|
|
468
|
-
expression: `(function() {
|
|
469
|
-
const canvases = [...document.querySelectorAll('canvas')];
|
|
470
|
-
return JSON.stringify(canvases.map(c => {
|
|
471
|
-
const rect = c.getBoundingClientRect();
|
|
472
|
-
return {
|
|
473
|
-
id: c.id || '', width: Math.round(rect.width), height: Math.round(rect.height),
|
|
474
|
-
x: Math.round(rect.left + rect.width / 2),
|
|
475
|
-
y: Math.round(rect.top + rect.height / 2),
|
|
476
|
-
hasWebGL: !!(c.getContext('webgl') || c.getContext('webgl2'))
|
|
477
|
-
};
|
|
478
|
-
}));
|
|
472
|
+
expression: `(function() {
|
|
473
|
+
const canvases = [...document.querySelectorAll('canvas')];
|
|
474
|
+
return JSON.stringify(canvases.map(c => {
|
|
475
|
+
const rect = c.getBoundingClientRect();
|
|
476
|
+
return {
|
|
477
|
+
id: c.id || '', width: Math.round(rect.width), height: Math.round(rect.height),
|
|
478
|
+
x: Math.round(rect.left + rect.width / 2),
|
|
479
|
+
y: Math.round(rect.top + rect.height / 2),
|
|
480
|
+
hasWebGL: !!(c.getContext('webgl') || c.getContext('webgl2'))
|
|
481
|
+
};
|
|
482
|
+
}));
|
|
479
483
|
})()`,
|
|
480
484
|
returnByValue: true,
|
|
481
485
|
}).catch(() => ({ result: { value: '[]' } }));
|
|
@@ -539,22 +543,41 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
539
543
|
lines.push("stable — no DOM changes since last map (use existing refs)");
|
|
540
544
|
}
|
|
541
545
|
else {
|
|
542
|
-
|
|
546
|
+
const maxLines = 40;
|
|
547
|
+
let printed = 0;
|
|
548
|
+
for (const r of richDiff.added) {
|
|
549
|
+
if (printed >= maxLines)
|
|
550
|
+
break;
|
|
543
551
|
lines.push(`+ [${r.id}] ${r.role} "${r.label}"`);
|
|
552
|
+
printed++;
|
|
553
|
+
}
|
|
544
554
|
for (const r of richDiff.changed) {
|
|
555
|
+
if (printed >= maxLines)
|
|
556
|
+
break;
|
|
545
557
|
const changes = Object.keys(r.changes);
|
|
546
558
|
const changeHints = changes.map(c => `${c}: "${r.changes[c].from}" → "${r.changes[c].to}"`).join(', ');
|
|
547
559
|
const actionable = r.actionable ? ` ⚡${r.recommended_action ? ` ${r.recommended_action}` : ''}` : '';
|
|
548
560
|
lines.push(`~ [${r.ref}] ${r.role} "${r.label}"${changeHints ? ` (${changeHints})` : ''}${actionable}`);
|
|
561
|
+
printed++;
|
|
562
|
+
}
|
|
563
|
+
for (const id of richDiff.removed) {
|
|
564
|
+
if (printed >= maxLines)
|
|
565
|
+
break;
|
|
566
|
+
lines.push(`- [${id}] (removed)`);
|
|
567
|
+
printed++;
|
|
549
568
|
}
|
|
550
|
-
for (const id of richDiff.removed)
|
|
551
|
-
lines.push(`- ${id}`);
|
|
552
569
|
if (richDiff.iframes) {
|
|
553
570
|
for (const f of richDiff.iframes) {
|
|
571
|
+
if (printed >= maxLines)
|
|
572
|
+
break;
|
|
554
573
|
const icon = f.status === "added" ? "+" : f.status === "removed" ? "-" : "~";
|
|
555
574
|
lines.push(`${icon} [iframe] ${f.frameUrl} (${f.status}, ${f.elementCount} elements)`);
|
|
575
|
+
printed++;
|
|
556
576
|
}
|
|
557
577
|
}
|
|
578
|
+
if (totalChanges > printed) {
|
|
579
|
+
lines.push(`... (+${totalChanges - printed} more changes omitted)`);
|
|
580
|
+
}
|
|
558
581
|
}
|
|
559
582
|
if (visualFallback)
|
|
560
583
|
lines.push(visualFallback);
|
|
@@ -601,8 +624,17 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
601
624
|
const lines = [];
|
|
602
625
|
if (hint)
|
|
603
626
|
lines.push(`> ${hint}`);
|
|
604
|
-
// Cure optional-login blindness by explicitly pointing out dismiss buttons
|
|
605
|
-
const
|
|
627
|
+
// Cure optional-login blindness by explicitly pointing out dismiss buttons on true modals
|
|
628
|
+
const isTitlebarControl = (el) => {
|
|
629
|
+
const sel = (el.selector || "").toLowerCase();
|
|
630
|
+
const lbl = (el.label || "").toLowerCase();
|
|
631
|
+
const isTopCorner = (el.y !== undefined && el.y < 45 && el.x !== undefined && el.x > 800);
|
|
632
|
+
return isTopCorner || sel.includes("win-close") || sel.includes("titlebar") || sel.includes("window-controls") || lbl === "close window";
|
|
633
|
+
};
|
|
634
|
+
const hasModalContext = map.elements.some(el => el.isModal || el.role === "dialog" || el.role === "alertdialog");
|
|
635
|
+
const dismissButton = map.elements.find(el => (hasModalContext || el.isModal) &&
|
|
636
|
+
!isTitlebarControl(el) &&
|
|
637
|
+
/(not now|skip|close|dismiss|no thanks|^x$)/i.test(el.label) &&
|
|
606
638
|
/(button|link)/i.test(el.role));
|
|
607
639
|
if (dismissButton) {
|
|
608
640
|
lines.push(`\n🚨 [POPUP DETECTED]: If an optional modal/login is blocking the page, try clicking [${dismissButton.id}] ("${dismissButton.label}") to dismiss it.\n`);
|
|
@@ -620,10 +652,10 @@ export async function browserMap(session, options = {}, targetId = "default", co
|
|
|
620
652
|
let activeElementId;
|
|
621
653
|
try {
|
|
622
654
|
const { result } = await session.Runtime.evaluate({
|
|
623
|
-
expression: `(function() {
|
|
624
|
-
const ae = document.activeElement;
|
|
625
|
-
if (!ae || ae === document.body) return null;
|
|
626
|
-
return ae.getAttribute('data-wt-id') || null;
|
|
655
|
+
expression: `(function() {
|
|
656
|
+
const ae = document.activeElement;
|
|
657
|
+
if (!ae || ae === document.body) return null;
|
|
658
|
+
return ae.getAttribute('data-wt-id') || null;
|
|
627
659
|
})()`,
|
|
628
660
|
returnByValue: true
|
|
629
661
|
});
|
package/dist/tools/recording.js
CHANGED
|
@@ -102,6 +102,15 @@ export async function browserRecording(session, options, config) {
|
|
|
102
102
|
keepFrames: options.keepFrames ?? false,
|
|
103
103
|
});
|
|
104
104
|
logAction("browser_recording", sessionId, `stopped (${count} frames, compiled: ${compileRes.videoPath || "none"})`);
|
|
105
|
+
if (!compileRes.success) {
|
|
106
|
+
return {
|
|
107
|
+
success: false,
|
|
108
|
+
recordings: Array.from(activeRecordings.values()),
|
|
109
|
+
frameCount: count,
|
|
110
|
+
dir: state.dir,
|
|
111
|
+
error: compileRes.error,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
105
114
|
return {
|
|
106
115
|
success: true,
|
|
107
116
|
recordings: Array.from(activeRecordings.values()),
|