@weavetab/mcp 2.5.0-beta.1 → 2.5.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/AGENTS.md +2 -1
  2. package/ARCHITECTURE.md +6 -6
  3. package/LICENSE +1 -1
  4. package/README.md +164 -802
  5. package/SECURITY.md +33 -22
  6. package/TROUBLESHOOTING.md +3 -3
  7. package/dist/cdp/bridge.d.ts +10 -0
  8. package/dist/cdp/bridge.js +285 -78
  9. package/dist/cdp/connector.d.ts +7 -3
  10. package/dist/cdp/connector.js +196 -75
  11. package/dist/cdp/helpers.d.ts +4 -0
  12. package/dist/cdp/helpers.js +181 -0
  13. package/dist/cdp/lock.d.ts +9 -6
  14. package/dist/cdp/lock.js +72 -59
  15. package/dist/cdp/omni_broker.d.ts +2 -1
  16. package/dist/cdp/omni_broker.js +13 -7
  17. package/dist/cdp/profiles.d.ts +10 -0
  18. package/dist/cdp/profiles.js +65 -0
  19. package/dist/cdp/walker.d.ts +25 -1
  20. package/dist/cdp/walker.js +354 -7
  21. package/dist/cli/plugin.d.ts +0 -2
  22. package/dist/cli/plugin.js +51 -696
  23. package/dist/cli/updater.d.ts +1 -1
  24. package/dist/cli/updater.js +27 -7
  25. package/dist/config/loader.d.ts +33 -6
  26. package/dist/config/loader.js +174 -22
  27. package/dist/desktop/discovery.d.ts +44 -0
  28. package/dist/desktop/discovery.js +364 -0
  29. package/dist/index.js +140 -64
  30. package/dist/intelligence/marks.d.ts +36 -0
  31. package/dist/intelligence/marks.js +30 -0
  32. package/dist/intelligence/patternCompiler.d.ts +118 -0
  33. package/dist/intelligence/patternCompiler.js +259 -0
  34. package/dist/intelligence/sessionMemory.d.ts +2 -0
  35. package/dist/intelligence/sessionMemory.js +18 -8
  36. package/dist/overlay/ghost.js +2 -2
  37. package/dist/overlay/injector.js +3 -5
  38. package/dist/plugin/extension_bridge.d.ts +40 -0
  39. package/dist/plugin/extension_bridge.js +158 -0
  40. package/dist/plugin/interface.d.ts +151 -15
  41. package/dist/plugin/loader.d.ts +19 -1
  42. package/dist/plugin/loader.js +331 -63
  43. package/dist/plugin/pipeline.d.ts +75 -0
  44. package/dist/plugin/pipeline.js +183 -0
  45. package/dist/plugin/registry.js +12 -1
  46. package/dist/plugin/storage.d.ts +51 -0
  47. package/dist/plugin/storage.js +164 -0
  48. package/dist/security/secrets.d.ts +1 -0
  49. package/dist/security/secrets.js +13 -0
  50. package/dist/sensors/actionability.js +1 -2
  51. package/dist/sensors/guard.js +4 -10
  52. package/dist/sensors/mutations.js +12 -0
  53. package/dist/sensors/operation_monitor.js +1 -0
  54. package/dist/server.d.ts +1 -1
  55. package/dist/server.js +600 -236
  56. package/dist/state/agent.d.ts +1 -0
  57. package/dist/state/agent.js +3 -0
  58. package/dist/state/delta.d.ts +12 -4
  59. package/dist/state/delta.js +67 -14
  60. package/dist/state/session.d.ts +8 -2
  61. package/dist/state/session.js +60 -13
  62. package/dist/tools/browser_detect.d.ts +16 -2
  63. package/dist/tools/browser_detect.js +54 -6
  64. package/dist/tools/burst.d.ts +17 -7
  65. package/dist/tools/burst.js +240 -83
  66. package/dist/tools/checkpoint.d.ts +38 -0
  67. package/dist/tools/checkpoint.js +271 -0
  68. package/dist/tools/click.d.ts +1 -0
  69. package/dist/tools/click.js +97 -98
  70. package/dist/tools/clipboard.js +31 -24
  71. package/dist/tools/design_extract.js +1 -1
  72. package/dist/tools/eval.js +1 -1
  73. package/dist/tools/fill.d.ts +20 -1
  74. package/dist/tools/fill.js +199 -40
  75. package/dist/tools/find.d.ts +16 -0
  76. package/dist/tools/find.js +67 -44
  77. package/dist/tools/github/index.d.ts +19 -4
  78. package/dist/tools/github/index.js +40 -4
  79. package/dist/tools/highlight.d.ts +6 -2
  80. package/dist/tools/highlight.js +158 -72
  81. package/dist/tools/inspect.js +1 -1
  82. package/dist/tools/key.js +65 -56
  83. package/dist/tools/navigate.d.ts +1 -1
  84. package/dist/tools/navigate.js +57 -44
  85. package/dist/tools/pdf.js +21 -13
  86. package/dist/tools/pointer.d.ts +18 -3
  87. package/dist/tools/pointer.js +89 -1
  88. package/dist/tools/read.d.ts +28 -8
  89. package/dist/tools/read.js +230 -56
  90. package/dist/tools/scrape.js +67 -13
  91. package/dist/tools/screenshot.js +21 -16
  92. package/dist/tools/scroll.js +6 -11
  93. package/dist/tools/select.js +8 -2
  94. package/dist/tools/snapshot.js +3 -34
  95. package/dist/tools/tabs.d.ts +10 -10
  96. package/dist/tools/tabs.js +55 -39
  97. package/dist/tools/thought.d.ts +2 -0
  98. package/dist/tools/thought.js +13 -14
  99. package/dist/tools/type.d.ts +2 -0
  100. package/dist/tools/type.js +109 -60
  101. package/dist/tools/type_secret.js +42 -76
  102. package/dist/tools/upload.js +1 -1
  103. package/dist/tools/video_compiler.d.ts +1 -1
  104. package/dist/tools/video_compiler.js +11 -12
  105. package/dist/version.d.ts +5 -0
  106. package/dist/version.js +5 -0
  107. package/docs/DOCUMENTATION.md +82 -36
  108. package/docs/PLUGINS.md +82 -499
  109. package/docs/TOOLS.md +51 -99
  110. package/docs/architecture/blind-injection.md +83 -0
  111. package/docs/architecture/dual-injection.md +32 -0
  112. package/docs/architecture/overview.md +49 -0
  113. package/docs/architecture/security.md +35 -0
  114. package/docs/architecture/transport.md +37 -0
  115. package/docs/captcha-stealth/captcha-solver.md +75 -0
  116. package/docs/captcha-stealth/stealth-physics.md +60 -0
  117. package/docs/cli/cli-commands.md +44 -0
  118. package/docs/cli/cli-flags.md +20 -0
  119. package/docs/cli/cli-introduction.md +20 -0
  120. package/docs/cli/cli-scripting.md +32 -0
  121. package/docs/cli/g-ide/ide-antigravity.md +24 -0
  122. package/docs/cli/g-ide/ide-claude-code.md +24 -0
  123. package/docs/cli/g-ide/ide-claude-desktop.md +32 -0
  124. package/docs/cli/g-ide/ide-cursor.md +31 -0
  125. package/docs/cli/g-ide/ide-generic.md +34 -0
  126. package/docs/cli/g-ide/ide-opencode.md +24 -0
  127. package/docs/cli/g-ide/ide-vscode-copilot.md +24 -0
  128. package/docs/cli/g-ide/ide-windsurf.md +24 -0
  129. package/docs/configuration/config-locations.md +24 -0
  130. package/docs/configuration/configuration.md +73 -0
  131. package/docs/configuration/env-variables.md +19 -0
  132. package/docs/configuration/folders.md +55 -0
  133. package/docs/desktop-automation/electron-apps.md +217 -0
  134. package/docs/get-started/installation.md +47 -0
  135. package/docs/get-started/introduction.md +60 -0
  136. package/docs/get-started/quickstart.md +54 -0
  137. package/docs/get-started/setup-wizard.md +26 -0
  138. package/docs/intelligence/site-profiles.md +151 -0
  139. package/docs/skills-ecosystem/skills-overview.md +35 -0
  140. package/docs/skills-ecosystem/skills-recipes.md +36 -0
  141. package/docs/tools/tools.md +631 -0
  142. package/docs/troubleshooting/trouble-browser.md +27 -0
  143. package/docs/troubleshooting/trouble-config.md +20 -0
  144. package/docs/troubleshooting/trouble-loop.md +31 -0
  145. package/ext-dist/background.js +10 -62
  146. package/ext-dist/content.js +333 -921
  147. package/ext-dist/hud/hud.js +9 -9
  148. package/ext-dist/manifest.json +2 -6
  149. package/ext-dist/styles.css +6 -1
  150. package/mcp.json +108 -72
  151. package/package.json +14 -9
  152. package/server.json +3 -3
  153. package/docs/DEV_MODE.md +0 -68
  154. package/docs/dev_plugin.md +0 -310
  155. package/ext-dist/sidepanel/sidepanel.js +0 -25
  156. package/ext-dist/sidepanel.html +0 -547
@@ -9,10 +9,13 @@ import { cacheActionMap, getCachedActionMap, getRawCachedActionMap } from "../st
9
9
  import { startReadingSimulation } from "../overlay/ghost.js";
10
10
  import { suppressOverlays, restoreOverlays } from "../cdp/extension.js";
11
11
  import { sendToExtension } from "../cdp/helpers.js";
12
+ import { hideWeaveTabOverlay, restoreWeaveTabOverlay } from "../cdp/helpers.js";
12
13
  import { analyzeVisualFallback } from "../intelligence/vision.js";
14
+ import { marksSet } from "../intelligence/marks.js";
13
15
  import { getConfigAgentId } from "../config/loader.js";
14
- // Cap at 80 elements; form/modal groups are always kept, utility trimmed last.
15
- const MAX_ELEMENTS = 80;
16
+ // Cap at 35 elements by default to keep prompt tokens lean.
17
+ // Form, modal, and primary interactive elements are prioritized first.
18
+ const MAX_ELEMENTS = 35;
16
19
  function withTimeout(p, ms) {
17
20
  return Promise.race([
18
21
  p,
@@ -26,33 +29,121 @@ function formatLite(el, activeElementId, changedIds) {
26
29
  parts.push("disabled");
27
30
  if (activeElementId && el.id === activeElementId)
28
31
  parts.push("FOCUSED");
29
- if (el.viewport && el.viewport !== "visible")
30
- parts.push(`offscreen:${el.viewport}`);
32
+ if (el.viewport && el.viewport !== "visible" && el.viewport !== "in-viewport") {
33
+ parts.push(el.viewport);
34
+ }
31
35
  // ── Value / checked / placeholder ──────────────────────────────────────────
32
36
  const role = (el.role || '').toLowerCase();
33
37
  const type = (el.type || '').toLowerCase();
34
- // Radio / checkbox checked state — critical for quiz forms
38
+ // Radio / checkbox checked state
35
39
  if ((role === 'radio' || role === 'checkbox' || type === 'radio' || type === 'checkbox') && el.value !== undefined) {
36
40
  parts.push(el.value === 'true' || el.value === 'checked' ? '✓ checked' : '○ unchecked');
37
41
  }
38
42
  else if (el.value) {
39
- // Truncate long values to avoid token waste
40
- const v = el.value.length > 60 ? el.value.slice(0, 57) + '…' : el.value;
41
- parts.push(`"${v}"`);
43
+ const v = el.value.length > 45 ? el.value.slice(0, 42) + '…' : el.value;
44
+ parts.push(`val:"${v}"`);
42
45
  }
43
46
  else if (el.placeholder) {
44
- parts.push(`placeholder: "${el.placeholder}"`);
47
+ const p = el.placeholder.length > 35 ? el.placeholder.slice(0, 32) + '…' : el.placeholder;
48
+ parts.push(`ph:"${p}"`);
45
49
  }
46
- // ── Type suffix for inputs (e.g. input[email], input[date]) ────────────────
47
- const typeLabel = (role === 'textbox' || role === 'searchbox' || role === 'combobox') && type
50
+ // ── Type suffix for inputs ──────────────────────────────────────────────────
51
+ const typeLabel = (role === 'textbox' || role === 'searchbox' || role === 'combobox') && type && type !== 'text'
48
52
  ? `/${type}` : '';
49
- // ── Link hint ───────────────────────────────────────────────────────────────
50
- if (role === 'link' && el.hrefType && el.hrefType !== 'same-origin') {
51
- parts.push(el.hrefType);
53
+ // ── Link hint (only if external) ───────────────────────────────────────────
54
+ if (role === 'link' && el.hrefType === 'external') {
55
+ parts.push('ext');
52
56
  }
53
57
  const state = parts.length > 0 ? ` (${parts.join(', ')})` : '';
54
58
  const changed = changedIds?.has(el.id) ? ' ★' : '';
55
- return `[${el.id}] ${el.role}${typeLabel} "${el.label}"${state}${changed}`;
59
+ const label = el.label.length > 50 ? el.label.slice(0, 47) + '…' : el.label;
60
+ return `[${el.id}] ${el.role}${typeLabel} "${label}"${state}${changed}`;
61
+ }
62
+ // ─── 3.0 High-density formatter helpers ──────────────────────────────────────
63
+ /** Collapse a cluster of nav/footer links into a compact summary token. */
64
+ function collapseNavCluster(cluster) {
65
+ if (cluster.length === 0)
66
+ return '';
67
+ if (cluster.length === 1)
68
+ return formatLite(cluster[0]);
69
+ const first = cluster[0].label.slice(0, 20);
70
+ const last = cluster[cluster.length - 1].label.slice(0, 20);
71
+ const ids = `${cluster[0].id}…${cluster[cluster.length - 1].id}`;
72
+ return `[${ids}] ${cluster.length} links: "${first}" … "${last}"`;
73
+ }
74
+ /**
75
+ * Format the main element list using 3.0 semantics:
76
+ * - Active modals hoisted to the top
77
+ * - Items under shared container headings grouped
78
+ * - Nav link clusters (4+ consecutive) collapsed
79
+ */
80
+ function formatMap3(elements, activeElementId, changedIds) {
81
+ const lines = [];
82
+ // ── 1. Hoist active modal elements ───────────────────────────────────────
83
+ const modalEls = elements.filter(e => e.isModal);
84
+ const mainEls = elements.filter(e => !e.isModal);
85
+ if (modalEls.length > 0) {
86
+ const modalTitle = modalEls[0].modalTitle || 'Dialog';
87
+ lines.push(`╔═ ACTIVE DIALOG: "${modalTitle}" ${'═'.repeat(Math.max(0, 30 - modalTitle.length))}╗`);
88
+ for (const el of modalEls) {
89
+ lines.push(' ' + formatLite(el, activeElementId, changedIds));
90
+ }
91
+ lines.push(`╚${'═'.repeat(42)}╝`);
92
+ if (mainEls.length > 0)
93
+ lines.push('');
94
+ }
95
+ // ── 2. Main elements with entity grouping + nav cluster collapsing ────────
96
+ const SECTION_LABEL = {
97
+ navigation: "── NAV ──────────────────",
98
+ main_content: "── MAIN ─────────────────",
99
+ form: "── FORM ─────────────────",
100
+ sidebar: "── SIDEBAR ──────────────",
101
+ modal: "── MODAL ────────────────",
102
+ utility: "── OTHER ────────────────",
103
+ };
104
+ let currentSection = null;
105
+ let currentContainer = null;
106
+ let navCluster = [];
107
+ const flushNavCluster = () => {
108
+ if (navCluster.length >= 4) {
109
+ lines.push(collapseNavCluster(navCluster));
110
+ }
111
+ else {
112
+ for (const ne of navCluster) {
113
+ lines.push(formatLite(ne, activeElementId, changedIds));
114
+ }
115
+ }
116
+ navCluster = [];
117
+ };
118
+ for (const el of mainEls) {
119
+ // Section header
120
+ if (el.group !== currentSection) {
121
+ flushNavCluster();
122
+ currentSection = el.group;
123
+ currentContainer = null;
124
+ lines.push(SECTION_LABEL[currentSection]);
125
+ }
126
+ // Accumulate nav/utility link cluster for potential collapsing
127
+ const isNavLink = (el.group === 'navigation' || el.group === 'utility') && el.role === 'link';
128
+ if (isNavLink) {
129
+ navCluster.push(el);
130
+ continue;
131
+ }
132
+ else {
133
+ flushNavCluster();
134
+ }
135
+ // Container/entity heading
136
+ const ctx = el.containerContext || null;
137
+ if (ctx && ctx !== currentContainer) {
138
+ currentContainer = ctx;
139
+ const header = ctx.length > 55 ? ctx.slice(0, 52) + '…' : ctx;
140
+ lines.push(` ┌ ${header}`);
141
+ }
142
+ const indent = ctx ? ' │ ' : '';
143
+ lines.push(indent + formatLite(el, activeElementId, changedIds));
144
+ }
145
+ flushNavCluster();
146
+ return lines;
56
147
  }
57
148
  /** Full markdown table mode — only used when full:true is explicitly passed */
58
149
  function formatFull(elements, url, title, hint) {
@@ -83,6 +174,99 @@ function formatFull(elements, url, title, hint) {
83
174
  }
84
175
  return lines.join('\n');
85
176
  }
177
+ /**
178
+ * Inject numbered Set-of-Marks badges over all interactive elements,
179
+ * take a screenshot, then strip the badges. Returns structured mark data
180
+ * and the annotated image. Zero page pollution after completion.
181
+ */
182
+ async function buildVisualGrounding(session, targetId, elements) {
183
+ try {
184
+ // Build mark data from DOM elements that have coordinates
185
+ const domMarks = [];
186
+ for (const el of elements) {
187
+ if (el.x === undefined || el.y === undefined)
188
+ continue;
189
+ const w = el.width ?? 80;
190
+ const h = el.height ?? 24;
191
+ domMarks.push({
192
+ mark: domMarks.length + 1,
193
+ x: el.x,
194
+ y: el.y,
195
+ width: w,
196
+ height: h,
197
+ ref: el.id,
198
+ label: el.label || undefined,
199
+ type: el.role === "link" ? "link"
200
+ : el.role === "button" ? "button"
201
+ : (el.role === "textbox" || el.role === "searchbox" || el.role === "combobox") ? "input"
202
+ : el.role === "canvas" ? "canvas_region"
203
+ : "unknown",
204
+ });
205
+ }
206
+ // Also discover canvas elements that may not be in the DOM map
207
+ const { result: canvasRes } = await session.Runtime.evaluate({
208
+ expression: `(function(){
209
+ var canvases = [...document.querySelectorAll('canvas')];
210
+ return JSON.stringify(canvases.map(function(c){
211
+ var r = c.getBoundingClientRect();
212
+ return r.width > 0 && r.height > 0 ? {
213
+ x: Math.round(r.left + r.width/2), y: Math.round(r.top + r.height/2),
214
+ w: Math.round(r.width), h: Math.round(r.height)
215
+ } : null;
216
+ }).filter(Boolean));
217
+ })()`,
218
+ returnByValue: true,
219
+ }).catch(() => ({ result: { value: '[]' } }));
220
+ const extraCanvas = JSON.parse(canvasRes.value ?? '[]');
221
+ // Add canvas regions not already covered by a nearby DOM mark
222
+ for (const c of extraCanvas) {
223
+ const alreadyCovered = domMarks.some(m => Math.abs(m.x - c.x) < 40 && Math.abs(m.y - c.y) < 40);
224
+ if (!alreadyCovered) {
225
+ domMarks.push({ mark: domMarks.length + 1, x: c.x, y: c.y, width: c.w, height: c.h, type: "canvas_region" });
226
+ }
227
+ }
228
+ if (domMarks.length === 0)
229
+ return null;
230
+ // Inject badges — fixed-position, shadow-safe, above everything
231
+ const injectExpr = `(function(marks){
232
+ var BADGE_KEY = '__wt_som_badges__';
233
+ if (window[BADGE_KEY]) window[BADGE_KEY].forEach(function(d){ d.parentNode && d.parentNode.removeChild(d); });
234
+ window[BADGE_KEY] = [];
235
+ marks.forEach(function(m){
236
+ var d = document.createElement('div');
237
+ d.setAttribute('data-wt-som', String(m.mark));
238
+ d.style.cssText = 'position:fixed;left:'+(m.x - 10)+'px;top:'+(m.y - 10)+'px;width:20px;height:20px;z-index:2147483647;pointer-events:none;background:#7c3aed;color:#fff;font:bold 11px/20px monospace;text-align:center;border-radius:50%;box-shadow:0 1px 4px rgba(0,0,0,.5);';
239
+ d.textContent = String(m.mark);
240
+ document.documentElement.appendChild(d);
241
+ window[BADGE_KEY].push(d);
242
+ });
243
+ return true;
244
+ })(${JSON.stringify(domMarks)})`;
245
+ await session.Runtime.evaluate({ expression: injectExpr, returnByValue: true }).catch(() => { });
246
+ // Hide Weavetab overlay so badges are the only overlay in the shot
247
+ await hideWeaveTabOverlay(session).catch(() => { });
248
+ // Capture screenshot
249
+ const { data } = await session.Page.captureScreenshot({ format: 'jpeg', quality: 75 });
250
+ // Restore overlay
251
+ await restoreWeaveTabOverlay(session).catch(() => { });
252
+ // Strip badges immediately
253
+ await session.Runtime.evaluate({
254
+ expression: `(function(){ var b=window['__wt_som_badges__']; if(b) b.forEach(function(d){ d.parentNode&&d.parentNode.removeChild(d); }); window['__wt_som_badges__']=[]; })()`,
255
+ returnByValue: true,
256
+ }).catch(() => { });
257
+ // Persist in mark cache for browser_click({ mark: N }) resolution
258
+ const markMap = new Map();
259
+ for (const m of domMarks) {
260
+ markMap.set(m.mark, { x: m.x, y: m.y, width: m.width, height: m.height, ref: m.ref, label: m.label, type: m.type });
261
+ }
262
+ marksSet(targetId, markMap);
263
+ return { marks: domMarks, imageBase64: data };
264
+ }
265
+ catch (err) {
266
+ logDebug(`[buildVisualGrounding] failed: ${err}`);
267
+ return null;
268
+ }
269
+ }
86
270
  export async function browserMap(session, options = {}, targetId = "default", config) {
87
271
  logDebug(`[browserMap] Initiating read for targetId: ${targetId}, options: ${JSON.stringify(options)}`);
88
272
  // ── Wait for DOM and Visual Stability ──────────────────────────────────────
@@ -174,7 +358,7 @@ export async function browserMap(session, options = {}, targetId = "default", co
174
358
  query: options.query,
175
359
  limit: options.limit,
176
360
  deep: options.deep,
177
- prune: options.prune,
361
+ prune: options.prune !== false,
178
362
  }, targetId), 8000).catch(() => _emptyMap);
179
363
  logDebug(`[browserMap] buildActionMap completed. Elements found: ${map.elements.length}`);
180
364
  // Auto-fallback to AX tree when DOM walker returns sparse or unlabeled-heavy results
@@ -335,19 +519,24 @@ export async function browserMap(session, options = {}, targetId = "default", co
335
519
  if (map.elements.length > 0 && newRefs.length === 0) {
336
520
  newRefs.push(...toRefEntries(map.elements));
337
521
  }
338
- // ── Delta mode ─────────────────────────────────────────────────────────────
339
- if (options.delta) {
522
+ // ── Delta / Auto-Delta mode ────────────────────────────────────────────────
523
+ // If delta is explicit OR if the user is re-reading the same page without full/force options,
524
+ // auto-delta returns only what changed (saving up to 95% prompt tokens).
525
+ const autoDeltaEligible = options.delta === true || (options.delta !== false &&
526
+ !options.full &&
527
+ !options.query &&
528
+ !options.scope);
529
+ if (autoDeltaEligible) {
340
530
  const richDiff = await computeRichDelta(targetId, map.url, map.elements);
341
- if (richDiff) {
342
- if ("delta_timeout" in richDiff) {
343
- // Delta mode timed out — fall through to full read
344
- }
345
- else {
531
+ if (richDiff && !("delta_timeout" in richDiff)) {
532
+ const totalChanges = richDiff.added.length + richDiff.changed.length + richDiff.removed.length;
533
+ // Auto-delta: if 0 changes or <= 15 changes, return the diff directly
534
+ if (options.delta === true || totalChanges <= 15) {
346
535
  const lines = [];
347
536
  const header = `Page: ${map.title || map.url} | ${map.elements.length} elements`;
348
537
  lines.push(header);
349
- if (richDiff.added.length === 0 && richDiff.changed.length === 0 && richDiff.removed.length === 0 && (!richDiff.iframes || richDiff.iframes.length === 0)) {
350
- lines.push("stable — no changes detected");
538
+ if (totalChanges === 0 && (!richDiff.iframes || richDiff.iframes.length === 0)) {
539
+ lines.push("stable — no DOM changes since last map (use existing refs)");
351
540
  }
352
541
  else {
353
542
  for (const r of richDiff.added)
@@ -372,7 +561,6 @@ export async function browserMap(session, options = {}, targetId = "default", co
372
561
  return lines.join("\n");
373
562
  }
374
563
  }
375
- // No prior snapshot → fall through to full read
376
564
  }
377
565
  storeSnapshot(targetId, map.url, newRefs);
378
566
  seedRichSnapshot(targetId, map.url, map.elements);
@@ -428,15 +616,7 @@ export async function browserMap(session, options = {}, targetId = "default", co
428
616
  ? ` (pruned ${beforePruneCount - map.elements.length})`
429
617
  : ""));
430
618
  lines.push("");
431
- // ── Semantic section grouping ────────────────────────────────────────────────
432
- const SECTION_LABEL = {
433
- navigation: "── NAV ──────────────────",
434
- main_content: "── MAIN ─────────────────",
435
- form: "── FORM ─────────────────",
436
- sidebar: "── SIDEBAR ──────────────",
437
- modal: "── MODAL ────────────────",
438
- utility: "── OTHER ────────────────",
439
- };
619
+ // ── Semantic section grouping (3.0) ────────────────────────────────────────
440
620
  let activeElementId;
441
621
  try {
442
622
  const { result } = await session.Runtime.evaluate({
@@ -452,36 +632,30 @@ export async function browserMap(session, options = {}, targetId = "default", co
452
632
  }
453
633
  }
454
634
  catch { }
455
- let currentSection = null;
456
- for (const el of capped) {
457
- if (el.group !== currentSection) {
458
- currentSection = el.group;
459
- lines.push(SECTION_LABEL[currentSection]);
460
- }
461
- lines.push(formatLite(el, activeElementId, changedIds));
462
- }
635
+ lines.push(...formatMap3(capped, activeElementId, changedIds));
463
636
  if (overflowCount > 0) {
464
637
  lines.push(`(+${overflowCount} more below fold — call browser_scroll down to reveal)`);
465
638
  }
466
639
  if (visualFallback)
467
640
  lines.push(visualFallback);
468
- // Context-scoped tool hint — collapse 34-tool surface to ~12 relevant tools.
469
- // Detected from map elements already computed above — zero extra latency.
470
- const hasForm = map.elements.some(e => e.group === "form" || e.role === "input" || e.role === "combobox" || e.role === "checkbox" || e.role === "radio");
471
- const hasSelect = map.elements.some(e => e.role === "combobox" || e.role === "listbox");
472
- const hasCanvas = map.url?.includes("canvas") || false;
473
- const isGithub = map.url?.includes("github.com") ?? false;
474
- const baseTools = ["browser_click", "browser_type", "browser_key", "browser_scroll", "browser_wait", "browser_navigate", "browser_tabs"];
475
- const formTools = hasForm ? ["browser_select", "browser_type_secret", "browser_find"] : [];
476
- const githubTools = isGithub ? ["github_analyze", "github_read", "github_issues", "github_get_pr"] : [];
477
- const canvasTools = hasCanvas ? ["browser_canvas", "browser_screenshot"] : ["browser_screenshot"];
478
- const diagTools = ["browser_inspect", "browser_highlight", "browser_scrape", "browser_burst"];
479
- const contextTools = [...new Set([...baseTools, ...formTools, ...githubTools, ...canvasTools, ...diagTools])];
480
- lines.push(`\n_tools [${contextTools.join(", ")}]`);
481
641
  const output = lines.join("\n");
482
642
  const wordCount = output.split(/\s+/).length;
483
643
  startReadingSimulation(session, wordCount, config);
484
644
  sendToExtension(session, { type: 'Weavetab_STATE', hud: { action: '' } }).catch(() => { });
645
+ // ── Visual Grounding (Set-of-Marks) ────────────────────────────────────────
646
+ // Trigger when: visual:true (always) OR visual:"auto" (DOM element count < 5).
647
+ const shouldVisual = options.visual === true ||
648
+ (options.visual === "auto" && capped.length < 5);
649
+ if (shouldVisual) {
650
+ const vg = await buildVisualGrounding(session, targetId, map.elements);
651
+ if (vg) {
652
+ const markSummary = vg.marks
653
+ .map(m => ` [${m.mark}] ${m.type ?? "unknown"}${m.ref ? ` ${m.ref}` : ""} "${m.label ?? ""}" @(${m.x},${m.y})`)
654
+ .join("\n");
655
+ const annotatedText = output + `\n\n── Visual Marks (use browser_click({ mark: N })) ──\n${markSummary}`;
656
+ return { text: annotatedText, imageBase64: vg.imageBase64, marks: vg.marks };
657
+ }
658
+ }
485
659
  return output;
486
660
  }
487
661
  // Backward-compat alias
@@ -1,3 +1,6 @@
1
+ import { getConfigAgentId } from "../config/loader.js";
2
+ import { SessionSecretTracker } from "../security/secrets.js";
3
+ import { sessionRegistry } from "../state/session.js";
1
4
  export async function browserScrape(session, options, _config) {
2
5
  const { query, type = "css", attributes = ["textContent"], multiple = true } = options;
3
6
  const res = await session.Runtime.evaluate({
@@ -34,14 +37,20 @@ export async function browserScrape(session, options, _config) {
34
37
  }
35
38
 
36
39
  const attrs = ${JSON.stringify(attributes)};
40
+ var secretRegex = /password|passcode|pin|passphrase|secret|token|auth.?key|access.?code|mot.?de.?passe|contrase[nñ]a|kennwort|senha/i;
37
41
  nodes.forEach(node => {
38
42
  const item = {};
39
43
 
40
44
  // Secret containment: redact any element marked as sensitive.
41
- // data-wt-secret is set by browser_type_secret at injection time.
42
45
  const isSecret = node.hasAttribute && (
43
46
  node.hasAttribute('data-wt-secret') ||
44
- (node.tagName === 'INPUT' && node.type === 'password')
47
+ (node.tagName === 'INPUT' && (
48
+ node.type === 'password' ||
49
+ secretRegex.test(node.getAttribute('autocomplete') || '') ||
50
+ secretRegex.test(node.getAttribute('name') || '') ||
51
+ secretRegex.test(node.getAttribute('id') || '') ||
52
+ secretRegex.test(node.getAttribute('aria-label') || '')
53
+ ))
45
54
  );
46
55
 
47
56
  if (isSecret) {
@@ -53,6 +62,9 @@ export async function browserScrape(session, options, _config) {
53
62
  item[attr] = node[attr]?.trim() || '';
54
63
  } else if (attr === 'outerHTML') {
55
64
  item[attr] = node.outerHTML;
65
+ } else if (attr === 'value') {
66
+ // Read live DOM property (not HTML attribute) so getter overrides work
67
+ item[attr] = node.value !== undefined ? String(node.value) : '';
56
68
  } else {
57
69
  item[attr] = node.getAttribute(attr) || node[attr] || '';
58
70
  }
@@ -98,6 +110,7 @@ export async function browserScrape(session, options, _config) {
98
110
  const fullExpr = type === 'xpath'
99
111
  ? `(function(){
100
112
  var attrs = ${JSON.stringify(attributes)};
113
+ var secretRegex = /password|passcode|pin|passphrase|secret|token|auth.?key|access.?code|mot.?de.?passe|contrase[nñ]a|kennwort|senha/i;
101
114
  var snap = document.evaluate(${JSON.stringify(query)}, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
102
115
  var limit = ${multiple} ? snap.snapshotLength : Math.min(snap.snapshotLength, 1);
103
116
  var results = [];
@@ -105,11 +118,25 @@ export async function browserScrape(session, options, _config) {
105
118
  var node = snap.snapshotItem(i);
106
119
  if (!node || node.nodeType !== 1) continue;
107
120
  var item = {};
108
- attrs.forEach(function(attr) {
109
- if (attr === 'textContent' || attr === 'innerText') { item[attr] = (node[attr] || '').trim(); }
110
- else if (attr === 'outerHTML') { item[attr] = node.outerHTML; }
111
- else { item[attr] = node.getAttribute ? (node.getAttribute(attr) || node[attr] || '') : ''; }
112
- });
121
+ var isSecret = node.hasAttribute && (
122
+ node.hasAttribute('data-wt-secret') ||
123
+ (node.tagName === 'INPUT' && (
124
+ node.type === 'password' ||
125
+ secretRegex.test(node.getAttribute('autocomplete') || '') ||
126
+ secretRegex.test(node.getAttribute('name') || '') ||
127
+ secretRegex.test(node.getAttribute('id') || '') ||
128
+ secretRegex.test(node.getAttribute('aria-label') || '')
129
+ ))
130
+ );
131
+ if (isSecret) {
132
+ attrs.forEach(function(attr) { item[attr] = '[REDACTED]'; });
133
+ } else {
134
+ attrs.forEach(function(attr) {
135
+ if (attr === 'textContent' || attr === 'innerText') { item[attr] = (node[attr] || '').trim(); }
136
+ else if (attr === 'outerHTML') { item[attr] = node.outerHTML; }
137
+ else { item[attr] = node.getAttribute ? (node.getAttribute(attr) || node[attr] || '') : ''; }
138
+ });
139
+ }
113
140
  results.push(item);
114
141
  }
115
142
  return { data: results };
@@ -120,13 +147,28 @@ export async function browserScrape(session, options, _config) {
120
147
  try { nodes = Array.from(window.__Weavetab_shadowQuery_all(${JSON.stringify(query)})); } catch(_){}
121
148
  }
122
149
  var attrs = ${JSON.stringify(attributes)};
150
+ var secretRegex = /password|passcode|pin|passphrase|secret|token|auth.?key|access.?code|mot.?de.?passe|contrase[nñ]a|kennwort|senha/i;
123
151
  return { data: nodes.map(function(node) {
124
152
  var item = {};
125
- attrs.forEach(function(attr) {
126
- if (attr === 'textContent' || attr === 'innerText') { item[attr] = (node[attr] || '').trim(); }
127
- else if (attr === 'outerHTML') { item[attr] = node.outerHTML; }
128
- else { item[attr] = node.getAttribute(attr) || node[attr] || ''; }
129
- });
153
+ var isSecret = node.hasAttribute && (
154
+ node.hasAttribute('data-wt-secret') ||
155
+ (node.tagName === 'INPUT' && (
156
+ node.type === 'password' ||
157
+ secretRegex.test(node.getAttribute('autocomplete') || '') ||
158
+ secretRegex.test(node.getAttribute('name') || '') ||
159
+ secretRegex.test(node.getAttribute('id') || '') ||
160
+ secretRegex.test(node.getAttribute('aria-label') || '')
161
+ ))
162
+ );
163
+ if (isSecret) {
164
+ attrs.forEach(function(attr) { item[attr] = '[REDACTED]'; });
165
+ } else {
166
+ attrs.forEach(function(attr) {
167
+ if (attr === 'textContent' || attr === 'innerText') { item[attr] = (node[attr] || '').trim(); }
168
+ else if (attr === 'outerHTML') { item[attr] = node.outerHTML; }
169
+ else { item[attr] = node.getAttribute(attr) || node[attr] || ''; }
170
+ });
171
+ }
130
172
  return item;
131
173
  })};
132
174
  })()`;
@@ -136,5 +178,17 @@ export async function browserScrape(session, options, _config) {
136
178
  }
137
179
  }
138
180
  }
139
- return { success: true, data: val?.data || [], message: `Extracted ${val?.data?.length || 0} elements` };
181
+ let targetId = "default";
182
+ const active = sessionRegistry.getActive(getConfigAgentId(_config));
183
+ if (active)
184
+ targetId = active.targetId;
185
+ const rawData = (val?.data || []);
186
+ const sanitizedData = rawData.map(item => {
187
+ const sanitizedItem = {};
188
+ for (const [k, v] of Object.entries(item)) {
189
+ sanitizedItem[k] = typeof v === 'string' ? SessionSecretTracker.redact(targetId, v) : v;
190
+ }
191
+ return sanitizedItem;
192
+ });
193
+ return { success: true, data: sanitizedData, message: `Extracted ${sanitizedData.length} elements` };
140
194
  }
@@ -1,4 +1,4 @@
1
- import { hideWeaveTabOverlay, restoreWeaveTabOverlay } from "../cdp/helpers.js";
1
+ import { hideWeaveTabOverlay, restoreWeaveTabOverlay, maskSecretFieldsForCapture, restoreSecretFieldsAfterCapture } from "../cdp/helpers.js";
2
2
  export async function browserScreenshot(session, options = {}, _config) {
3
3
  const format = options.format ?? "png";
4
4
  const quality = format === "jpeg" ? (options.quality ?? 90) : undefined;
@@ -42,33 +42,38 @@ export async function browserScreenshot(session, options = {}, _config) {
42
42
  if (clip)
43
43
  params.clip = clip;
44
44
  }
45
- // ── AI-blind: hide Weavetab overlay elements before screenshot ──────────────
45
+ // ── AI-blind: hide Weavetab overlay elements and mask secrets before screenshot ─
46
46
  // The chat panel, ghost cursor, thought bubble, HUD iframe, and all scripted
47
- // UI are FOR THE USER only. The AI must never see or interact with them.
48
- // hideOverlay defaults to true — only set to false if explicitly requested.
47
+ // UI are FOR THE USER only. Secret values are masked to prevent visual leaks.
49
48
  const shouldHide = options.hideOverlay !== false;
50
49
  if (shouldHide) {
51
50
  await hideWeaveTabOverlay(session);
52
51
  }
52
+ await maskSecretFieldsForCapture(session);
53
53
  let data;
54
54
  try {
55
- const res = await session.Page.captureScreenshot(params);
56
- data = res.data;
57
- }
58
- catch (e) {
59
- if (params.captureBeyondViewport) {
60
- params.captureBeyondViewport = false;
61
- delete params.clip;
55
+ try {
62
56
  const res = await session.Page.captureScreenshot(params);
63
57
  data = res.data;
64
58
  }
65
- else {
66
- throw e;
59
+ catch (e) {
60
+ if (params.captureBeyondViewport) {
61
+ params.captureBeyondViewport = false;
62
+ delete params.clip;
63
+ const res = await session.Page.captureScreenshot(params);
64
+ data = res.data;
65
+ }
66
+ else {
67
+ throw e;
68
+ }
67
69
  }
68
70
  }
69
- // Restore overlay visibility immediately after capture
70
- if (shouldHide) {
71
- await restoreWeaveTabOverlay(session);
71
+ finally {
72
+ // Restore secret fields and overlay visibility immediately after capture
73
+ await restoreSecretFieldsAfterCapture(session);
74
+ if (shouldHide) {
75
+ await restoreWeaveTabOverlay(session);
76
+ }
72
77
  }
73
78
  // ─────────────────────────────────────────────────────────────────────────────
74
79
  const width = clip ? clip.width : vp.w;
@@ -5,8 +5,7 @@
5
5
  */
6
6
  import { buildActionMap, findBestMatch, safeSelector } from "../cdp/walker.js";
7
7
  import { getCachedActionMap, cacheActionMap } from "../state/cache.js";
8
- import { computeDelta, storeSnapshot } from "../state/delta.js";
9
- import { waitForDOMStable } from "../sensors/guard.js";
8
+ import { storeSnapshot } from "../state/delta.js";
10
9
  import { sessionRegistry } from "../state/session.js";
11
10
  import { logAction } from "../audit/logger.js";
12
11
  import { getConfigAgentId } from "../config/loader.js";
@@ -175,22 +174,18 @@ export async function browserScroll(session, args, config) {
175
174
  x: vp.x, y: vp.y,
176
175
  deltaX: chunkX, deltaY: chunkY
177
176
  });
178
- await sleep(profile.baseDelay + Math.random() * 8);
177
+ await sleep(profile.baseDelay + Math.random() * 4);
179
178
  }
180
179
  }
181
- await waitForDOMStable(session, 80, 600).catch(() => { });
182
- await sendToExtension(session, { type: 'Weavetab_STATE', hud: { action: '' } });
183
- const nextMap = await buildActionMap(session, { lite: true }, targetId);
184
- cacheActionMap(targetId, nextMap);
185
- const diff = computeDelta(targetId, nextMap.url, nextMap.elements);
186
- const changesCount = diff ? diff.added.length + diff.removed.length + diff.changed.length : 0;
180
+ await sleep(40);
181
+ await sendToExtension(session, { type: 'Weavetab_STATE', hud: { action: '' } }).catch(() => { });
187
182
  logAction("browser_scroll", direction, `${distance}px`);
188
183
  return {
189
184
  success: true,
190
185
  scrolled: id ?? label ?? "window",
191
186
  direction,
192
187
  distance,
193
- new_visible_count: nextMap.elements.length,
194
- dom_changed: changesCount > 0,
188
+ new_visible_count: 0,
189
+ dom_changed: true,
195
190
  };
196
191
  }
@@ -9,6 +9,12 @@ async function getSelector(session, options, config) {
9
9
  return options.selector;
10
10
  if (options.ref || options.id) {
11
11
  const ref = (options.ref || options.id);
12
+ if (ref.startsWith("w:")) {
13
+ return `[data-wt-id="${ref.slice(2)}"]`;
14
+ }
15
+ if (ref.startsWith("#") || ref.startsWith(".") || ref.startsWith("[")) {
16
+ return ref;
17
+ }
12
18
  let targetId = "default";
13
19
  const active = sessionRegistry.getActive(getConfigAgentId(config));
14
20
  if (active)
@@ -20,13 +26,13 @@ async function getSelector(session, options, config) {
20
26
  map = await buildActionMap(session, { lite: true }, targetId);
21
27
  cacheActionMap(targetId, map);
22
28
  }
23
- const element = map.elements.find(el => el.id === ref || el.id.endsWith(ref.startsWith("w:") ? ref.slice(2) : ref));
29
+ const element = map.elements.find(el => el.id === ref || el.label === ref || el.id.endsWith(ref.startsWith("w:") ? ref.slice(2) : ref));
24
30
  if (element) {
25
31
  const sel = safeSelector(element);
26
32
  if (sel)
27
33
  return sel;
28
34
  }
29
- throw new Error(`Could not resolve element for ref/id: ${ref}`);
35
+ return `[name="${ref}"], #${ref}, [aria-label="${ref}"]`;
30
36
  }
31
37
  if (options.nodeId) {
32
38
  const { nodeIds } = await session.DOM.pushNodesByBackendIdsToFrontend({ backendNodeIds: [options.nodeId] });