@hypothesi/tauri-mcp-server 0.8.3 → 0.9.0

package/README.md

@@ -11,7 +11,7 @@ A **Model Context Protocol (MCP) server** that enables AI assistants like Claude
 
 | Category | Capabilities |
 |----------|-------------|
-| 🎯 **UI Automation** | Screenshots, clicks, typing, scrolling, element finding |
+| 🎯 **UI Automation** | Screenshots, clicks, typing, scrolling, element finding, visual element picker |
 | 🔍 **IPC Monitoring** | Capture and inspect Tauri IPC calls in real-time |
 | 📱 **Mobile Dev** | List Android emulators & iOS simulators |
 | 📋 **Logs** | Stream console, Android logcat, iOS, and system logs |
@@ -86,7 +86,7 @@ await driver_session({ action: "stop", appIdentifier: 9223 })
 await driver_session({ action: "stop" })
 ```
 
-## Available Tools (18 total)
+## Available Tools (20 total)
 
 ### Setup & Configuration
 
@@ -108,6 +108,8 @@ await driver_session({ action: "stop" })
 | `webview_get_styles` | Get computed CSS styles |
 | `webview_execute_js` | Execute JavaScript in webview |
 | `webview_dom_snapshot` | Get structured DOM snapshot (accessibility or structure) |
+| `webview_select_element` | Visual element picker — user clicks an element, returns metadata + screenshot |
+| `webview_get_pointed_element` | Get metadata for element user Alt+Shift+Clicked |
 | `manage_window` | List windows, get info, or resize |
 
 ### IPC & Plugin

package/dist/driver/element-picker.js

@@ -0,0 +1,272 @@
+/**
+ * Element picker module for MCP Server Tauri.
+ *
+ * Provides two tools:
+ * - selectElement: Agent-initiated picker overlay (user clicks element)
+ * - getPointedElement: Retrieve element user pointed at via Alt+Shift+Click
+ */
+import { z } from 'zod';
+import { executeInWebview, executeAsyncInWebview } from './webview-executor.js';
+import { ensureSessionAndConnect } from './plugin-client.js';
+import { SCRIPTS, buildScript } from './scripts/index.js';
+import { WindowTargetSchema } from './webview-interactions.js';
+import { getHtml2CanvasSource, HTML2CANVAS_SCRIPT_ID, } from './scripts/html2canvas-loader.js';
+import { registerScript, isScriptRegistered } from './script-manager.js';
+// ============================================================================
+// Schemas
+// ============================================================================
+export const SelectElementSchema = WindowTargetSchema.extend({
+    timeout: z.number().min(5000).max(120000).optional().default(60000)
+        .describe('Timeout in ms for user to pick an element (5000-120000, default 60000)'),
+});
+export const GetPointedElementSchema = WindowTargetSchema.extend({});
+// ============================================================================
+// Helpers
+// ============================================================================
+/**
+ * Format element metadata into a readable text block.
+ */
+function formatElementMetadata(meta) {
+    const lines = [];
+    lines.push(`## Element: <${meta.tag}>`);
+    if (meta.id) {
+        lines.push(`**ID:** ${meta.id}`);
+    }
+    if (meta.classes.length > 0) {
+        lines.push(`**Classes:** ${meta.classes.join(', ')}`);
+    }
+    lines.push(`**CSS Selector:** \`${meta.cssSelector}\``);
+    if (meta.xpath) {
+        lines.push(`**XPath:** \`${meta.xpath}\``);
+    }
+    // Bounding rect
+    const r = meta.boundingRect;
+    lines.push(`**Bounding Rect:** ${Math.round(r.width)}x${Math.round(r.height)} at (${Math.round(r.x)}, ${Math.round(r.y)})`);
+    // Attributes (skip id and class which are already shown)
+    const attrEntries = Object.entries(meta.attributes).filter(([k]) => { return k !== 'id' && k !== 'class'; });
+    if (attrEntries.length > 0) {
+        lines.push(`**Attributes:** ${attrEntries.map(([k, v]) => { return `${k}="${v}"`; }).join(', ')}`);
+    }
+    if (meta.textContent) {
+        const text = meta.textContent.length > 200
+            ? meta.textContent.substring(0, 200) + '...'
+            : meta.textContent;
+        lines.push(`**Text Content:** ${text}`);
+    }
+    // Computed styles (only non-default interesting ones)
+    const styleEntries = Object.entries(meta.computedStyles);
+    if (styleEntries.length > 0) {
+        lines.push('**Computed Styles:**');
+        for (const [prop, val] of styleEntries) {
+            lines.push(`  ${prop}: ${val}`);
+        }
+    }
+    if (meta.parentChain.length > 0) {
+        lines.push('**Parent Chain:**');
+        for (const parent of meta.parentChain) {
+            let desc = `  <${parent.tag}>`;
+            if (parent.id) {
+                desc += `#${parent.id}`;
+            }
+            if (parent.classes && parent.classes.length > 0) {
+                desc += `.${parent.classes.join('.')}`;
+            }
+            if (parent.boundingRect) {
+                desc += ` (${Math.round(parent.boundingRect.width)}x${Math.round(parent.boundingRect.height)})`;
+            }
+            lines.push(desc);
+        }
+    }
+    return lines.join('\n');
+}
+/**
+ * Inject a script that removes all picker highlight elements from the DOM.
+ */
+async function cleanupPickerHighlights(windowId, appIdentifier) {
+    const script = `(function() {
+      var els = document.querySelectorAll('[data-mcp-picker]');
+      for (var i = 0; i < els.length; i++) { els[i].parentNode.removeChild(els[i]); }
+      return 'Cleaned up ' + els.length + ' picker elements';
+   })()`;
+    try {
+        await executeInWebview(script, windowId, appIdentifier);
+    }
+    catch {
+        // Best effort cleanup
+    }
+}
+/**
+ * Capture a screenshot of a specific element using html2canvas.
+ * Returns the base64 data URL of the cropped element image, or null on failure.
+ */
+async function captureElementScreenshot(cssSelector, windowId) {
+    // Ensure html2canvas is loaded in the webview
+    try {
+        const isRegistered = await isScriptRegistered(HTML2CANVAS_SCRIPT_ID);
+        if (!isRegistered) {
+            const source = getHtml2CanvasSource();
+            await registerScript(HTML2CANVAS_SCRIPT_ID, 'inline', source);
+        }
+    }
+    catch {
+        // Script manager unavailable — we'll inline the library in the capture script
+    }
+    const escapedSelector = cssSelector.replace(/\\/g, '\\\\').replace(/'/g, '\\\'');
+    // Build a script that captures just the element with html2canvas
+    const captureScript = `
+      const html2canvasFn = typeof html2canvas !== 'undefined' ? html2canvas :
+                           (typeof window !== 'undefined' && window.html2canvas) ? window.html2canvas :
+                           (typeof self !== 'undefined' && self.html2canvas) ? self.html2canvas :
+                           (typeof globalThis !== 'undefined' && globalThis.html2canvas) ? globalThis.html2canvas : null;
+
+      if (!html2canvasFn) {
+         throw new Error('html2canvas not loaded');
+      }
+
+      const el = document.querySelector('${escapedSelector}');
+      if (!el) {
+         throw new Error('Element not found for screenshot');
+      }
+
+      const canvas = await html2canvasFn(el, {
+         backgroundColor: null,
+         scale: window.devicePixelRatio || 1,
+         logging: false,
+         useCORS: true,
+         allowTaint: false,
+         imageTimeout: 5000,
+      });
+
+      if (!canvas) {
+         throw new Error('html2canvas returned null canvas');
+      }
+
+      const dataUrl = canvas.toDataURL('image/png');
+      if (!dataUrl || !dataUrl.startsWith('data:image/')) {
+         throw new Error('Invalid data URL from canvas');
+      }
+
+      return dataUrl;
+   `;
+    try {
+        const dataUrl = await executeAsyncInWebview(captureScript, windowId, 10000);
+        if (!dataUrl || !dataUrl.startsWith('data:image/')) {
+            return null;
+        }
+        // Extract base64 data from data URL
+        const commaIndex = dataUrl.indexOf(',');
+        if (commaIndex === -1) {
+            return null;
+        }
+        return {
+            type: 'image',
+            data: dataUrl.substring(commaIndex + 1),
+            mimeType: 'image/png',
+        };
+    }
+    catch {
+        return null;
+    }
+}
+// ============================================================================
+// selectElement - Agent-initiated picker
+// ============================================================================
+export async function selectElement(options) {
+    const { timeout = 60000, windowId, appIdentifier } = options;
+    const client = await ensureSessionAndConnect(appIdentifier);
+    // Generate unique picker ID
+    const pickerId = `picker_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
+    // Set up event listener FIRST (before injection to avoid race condition)
+    const eventPromise = new Promise((resolve, reject) => {
+        // eslint-disable-next-line prefer-const
+        let timeoutHandle;
+        const handler = (message) => {
+            if (message.type !== 'element_picked') {
+                return;
+            }
+            const payload = message.payload;
+            if (!payload || payload.pickerId !== pickerId) {
+                return;
+            }
+            clearTimeout(timeoutHandle);
+            client.removeListener('event', handler);
+            resolve(payload);
+        };
+        client.on('event', handler);
+        timeoutHandle = setTimeout(() => {
+            client.removeListener('event', handler);
+            // Clean up picker UI on timeout
+            cleanupPickerHighlights(windowId, appIdentifier);
+            reject(new Error(`Element picker timed out after ${timeout}ms. User did not select an element.`));
+        }, timeout);
+    });
+    // Inject picker overlay (this returns quickly within the 5s execute_js timeout)
+    const script = buildScript(SCRIPTS.elementPicker, { mode: 'pick', pickerId });
+    await executeInWebview(script, windowId, appIdentifier);
+    // Wait for user interaction
+    const result = await eventPromise;
+    // Handle cancellation
+    if (result.cancelled) {
+        return [{ type: 'text', text: 'Element picker was cancelled by the user.' }];
+    }
+    // Element was picked
+    const element = result.element;
+    if (!element) {
+        await cleanupPickerHighlights(windowId, appIdentifier);
+        return [{ type: 'text', text: 'Element picker returned no element data.' }];
+    }
+    // Clean up all picker UI BEFORE taking the screenshot
+    await cleanupPickerHighlights(windowId, appIdentifier);
+    const content = [];
+    // Add formatted metadata
+    content.push({ type: 'text', text: formatElementMetadata(element) });
+    // Capture element-only screenshot (no picker overlays visible)
+    const screenshot = await captureElementScreenshot(element.cssSelector, windowId);
+    if (screenshot) {
+        content.push(screenshot);
+    }
+    else {
+        content.push({ type: 'text', text: '(Element screenshot capture failed)' });
+    }
+    return content;
+}
+// ============================================================================
+// getPointedElement - Retrieve user-pointed element
+// ============================================================================
+export async function getPointedElement(options) {
+    const { windowId, appIdentifier } = options;
+    // Read and clear the pointed element
+    const readScript = `(function() {
+      var data = window.__MCP_POINTED_ELEMENT__;
+      window.__MCP_POINTED_ELEMENT__ = null;
+      return data ? JSON.stringify(data) : null;
+   })()`;
+    const raw = await executeInWebview(readScript, windowId, appIdentifier);
+    if (!raw || raw === 'null' || raw === 'undefined') {
+        return [
+            {
+                type: 'text',
+                text: 'No element has been pointed. Use Alt+Shift+Click on an element in the Tauri app first.',
+            },
+        ];
+    }
+    let element;
+    try {
+        element = JSON.parse(raw);
+    }
+    catch {
+        return [{ type: 'text', text: `Failed to parse pointed element data: ${raw.substring(0, 200)}` }];
+    }
+    const content = [];
+    // Add formatted metadata
+    content.push({ type: 'text', text: formatElementMetadata(element) });
+    // Capture element-only screenshot (no overlays)
+    const screenshot = await captureElementScreenshot(element.cssSelector, windowId);
+    if (screenshot) {
+        content.push(screenshot);
+    }
+    else {
+        content.push({ type: 'text', text: '(Element screenshot capture failed)' });
+    }
+    return content;
+}

package/dist/driver/scripts/element-picker.js

@@ -0,0 +1,395 @@
+/**
+ * Element picker overlay for MCP Server Tauri.
+ *
+ * Activated by the agent via webview_select_element. Displays a hover highlight,
+ * context tooltip, and cancel bar. On click (desktop) or two-tap (mobile) the
+ * selected element's metadata is emitted as a Tauri event so the MCP server can
+ * retrieve it asynchronously.
+ *
+ * @param {Object} params
+ * @param {string} params.mode - 'pick' (agent-initiated picker)
+ * @param {string} params.pickerId - Unique identifier for this picker session
+ */
+(function(params) {
+   var mode = params.mode;
+   var pickerId = params.pickerId;
+
+   // Duplicate-activation guard
+   if (window.__MCP_PICKER_ACTIVE__) {
+      // Cancel the previous picker
+      var prevId = window.__MCP_PICKER_ACTIVE__;
+      cleanup();
+      if (window.__TAURI__ && window.__TAURI__.event && window.__TAURI__.event.emit) {
+         window.__TAURI__.event.emit('__element_picked', { pickerId: prevId, cancelled: true });
+      }
+   }
+   window.__MCP_PICKER_ACTIVE__ = pickerId;
+
+   var isTouch = ('ontouchstart' in window) || (navigator.maxTouchPoints > 0);
+   var highlight = null;
+   var tooltip = null;
+   var cancelBar = null;
+   var trackedElement = null; // element currently being highlighted
+   var rafId = null; // requestAnimationFrame handle for cyclic repositioning
+
+   // ── Velocity-based hover throttling state ──────────────────────────────
+   var lastMouseX = 0;
+   var lastMouseY = 0;
+   var lastMouseTime = 0;
+   var mouseVelocity = 0;
+   var hoverUpdateTimer = null;
+
+   // ── Cancel bar ─────────────────────────────────────────────────────────
+   cancelBar = document.createElement('div');
+   cancelBar.setAttribute('data-mcp-picker', 'cancel-bar');
+   var cancelBarAtTop = true;
+   cancelBar.style.cssText =
+      'position:fixed;top:0;left:0;right:0;z-index:2147483647;height:40px;' +
+      'background:rgba(30,41,59,0.95);display:flex;align-items:center;' +
+      'justify-content:space-between;padding:0 12px;font:13px/1 system-ui,sans-serif;' +
+      'color:#E2E8F0;box-sizing:border-box;' +
+      'transition:top 0.2s ease,bottom 0.2s ease;';
+
+   var cancelText = document.createElement('span');
+   cancelText.textContent = 'MCP Element Picker \u2014 Click to select | ESC or tap X to cancel';
+   cancelText.style.cssText = 'overflow:hidden;text-overflow:ellipsis;white-space:nowrap;flex:1;';
+
+   // Move button — toggles bar between top and bottom
+   var moveBtn = document.createElement('button');
+   moveBtn.setAttribute('data-mcp-picker', 'move-btn');
+   moveBtn.textContent = '\u2193'; // ↓
+   moveBtn.title = 'Move bar to bottom';
+   moveBtn.style.cssText =
+      'background:none;border:none;color:#94A3B8;font-size:16px;cursor:pointer;' +
+      'width:32px;height:40px;display:flex;align-items:center;justify-content:center;' +
+      'flex-shrink:0;';
+   function toggleCancelBarPosition(e) {
+      e.stopPropagation();
+      e.preventDefault();
+      cancelBarAtTop = !cancelBarAtTop;
+      cancelBar.style.top = cancelBarAtTop ? '0' : 'auto';
+      cancelBar.style.bottom = cancelBarAtTop ? 'auto' : '0';
+      moveBtn.textContent = cancelBarAtTop ? '\u2193' : '\u2191'; // ↓ or ↑
+      moveBtn.title = cancelBarAtTop ? 'Move bar to bottom' : 'Move bar to top';
+   }
+   moveBtn.addEventListener('click', toggleCancelBarPosition);
+   moveBtn.addEventListener('touchend', toggleCancelBarPosition);
+
+   var cancelBtn = document.createElement('button');
+   cancelBtn.setAttribute('data-mcp-picker', 'cancel-btn');
+   cancelBtn.textContent = '\u2715';
+   cancelBtn.style.cssText =
+      'background:none;border:none;color:#E2E8F0;font-size:20px;cursor:pointer;' +
+      'width:40px;height:40px;display:flex;align-items:center;justify-content:center;' +
+      'flex-shrink:0;';
+   cancelBtn.addEventListener('click', function(e) {
+      e.stopPropagation();
+      cancelPicker();
+   });
+   cancelBtn.addEventListener('touchend', function(e) {
+      e.stopPropagation();
+      e.preventDefault();
+      cancelPicker();
+   });
+
+   cancelBar.appendChild(cancelText);
+   cancelBar.appendChild(moveBtn);
+   cancelBar.appendChild(cancelBtn);
+   document.body.appendChild(cancelBar);
+
+   // ── Highlight helpers ──────────────────────────────────────────────────
+   function createHighlight() {
+      var el = document.createElement('div');
+      el.setAttribute('data-mcp-picker', 'highlight');
+      el.style.cssText =
+         'position:fixed;z-index:2147483645;pointer-events:none;' +
+         'background:rgba(59,130,246,0.15);border:2px solid #3B82F6;';
+      document.body.appendChild(el);
+      return el;
+   }
+
+   function positionHighlight(el, target) {
+      var rect = target.getBoundingClientRect();
+      el.style.top = rect.top + 'px';
+      el.style.left = rect.left + 'px';
+      el.style.width = rect.width + 'px';
+      el.style.height = rect.height + 'px';
+   }
+
+   // ── Cyclic highlight repositioning via rAF ─────────────────────────────
+   // Keeps the highlight tracking animated or repositioned elements at ~60fps.
+   function startTracking() {
+      if (rafId) return;
+      function tick() {
+         if (trackedElement && highlight) {
+            positionHighlight(highlight, trackedElement);
+         }
+         rafId = requestAnimationFrame(tick);
+      }
+      rafId = requestAnimationFrame(tick);
+   }
+
+   function stopTracking() {
+      if (rafId) {
+         cancelAnimationFrame(rafId);
+         rafId = null;
+      }
+   }
+
+   // ── Tooltip helpers ────────────────────────────────────────────────────
+   function showTooltip(target) {
+      if (!tooltip) {
+         tooltip = document.createElement('div');
+         tooltip.setAttribute('data-mcp-picker', 'tooltip');
+         tooltip.style.cssText =
+            'position:fixed;z-index:2147483646;pointer-events:none;' +
+            'background:#1E293B;color:#E2E8F0;font:12px/1.4 monospace;' +
+            'padding:4px 8px;border-radius:4px;white-space:nowrap;max-width:300px;' +
+            'overflow:hidden;text-overflow:ellipsis;';
+         document.body.appendChild(tooltip);
+      }
+
+      var rect = target.getBoundingClientRect();
+      var tag = target.tagName.toLowerCase();
+      var id = target.id ? '#' + target.id : '';
+      var cls = target.className && typeof target.className === 'string'
+         ? '.' + target.className.trim().split(/\s+/).join('.')
+         : '';
+      var label = (tag + id + cls);
+      if (label.length > 60) label = label.substring(0, 57) + '...';
+      label += ' (' + Math.round(rect.width) + '\u00d7' + Math.round(rect.height) + ')';
+      tooltip.textContent = label;
+
+      // Position above or below the element
+      var tooltipTop = rect.top - 28;
+      if (tooltipTop < 44) { // below the cancel bar (40px + 4px padding)
+         tooltipTop = rect.bottom + 4;
+      }
+      tooltip.style.top = tooltipTop + 'px';
+      tooltip.style.left = Math.max(4, rect.left) + 'px';
+   }
+
+   function hideTooltip() {
+      if (tooltip && tooltip.parentNode) {
+         tooltip.parentNode.removeChild(tooltip);
+         tooltip = null;
+      }
+   }
+
+   // ── Element detection ──────────────────────────────────────────────────
+   // Uses the shared elementsFromPoint helper from bridge.js when available,
+   // falls back to document.elementFromPoint.
+   function findElementAt(x, y) {
+      if (window.__MCP_GET_ELEMENT_AT_POINT__) {
+         return window.__MCP_GET_ELEMENT_AT_POINT__(x, y);
+      }
+      // Fallback: hide highlight, use single elementFromPoint
+      if (highlight) highlight.style.display = 'none';
+      var el = document.elementFromPoint(x, y);
+      if (highlight) highlight.style.display = '';
+      return el;
+   }
+
+   // ── Picker element detection guard (fallback for findElementAt) ────────
+   function isPickerUI(el) {
+      while (el) {
+         if (el.getAttribute && el.getAttribute('data-mcp-picker')) {
+            return true;
+         }
+         el = el.parentElement;
+      }
+      return false;
+   }
+
+   // ── Selection ──────────────────────────────────────────────────────────
+   function selectElement(target) {
+      var metadata;
+      if (window.__MCP_COLLECT_ELEMENT_METADATA__) {
+         metadata = window.__MCP_COLLECT_ELEMENT_METADATA__(target);
+      } else {
+         metadata = { tag: target.tagName.toLowerCase(), cssSelector: '' };
+      }
+
+      // Store in window for later retrieval
+      window.__MCP_PICKED_ELEMENT__ = metadata;
+
+      // Remove overlay + tooltip but keep highlight for screenshot
+      removeCancelBar();
+      hideTooltip();
+      stopTracking();
+      removeListeners();
+
+      // Emit Tauri event
+      if (window.__TAURI__ && window.__TAURI__.event && window.__TAURI__.event.emit) {
+         window.__TAURI__.event.emit('__element_picked', { pickerId: pickerId, element: metadata });
+      }
+
+      window.__MCP_PICKER_ACTIVE__ = null;
+      return 'Element selected: ' + metadata.tag + (metadata.id ? '#' + metadata.id : '');
+   }
+
+   // ── Cancellation ──────────────────────────────────────────────────────
+   function cancelPicker() {
+      cleanup();
+      if (window.__TAURI__ && window.__TAURI__.event && window.__TAURI__.event.emit) {
+         window.__TAURI__.event.emit('__element_picked', { pickerId: pickerId, cancelled: true });
+      }
+      window.__MCP_PICKER_ACTIVE__ = null;
+   }
+
+   // ── Cleanup ────────────────────────────────────────────────────────────
+   function removeCancelBar() {
+      if (cancelBar && cancelBar.parentNode) {
+         cancelBar.parentNode.removeChild(cancelBar);
+         cancelBar = null;
+      }
+   }
+
+   function removeHighlight() {
+      if (highlight && highlight.parentNode) {
+         highlight.parentNode.removeChild(highlight);
+         highlight = null;
+      }
+   }
+
+   function cleanup() {
+      removeCancelBar();
+      removeHighlight();
+      hideTooltip();
+      stopTracking();
+      removeListeners();
+      trackedElement = null;
+      if (hoverUpdateTimer) {
+         clearTimeout(hoverUpdateTimer);
+         hoverUpdateTimer = null;
+      }
+      window.__MCP_PICKER_ACTIVE__ = null;
+   }
+
+   // ── Core hover-update logic (shared by throttled and immediate paths) ──
+   function updateHoveredElement() {
+      var el = findElementAt(lastMouseX, lastMouseY);
+
+      if (!el || isPickerUI(el)) return;
+
+      if (el === trackedElement) return; // no change
+      trackedElement = el;
+
+      if (!highlight) {
+         highlight = createHighlight();
+         startTracking();
+      }
+      positionHighlight(highlight, el);
+      showTooltip(el);
+   }
+
+   // ── Desktop: hover + click with velocity throttling ────────────────────
+   function onMouseMove(e) {
+      var now = performance.now();
+      var dx = e.clientX - lastMouseX;
+      var dy = e.clientY - lastMouseY;
+      var dt = now - lastMouseTime;
+      var distance = Math.sqrt(dx * dx + dy * dy);
+
+      lastMouseX = e.clientX;
+      lastMouseY = e.clientY;
+
+      // Calculate velocity in pixels per second
+      mouseVelocity = dt > 0 ? (distance / dt) * 1000 : 0;
+      lastMouseTime = now;
+
+      // If moving fast (>600 px/s), throttle updates to ~28fps
+      if (mouseVelocity > 600) {
+         if (hoverUpdateTimer) {
+            clearTimeout(hoverUpdateTimer);
+         }
+         hoverUpdateTimer = setTimeout(updateHoveredElement, 36); // ~28fps
+      } else {
+         if (hoverUpdateTimer) {
+            clearTimeout(hoverUpdateTimer);
+            hoverUpdateTimer = null;
+         }
+         updateHoveredElement();
+      }
+   }
+
+   function onClick(e) {
+      // Let clicks on picker UI (cancel, move buttons) pass through to their handlers
+      if (isPickerUI(e.target)) return;
+
+      e.preventDefault();
+      e.stopPropagation();
+
+      var el = findElementAt(e.clientX, e.clientY);
+
+      if (!el || isPickerUI(el)) return;
+
+      selectElement(el);
+   }
+
+   // ── Mobile: two-tap ────────────────────────────────────────────────────
+   var lastTapTarget = null;
+
+   function onTouchEnd(e) {
+      var touch = e.changedTouches[0];
+      if (!touch) return;
+
+      var el = findElementAt(touch.clientX, touch.clientY);
+
+      // Let taps on picker UI (cancel, move buttons) pass through to their handlers
+      if (!el || isPickerUI(el)) return;
+
+      e.preventDefault();
+      e.stopPropagation();
+
+      if (lastTapTarget === el) {
+         // Second tap on same element -> confirm
+         selectElement(el);
+      } else {
+         // First tap (or different element) -> highlight
+         lastTapTarget = el;
+         trackedElement = el;
+
+         if (!highlight) {
+            highlight = createHighlight();
+            startTracking();
+         }
+         positionHighlight(highlight, el);
+         showTooltip(el);
+
+         if (cancelText) {
+            cancelText.textContent = 'Tap element again to send | X Cancel';
+         }
+      }
+   }
+
+   // ── Keyboard escape ────────────────────────────────────────────────────
+   function onKeyDown(e) {
+      if (e.key === 'Escape') {
+         e.preventDefault();
+         e.stopPropagation();
+         cancelPicker();
+      }
+   }
+
+   // ── Listener management ────────────────────────────────────────────────
+   function removeListeners() {
+      document.removeEventListener('mousemove', onMouseMove, true);
+      document.removeEventListener('click', onClick, true);
+      document.removeEventListener('touchend', onTouchEnd, true);
+      document.removeEventListener('keydown', onKeyDown, true);
+   }
+
+   // Attach appropriate listeners based on device
+   document.addEventListener('keydown', onKeyDown, true);
+
+   if (isTouch) {
+      document.addEventListener('touchend', onTouchEnd, true);
+   }
+
+   // Always attach mouse listeners (hybrid devices like touch laptops)
+   document.addEventListener('mousemove', onMouseMove, true);
+   document.addEventListener('click', onClick, true);
+
+   return 'Picker activated (id: ' + pickerId + ')';
+})

package/dist/driver/scripts/index.js

@@ -22,6 +22,7 @@ export const SCRIPTS = {
     focus: loadScript('focus'),
     findElement: loadScript('find-element'),
     domSnapshot: loadScript('dom-snapshot'),
+    elementPicker: loadScript('element-picker'),
 };
 /** Script ID used for resolve-ref in the script registry. */
 export const RESOLVE_REF_SCRIPT_ID = '__mcp_resolve_ref__';

package/dist/prompts-registry.js

@@ -91,6 +91,28 @@ Once changes are approved and made:
 - The plugin only runs in debug builds so it won't affect production
 - The WebSocket server binds to \`0.0.0.0:9223\` by default
 - For localhost-only access, use \`Builder::new().bind_address("127.0.0.1").build()\``;
+const SELECT_ELEMENT_PROMPT = (message) => {
+    const lines = [
+        'The user wants to visually select an element in their running Tauri app so they can discuss it with you.',
+        '',
+        'Follow these steps:',
+        '',
+        '1. **Ensure a session is active** - Use `driver_session` with action "start" if not already connected',
+        '',
+        '2. **Activate the element picker** - Call `webview_select_element` to show the picker overlay in the app.',
+        'The user will see a blue highlight following their cursor and can click to select an element.',
+        'They can press Escape or click X to cancel.',
+        '',
+        '3. **Review the result** - You will receive the element\'s metadata (tag, id, classes, CSS selector, XPath,',
+        'bounding rect, attributes, computed styles, parent chain) and an annotated screenshot with the element highlighted.',
+        '',
+        '4. **Respond to the user** - Use the element context and screenshot to address their request.',
+    ];
+    if (message) {
+        lines.push('', '## User\'s Message About the Element', '', message);
+    }
+    return lines.join('\n');
+};
 /**
  * Complete registry of all available prompts
  */
@@ -114,6 +136,30 @@ export const PROMPTS = [
             ];
         },
     },
+    {
+        name: 'select',
+        description: 'Visually select an element in the running Tauri app. ' +
+            'Activates a picker overlay — click an element to send its metadata and an annotated screenshot to the agent. ' +
+            'Optionally include a message describing what you want to do with the element.',
+        arguments: [
+            {
+                name: 'message',
+                description: 'What you want to discuss or do with the selected element (e.g. "this button should be green instead of blue")',
+                required: false,
+            },
+        ],
+        handler: (args) => {
+            return [
+                {
+                    role: 'user',
+                    content: {
+                        type: 'text',
+                        text: SELECT_ELEMENT_PROMPT(args.message),
+                    },
+                },
+            ];
+        },
+    },
     {
         name: 'setup',
         description: 'Set up or update the MCP Bridge plugin in a Tauri project. ' +

package/dist/tools-registry.js

@@ -8,6 +8,7 @@ import { manageDriverSession, ManageDriverSessionSchema, } from './driver/sessio
 import { readLogs, ReadLogsSchema } from './monitor/logs.js';
 import { executeIPCCommand, manageIPCMonitoring, getIPCEvents, emitTestEvent, getBackendState, manageWindow, ExecuteIPCCommandSchema, ManageIPCMonitoringSchema, GetIPCEventsSchema, EmitTestEventSchema, GetBackendStateSchema, ManageWindowSchema, } from './driver/plugin-commands.js';
 import { interact, screenshot, keyboard, waitFor, getStyles, executeJavaScript, findElement, domSnapshot, InteractSchema, ScreenshotSchema, KeyboardSchema, WaitForSchema, GetStylesSchema, ExecuteJavaScriptSchema, FindElementSchema, DomSnapshotSchema, } from './driver/webview-interactions.js';
+import { selectElement, getPointedElement, SelectElementSchema, GetPointedElementSchema, } from './driver/element-picker.js';
 import { PLUGIN_VERSION_CARGO } from './version.js';
 /**
  * Standard multi-app description for webview tools.
@@ -409,6 +410,56 @@ export const TOOLS = [
             });
         },
     },
+    // Element Picker Tools
+    {
+        name: 'webview_select_element',
+        description: '[Tauri Apps Only] Activates an element picker overlay in the Tauri app. ' +
+            'The user visually selects an element by clicking it, and the tool returns ' +
+            'rich element metadata (tag, id, classes, attributes, text, bounding rect, ' +
+            'CSS selector, computed styles, parent chain) plus an annotated screenshot ' +
+            'with the element highlighted. ' +
+            'Requires active driver_session. ' +
+            MULTI_APP_DESC,
+        category: TOOL_CATEGORIES.UI_AUTOMATION,
+        schema: SelectElementSchema,
+        annotations: {
+            title: 'Select Element (Visual Picker)',
+            readOnlyHint: true,
+            openWorldHint: false,
+        },
+        handler: async (args) => {
+            const parsed = SelectElementSchema.parse(args);
+            return await selectElement({
+                timeout: parsed.timeout,
+                windowId: parsed.windowId,
+                appIdentifier: parsed.appIdentifier,
+            });
+        },
+    },
+    {
+        name: 'webview_get_pointed_element',
+        description: '[Tauri Apps Only] Retrieves element metadata for an element the user previously ' +
+            'pointed at via Alt+Shift+Click in the Tauri app. Returns the same rich metadata ' +
+            'as webview_select_element (tag, id, classes, attributes, text, bounding rect, ' +
+            'CSS selector, computed styles, parent chain) plus an annotated screenshot. ' +
+            'The user must Alt+Shift+Click an element first before calling this tool. ' +
+            'Requires active driver_session. ' +
+            MULTI_APP_DESC,
+        category: TOOL_CATEGORIES.UI_AUTOMATION,
+        schema: GetPointedElementSchema,
+        annotations: {
+            title: 'Get Pointed Element',
+            readOnlyHint: true,
+            openWorldHint: false,
+        },
+        handler: async (args) => {
+            const parsed = GetPointedElementSchema.parse(args);
+            return await getPointedElement({
+                windowId: parsed.windowId,
+                appIdentifier: parsed.appIdentifier,
+            });
+        },
+    },
     // IPC & Plugin Tools
     {
         name: 'ipc_execute_command',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hypothesi/tauri-mcp-server",
-  "version": "0.8.3",
+  "version": "0.9.0",
   "mcpName": "io.github.hypothesi/mcp-server-tauri",
   "description": "A Model Context Protocol server for use with Tauri v2 applications",
   "type": "module",