@hypothesi/tauri-mcp-server 0.8.3 → 0.10.0

package/README.md

@@ -11,7 +11,7 @@ A **Model Context Protocol (MCP) server** that enables AI assistants like Claude
 
 | Category | Capabilities |
 |----------|-------------|
-| 🎯 **UI Automation** | Screenshots, clicks, typing, scrolling, element finding |
+| 🎯 **UI Automation** | Screenshots, clicks, typing, scrolling, element finding, visual element picker |
 | 🔍 **IPC Monitoring** | Capture and inspect Tauri IPC calls in real-time |
 | 📱 **Mobile Dev** | List Android emulators & iOS simulators |
 | 📋 **Logs** | Stream console, Android logcat, iOS, and system logs |
@@ -52,6 +52,18 @@ npx -y install-mcp @hypothesi/tauri-mcp-server --client claude-code
 
 Supported clients: `claude-code`, `cursor`, `windsurf`, `vscode`, `cline`, `roo-cline`, `claude`, `zed`, `goose`, `warp`, `codex`
 
+## Terminal CLI
+
+If you want to call the same tools directly from a shell, install the companion CLI package:
+
+```bash
+npm install -g @hypothesi/tauri-mcp-cli
+tauri-mcp driver-session start --port 9223
+tauri-mcp driver-session status --json
+```
+
+The CLI keeps the underlying MCP server warm in the background so stateful commands such as `driver_session` work across separate invocations.
+
 ## Multi-App Support
 
 The MCP server supports connecting to multiple Tauri apps simultaneously. Each app runs on a unique port, and the most recently connected app becomes the "default" app.
@@ -86,7 +98,7 @@ await driver_session({ action: "stop", appIdentifier: 9223 })
 await driver_session({ action: "stop" })
 ```
 
-## Available Tools (18 total)
+## Available Tools (20 total)
 
 ### Setup & Configuration
 
@@ -108,6 +120,8 @@ await driver_session({ action: "stop" })
 | `webview_get_styles` | Get computed CSS styles |
 | `webview_execute_js` | Execute JavaScript in webview |
 | `webview_dom_snapshot` | Get structured DOM snapshot (accessibility or structure) |
+| `webview_select_element` | Visual element picker — user clicks an element, returns metadata + screenshot |
+| `webview_get_pointed_element` | Get metadata for element user Alt+Shift+Clicked |
 | `manage_window` | List windows, get info, or resize |
 
 ### IPC & Plugin

package/dist/api.d.ts

@@ -0,0 +1,2 @@
+export { TOOLS } from './tools-registry.js';
+export { createMcpServer, getCliToolDefinitions, startStdioServer } from './server.js';

package/dist/api.js

@@ -0,0 +1,2 @@
+export { TOOLS } from './tools-registry.js';
+export { createMcpServer, getCliToolDefinitions, startStdioServer } from './server.js';

package/dist/config.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Configuration for the MCP Bridge connection.
+ *
+ * This module provides configuration options for connecting to Tauri apps,
+ * with support for environment variables and sensible defaults.
+ */
+export interface BridgeConfig {
+    host: string;
+    port: number;
+}
+/**
+ * Gets the default host for MCP Bridge connections.
+ *
+ * Resolution priority:
+ * 1. MCP_BRIDGE_HOST environment variable
+ * 2. TAURI_DEV_HOST environment variable (set by Tauri CLI for mobile dev)
+ * 3. 'localhost' (default)
+ */
+export declare function getDefaultHost(): string;
+/**
+ * Gets the default port for MCP Bridge connections.
+ *
+ * Resolution priority:
+ * 1. MCP_BRIDGE_PORT environment variable
+ * 2. 9223 (default)
+ */
+export declare function getDefaultPort(): number;
+/**
+ * Gets the full bridge configuration from environment variables.
+ */
+export declare function getConfig(): BridgeConfig;
+/**
+ * Builds a WebSocket URL from host and port.
+ */
+export declare function buildWebSocketURL(host: string, port: number): string;

package/dist/driver/app-discovery.d.ts

@@ -0,0 +1,75 @@
+/**
+ * App discovery and session management for multiple Tauri instances.
+ *
+ * This module handles discovering and connecting to multiple Tauri apps
+ * running with MCP Bridge on the same machine or remote devices using port scanning.
+ */
+import { PluginClient } from './plugin-client.js';
+export interface AppInstance {
+    host: string;
+    port: number;
+    available: boolean;
+}
+export interface SessionInfo {
+    appId: string;
+    name: string;
+    host: string;
+    port: number;
+    client?: PluginClient;
+    connected: boolean;
+}
+/**
+ * Manages discovery and connection to multiple Tauri app instances
+ */
+export declare class AppDiscovery {
+    private _activeSessions;
+    private _host;
+    private _basePort;
+    private _maxPorts;
+    constructor(host?: string, basePort?: number);
+    /**
+     * Gets the configured host.
+     */
+    get host(): string;
+    /**
+     * Sets the host for discovery.
+     */
+    setHost(host: string): void;
+    /**
+     * Discovers available Tauri app instances by scanning ports
+     */
+    discoverApps(): Promise<AppInstance[]>;
+    /**
+     * Connects to a specific app on a host and port
+     */
+    connectToPort(port: number, appName?: string, host?: string): Promise<SessionInfo>;
+    /**
+     * Gets the first available app
+     */
+    getFirstAvailableApp(): Promise<AppInstance | null>;
+    /**
+     * Disconnects from a specific session
+     */
+    disconnectSession(sessionId: string): Promise<void>;
+    /**
+     * Disconnects from all apps
+     */
+    disconnectAll(): Promise<void>;
+    /**
+     * Gets the active session by ID
+     */
+    getSession(sessionId: string): SessionInfo | undefined;
+    /**
+     * Gets all active sessions
+     */
+    getAllSessions(): SessionInfo[];
+    /**
+     * Try to connect to the default port
+     */
+    connectToDefaultPort(): Promise<SessionInfo>;
+    /**
+     * Check if a port is in use (likely a Tauri app)
+     */
+    private _isPortInUse;
+}
+export declare const appDiscovery: AppDiscovery;

package/dist/driver/element-picker.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Element picker module for MCP Server Tauri.
+ *
+ * Provides two tools:
+ * - selectElement: Agent-initiated picker overlay (user clicks element)
+ * - getPointedElement: Retrieve element user pointed at via Alt+Shift+Click
+ */
+import { z } from 'zod';
+import type { ToolContent } from '../tools-registry.js';
+export declare const SelectElementSchema: z.ZodObject<{
+    windowId: z.ZodOptional<z.ZodString>;
+    appIdentifier: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
+} & {
+    timeout: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+}, "strip", z.ZodTypeAny, {
+    timeout: number;
+    windowId?: string | undefined;
+    appIdentifier?: string | number | undefined;
+}, {
+    windowId?: string | undefined;
+    appIdentifier?: string | number | undefined;
+    timeout?: number | undefined;
+}>;
+export declare const GetPointedElementSchema: z.ZodObject<{
+    windowId: z.ZodOptional<z.ZodString>;
+    appIdentifier: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
+}, "strip", z.ZodTypeAny, {
+    windowId?: string | undefined;
+    appIdentifier?: string | number | undefined;
+}, {
+    windowId?: string | undefined;
+    appIdentifier?: string | number | undefined;
+}>;
+export declare function selectElement(options: {
+    timeout?: number;
+    windowId?: string;
+    appIdentifier?: string | number;
+}): Promise<ToolContent[]>;
+export declare function getPointedElement(options: {
+    windowId?: string;
+    appIdentifier?: string | number;
+}): Promise<ToolContent[]>;

package/dist/driver/element-picker.js

@@ -0,0 +1,272 @@
+/**
+ * Element picker module for MCP Server Tauri.
+ *
+ * Provides two tools:
+ * - selectElement: Agent-initiated picker overlay (user clicks element)
+ * - getPointedElement: Retrieve element user pointed at via Alt+Shift+Click
+ */
+import { z } from 'zod';
+import { executeInWebview, executeAsyncInWebview } from './webview-executor.js';
+import { ensureSessionAndConnect } from './plugin-client.js';
+import { SCRIPTS, buildScript } from './scripts/index.js';
+import { WindowTargetSchema } from './webview-interactions.js';
+import { getHtml2CanvasSource, HTML2CANVAS_SCRIPT_ID, } from './scripts/html2canvas-loader.js';
+import { registerScript, isScriptRegistered } from './script-manager.js';
+// ============================================================================
+// Schemas
+// ============================================================================
+export const SelectElementSchema = WindowTargetSchema.extend({
+    timeout: z.number().min(5000).max(120000).optional().default(60000)
+        .describe('Timeout in ms for user to pick an element (5000-120000, default 60000)'),
+});
+export const GetPointedElementSchema = WindowTargetSchema.extend({});
+// ============================================================================
+// Helpers
+// ============================================================================
+/**
+ * Format element metadata into a readable text block.
+ */
+function formatElementMetadata(meta) {
+    const lines = [];
+    lines.push(`## Element: <${meta.tag}>`);
+    if (meta.id) {
+        lines.push(`**ID:** ${meta.id}`);
+    }
+    if (meta.classes.length > 0) {
+        lines.push(`**Classes:** ${meta.classes.join(', ')}`);
+    }
+    lines.push(`**CSS Selector:** \`${meta.cssSelector}\``);
+    if (meta.xpath) {
+        lines.push(`**XPath:** \`${meta.xpath}\``);
+    }
+    // Bounding rect
+    const r = meta.boundingRect;
+    lines.push(`**Bounding Rect:** ${Math.round(r.width)}x${Math.round(r.height)} at (${Math.round(r.x)}, ${Math.round(r.y)})`);
+    // Attributes (skip id and class which are already shown)
+    const attrEntries = Object.entries(meta.attributes).filter(([k]) => { return k !== 'id' && k !== 'class'; });
+    if (attrEntries.length > 0) {
+        lines.push(`**Attributes:** ${attrEntries.map(([k, v]) => { return `${k}="${v}"`; }).join(', ')}`);
+    }
+    if (meta.textContent) {
+        const text = meta.textContent.length > 200
+            ? meta.textContent.substring(0, 200) + '...'
+            : meta.textContent;
+        lines.push(`**Text Content:** ${text}`);
+    }
+    // Computed styles (only non-default interesting ones)
+    const styleEntries = Object.entries(meta.computedStyles);
+    if (styleEntries.length > 0) {
+        lines.push('**Computed Styles:**');
+        for (const [prop, val] of styleEntries) {
+            lines.push(`  ${prop}: ${val}`);
+        }
+    }
+    if (meta.parentChain.length > 0) {
+        lines.push('**Parent Chain:**');
+        for (const parent of meta.parentChain) {
+            let desc = `  <${parent.tag}>`;
+            if (parent.id) {
+                desc += `#${parent.id}`;
+            }
+            if (parent.classes && parent.classes.length > 0) {
+                desc += `.${parent.classes.join('.')}`;
+            }
+            if (parent.boundingRect) {
+                desc += ` (${Math.round(parent.boundingRect.width)}x${Math.round(parent.boundingRect.height)})`;
+            }
+            lines.push(desc);
+        }
+    }
+    return lines.join('\n');
+}
+/**
+ * Inject a script that removes all picker highlight elements from the DOM.
+ */
+async function cleanupPickerHighlights(windowId, appIdentifier) {
+    const script = `(function() {
+      var els = document.querySelectorAll('[data-mcp-picker]');
+      for (var i = 0; i < els.length; i++) { els[i].parentNode.removeChild(els[i]); }
+      return 'Cleaned up ' + els.length + ' picker elements';
+   })()`;
+    try {
+        await executeInWebview(script, windowId, appIdentifier);
+    }
+    catch {
+        // Best effort cleanup
+    }
+}
+/**
+ * Capture a screenshot of a specific element using html2canvas.
+ * Returns the base64 data URL of the cropped element image, or null on failure.
+ */
+async function captureElementScreenshot(cssSelector, windowId) {
+    // Ensure html2canvas is loaded in the webview
+    try {
+        const isRegistered = await isScriptRegistered(HTML2CANVAS_SCRIPT_ID);
+        if (!isRegistered) {
+            const source = getHtml2CanvasSource();
+            await registerScript(HTML2CANVAS_SCRIPT_ID, 'inline', source);
+        }
+    }
+    catch {
+        // Script manager unavailable — we'll inline the library in the capture script
+    }
+    const escapedSelector = cssSelector.replace(/\\/g, '\\\\').replace(/'/g, '\\\'');
+    // Build a script that captures just the element with html2canvas
+    const captureScript = `
+      const html2canvasFn = typeof html2canvas !== 'undefined' ? html2canvas :
+                           (typeof window !== 'undefined' && window.html2canvas) ? window.html2canvas :
+                           (typeof self !== 'undefined' && self.html2canvas) ? self.html2canvas :
+                           (typeof globalThis !== 'undefined' && globalThis.html2canvas) ? globalThis.html2canvas : null;
+
+      if (!html2canvasFn) {
+         throw new Error('html2canvas not loaded');
+      }
+
+      const el = document.querySelector('${escapedSelector}');
+      if (!el) {
+         throw new Error('Element not found for screenshot');
+      }
+
+      const canvas = await html2canvasFn(el, {
+         backgroundColor: null,
+         scale: window.devicePixelRatio || 1,
+         logging: false,
+         useCORS: true,
+         allowTaint: false,
+         imageTimeout: 5000,
+      });
+
+      if (!canvas) {
+         throw new Error('html2canvas returned null canvas');
+      }
+
+      const dataUrl = canvas.toDataURL('image/png');
+      if (!dataUrl || !dataUrl.startsWith('data:image/')) {
+         throw new Error('Invalid data URL from canvas');
+      }
+
+      return dataUrl;
+   `;
+    try {
+        const dataUrl = await executeAsyncInWebview(captureScript, windowId, 10000);
+        if (!dataUrl || !dataUrl.startsWith('data:image/')) {
+            return null;
+        }
+        // Extract base64 data from data URL
+        const commaIndex = dataUrl.indexOf(',');
+        if (commaIndex === -1) {
+            return null;
+        }
+        return {
+            type: 'image',
+            data: dataUrl.substring(commaIndex + 1),
+            mimeType: 'image/png',
+        };
+    }
+    catch {
+        return null;
+    }
+}
+// ============================================================================
+// selectElement - Agent-initiated picker
+// ============================================================================
+export async function selectElement(options) {
+    const { timeout = 60000, windowId, appIdentifier } = options;
+    const client = await ensureSessionAndConnect(appIdentifier);
+    // Generate unique picker ID
+    const pickerId = `picker_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
+    // Set up event listener FIRST (before injection to avoid race condition)
+    const eventPromise = new Promise((resolve, reject) => {
+        // eslint-disable-next-line prefer-const
+        let timeoutHandle;
+        const handler = (message) => {
+            if (message.type !== 'element_picked') {
+                return;
+            }
+            const payload = message.payload;
+            if (!payload || payload.pickerId !== pickerId) {
+                return;
+            }
+            clearTimeout(timeoutHandle);
+            client.removeListener('event', handler);
+            resolve(payload);
+        };
+        client.on('event', handler);
+        timeoutHandle = setTimeout(() => {
+            client.removeListener('event', handler);
+            // Clean up picker UI on timeout
+            cleanupPickerHighlights(windowId, appIdentifier);
+            reject(new Error(`Element picker timed out after ${timeout}ms. User did not select an element.`));
+        }, timeout);
+    });
+    // Inject picker overlay (this returns quickly within the 5s execute_js timeout)
+    const script = buildScript(SCRIPTS.elementPicker, { mode: 'pick', pickerId });
+    await executeInWebview(script, windowId, appIdentifier);
+    // Wait for user interaction
+    const result = await eventPromise;
+    // Handle cancellation
+    if (result.cancelled) {
+        return [{ type: 'text', text: 'Element picker was cancelled by the user.' }];
+    }
+    // Element was picked
+    const element = result.element;
+    if (!element) {
+        await cleanupPickerHighlights(windowId, appIdentifier);
+        return [{ type: 'text', text: 'Element picker returned no element data.' }];
+    }
+    // Clean up all picker UI BEFORE taking the screenshot
+    await cleanupPickerHighlights(windowId, appIdentifier);
+    const content = [];
+    // Add formatted metadata
+    content.push({ type: 'text', text: formatElementMetadata(element) });
+    // Capture element-only screenshot (no picker overlays visible)
+    const screenshot = await captureElementScreenshot(element.cssSelector, windowId);
+    if (screenshot) {
+        content.push(screenshot);
+    }
+    else {
+        content.push({ type: 'text', text: '(Element screenshot capture failed)' });
+    }
+    return content;
+}
+// ============================================================================
+// getPointedElement - Retrieve user-pointed element
+// ============================================================================
+export async function getPointedElement(options) {
+    const { windowId, appIdentifier } = options;
+    // Read and clear the pointed element
+    const readScript = `(function() {
+      var data = window.__MCP_POINTED_ELEMENT__;
+      window.__MCP_POINTED_ELEMENT__ = null;
+      return data ? JSON.stringify(data) : null;
+   })()`;
+    const raw = await executeInWebview(readScript, windowId, appIdentifier);
+    if (!raw || raw === 'null' || raw === 'undefined') {
+        return [
+            {
+                type: 'text',
+                text: 'No element has been pointed. Use Alt+Shift+Click on an element in the Tauri app first.',
+            },
+        ];
+    }
+    let element;
+    try {
+        element = JSON.parse(raw);
+    }
+    catch {
+        return [{ type: 'text', text: `Failed to parse pointed element data: ${raw.substring(0, 200)}` }];
+    }
+    const content = [];
+    // Add formatted metadata
+    content.push({ type: 'text', text: formatElementMetadata(element) });
+    // Capture element-only screenshot (no overlays)
+    const screenshot = await captureElementScreenshot(element.cssSelector, windowId);
+    if (screenshot) {
+        content.push(screenshot);
+    }
+    else {
+        content.push({ type: 'text', text: '(Element screenshot capture failed)' });
+    }
+    return content;
+}

package/dist/driver/plugin-client.d.ts

@@ -0,0 +1,100 @@
+import { EventEmitter } from 'events';
+interface PluginCommand {
+    id?: string;
+    command: string;
+    args?: unknown;
+}
+export interface PluginResponse {
+    id?: string;
+    success: boolean;
+    data?: unknown;
+    error?: string;
+    windowContext?: {
+        windowLabel: string;
+        totalWindows: number;
+        warning?: string;
+    };
+}
+/**
+ * Client to communicate with the MCP Bridge plugin's WebSocket server
+ */
+export declare class PluginClient extends EventEmitter {
+    private _ws;
+    private _url;
+    private _host;
+    private _port;
+    private _reconnectAttempts;
+    private _shouldReconnect;
+    private _reconnectDelay;
+    private _pendingRequests;
+    /**
+     * Constructor for PluginClient
+     * @param host Host address of the WebSocket server
+     * @param port Port number of the WebSocket server
+     */
+    constructor(host: string, port: number);
+    /**
+     * Creates a PluginClient with default configuration from environment.
+     */
+    static create_default(): PluginClient;
+    /**
+     * Gets the host this client is configured to connect to.
+     */
+    get host(): string;
+    /**
+     * Gets the port this client is configured to connect to.
+     */
+    get port(): number;
+    /**
+     * Connect to the plugin's WebSocket server
+     */
+    connect(): Promise<void>;
+    /**
+     * Disconnect from the plugin
+     */
+    disconnect(): void;
+    /**
+     * Send a command to the plugin and wait for response.
+     *
+     * Automatically retries on transient "not found" errors (e.g. window not
+     * yet registered after WebSocket connect) with exponential backoff.
+     */
+    sendCommand(command: PluginCommand, timeoutMs?: number): Promise<PluginResponse>;
+    /**
+     * Check if connected
+     */
+    isConnected(): boolean;
+}
+/**
+ * Gets the existing singleton PluginClient without creating or modifying it.
+ * Use this for status checks where you don't want to affect the current connection.
+ *
+ * @returns The existing PluginClient or null if none exists
+ */
+export declare function getExistingPluginClient(): PluginClient | null;
+/**
+ * Gets or creates a singleton PluginClient.
+ *
+ * If host/port are provided and differ from the existing client's configuration,
+ * the existing client is disconnected and a new one is created. This ensures
+ * that session start with a specific port always uses that port.
+ *
+ * @param host Optional host override
+ * @param port Optional port override
+ */
+export declare function getPluginClient(host?: string, port?: number): PluginClient;
+/**
+ * Resets the singleton client (useful for reconnecting with different config).
+ */
+export declare function resetPluginClient(): void;
+export declare function connectPlugin(host?: string, port?: number): Promise<void>;
+/**
+ * Ensures a session is active and connects to the plugin using session config.
+ * This should be used by all tools that require a connected Tauri app.
+ *
+ * @param appIdentifier - Optional app identifier to target specific app
+ * @throws Error if no session is active
+ */
+export declare function ensureSessionAndConnect(appIdentifier?: string | number): Promise<PluginClient>;
+export declare function disconnectPlugin(): Promise<void>;
+export {};