@wong2kim/wmux 1.1.1 → 2.0.0

package/dist/mcp/mcp/playwright/tools/inspection.js

@@ -0,0 +1,459 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerInspectionTools = registerInspectionTools;
+const zod_1 = require("zod");
+const PlaywrightEngine_1 = require("../PlaywrightEngine");
+const snapshot_1 = require("../snapshot");
+const anti_detection_1 = require("../anti-detection");
+const security_1 = require("../security");
+const wmux_client_1 = require("../../wmux-client");
+// Optional surfaceId schema reused across tools
+const optionalSurfaceId = zod_1.z
+    .string()
+    .optional()
+    .describe('Target a specific surface by ID. Omit to use the active surface.');
+const consoleMessages = new Map();
+const networkRequests = new Map();
+// Track which pages already have listeners attached
+const attachedConsolePages = new WeakSet();
+const attachedNetworkPages = new WeakSet();
+function ensureConsoleListener(page, surfaceKey) {
+    if (attachedConsolePages.has(page))
+        return;
+    attachedConsolePages.add(page);
+    if (!consoleMessages.has(surfaceKey)) {
+        consoleMessages.set(surfaceKey, []);
+    }
+    page.on('console', (msg) => {
+        const entries = consoleMessages.get(surfaceKey);
+        if (entries) {
+            entries.push({ level: msg.type(), text: msg.text() });
+        }
+    });
+}
+function ensureNetworkListener(page, surfaceKey) {
+    if (attachedNetworkPages.has(page))
+        return;
+    attachedNetworkPages.add(page);
+    if (!networkRequests.has(surfaceKey)) {
+        networkRequests.set(surfaceKey, []);
+    }
+    page.on('request', (request) => {
+        const entries = networkRequests.get(surfaceKey);
+        if (entries) {
+            entries.push({
+                url: request.url(),
+                method: request.method(),
+            });
+        }
+    });
+    page.on('response', (response) => {
+        const entries = networkRequests.get(surfaceKey);
+        if (!entries)
+            return;
+        const url = response.url();
+        // Find the matching request entry (last one with same URL and no status yet)
+        for (let i = entries.length - 1; i >= 0; i--) {
+            if (entries[i].url === url && entries[i].status === undefined) {
+                entries[i].status = response.status();
+                // Store response headers for later body retrieval
+                const headers = response.headers();
+                entries[i].response = { headers };
+                // Only eagerly capture body for text-based content types
+                const contentType = headers['content-type'] ?? '';
+                const isTextual = contentType.startsWith('text/') ||
+                    contentType.includes('application/json') ||
+                    contentType.includes('application/xml') ||
+                    contentType.includes('application/xhtml') ||
+                    contentType.includes('+json') ||
+                    contentType.includes('+xml');
+                if (isTextual) {
+                    response
+                        .text()
+                        .then((body) => {
+                        if (entries[i].response) {
+                            entries[i].response.body = body;
+                        }
+                    })
+                        .catch(() => {
+                        // Body may not be available for all responses
+                    });
+                }
+                break;
+            }
+        }
+    });
+}
+function surfaceKey(surfaceId) {
+    return surfaceId ?? '__default__';
+}
+/**
+ * Simple glob-like URL matching.
+ * Supports '*' as wildcard for any sequence of characters.
+ */
+function matchesGlob(url, pattern) {
+    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+    const regex = new RegExp('^' + escaped.replace(/\*/g, '.*') + '$', 'i');
+    return regex.test(url);
+}
+// ---------------------------------------------------------------------------
+// Registration
+// ---------------------------------------------------------------------------
+/**
+ * Register inspection-related MCP tools on the given server.
+ *
+ * Tools:
+ *  - browser_snapshot       -- accessibility tree snapshot
+ *  - browser_screenshot     -- page or element screenshot
+ *  - browser_evaluate       -- evaluate JS expression
+ *  - browser_console        -- retrieve console messages
+ *  - browser_network        -- retrieve network requests
+ *  - browser_response_body  -- retrieve response body by URL pattern
+ *  - browser_highlight      -- visually highlight an element
+ */
+function registerInspectionTools(server) {
+    const engine = PlaywrightEngine_1.PlaywrightEngine.getInstance();
+    // -----------------------------------------------------------------------
+    // browser_snapshot
+    // -----------------------------------------------------------------------
+    server.tool('browser_snapshot', 'Take an accessibility tree snapshot of the current page. Returns a text representation of the page structure with interactive elements annotated with ref numbers.', {
+        format: zod_1.z
+            .enum(['ai', 'aria'])
+            .optional()
+            .describe('Snapshot format. "ai" annotates interactive elements with ref numbers (default). "aria" returns the full tree.'),
+        ref: zod_1.z
+            .string()
+            .optional()
+            .describe('Reserved for future use: ref number to scope the snapshot to a subtree.'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ format, surfaceId }) => {
+        try {
+            // Try Playwright for full snapshot
+            const page = await engine.getPage(surfaceId).catch(() => null);
+            if (page) {
+                const snapshot = await (0, snapshot_1.generateSnapshot)(page, { format: format ?? 'ai' });
+                return {
+                    content: [{ type: 'text', text: snapshot }],
+                };
+            }
+            // Fallback: extract page structure via RPC evaluation
+            // Tags interactive elements with data-wmux-ref so interaction tools can resolve them
+            const result = await (0, wmux_client_1.sendRpc)('browser.evaluate', {
+                expression: `(() => {
+            const sel = 'a[href], button, input:not([type="hidden"]), textarea, select, [role="button"], [role="link"], [role="textbox"], [role="checkbox"], [role="radio"], [role="combobox"], [role="searchbox"], [role="tab"], [contenteditable="true"]';
+            const interactives = [...document.querySelectorAll(sel)].slice(0, 100);
+            interactives.forEach((el, i) => el.setAttribute('data-wmux-ref', String(i)));
+            const title = document.title;
+            const url = location.href;
+            const lines = ['Page: ' + title, 'URL: ' + url, ''];
+            document.querySelectorAll('h1,h2,h3').forEach(h => {
+              lines.push(h.tagName + ': ' + (h.textContent || '').trim().substring(0, 80));
+            });
+            lines.push('', 'Interactive elements (use ref number for click/fill/type):');
+            interactives.forEach((el, i) => {
+              const tag = el.tagName.toLowerCase();
+              const role = el.getAttribute('role') || '';
+              const text = (el.textContent || '').trim().substring(0, 60);
+              const label = el.getAttribute('aria-label') || '';
+              const name = el.getAttribute('name') || '';
+              const type = el.getAttribute('type') || '';
+              const placeholder = el.getAttribute('placeholder') || '';
+              const href = el.getAttribute('href') || '';
+              let desc = '  [ref=' + i + '] ' + tag;
+              if (type) desc += '[type=' + type + ']';
+              if (role) desc += '[role=' + role + ']';
+              if (name) desc += ' name="' + name + '"';
+              if (label) desc += ' "' + label + '"';
+              else if (text) desc += ' "' + text + '"';
+              if (placeholder) desc += ' placeholder="' + placeholder + '"';
+              if (href) desc += ' -> ' + href.substring(0, 60);
+              lines.push(desc);
+            });
+            return lines.join('\\n');
+          })()`,
+                ...(surfaceId && { surfaceId }),
+            });
+            return {
+                content: [{ type: 'text', text: result.value }],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+    // -----------------------------------------------------------------------
+    // browser_screenshot
+    // -----------------------------------------------------------------------
+    server.tool('browser_screenshot', 'Take a screenshot of the current page or a specific element. Returns the image as base64-encoded PNG. Requires browser_open to be called first to establish a connection, even if a browser panel is already visible.', {
+        fullPage: zod_1.z
+            .boolean()
+            .optional()
+            .describe('Capture the full scrollable page instead of just the viewport (default false).'),
+        ref: zod_1.z
+            .string()
+            .optional()
+            .describe('Ref number of an element to screenshot (from browser_snapshot). Omit for full page.'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ fullPage, ref, surfaceId }) => {
+        try {
+            // Try Playwright for element-level screenshots (ref)
+            if (ref) {
+                const page = await engine.getPage(surfaceId);
+                if (page) {
+                    const el = await (0, snapshot_1.resolveRef)(page, ref);
+                    if (!el) {
+                        throw new Error(`Could not resolve ref="${ref}" to an element.`);
+                    }
+                    const buffer = (await el.screenshot());
+                    return {
+                        content: [{ type: 'image', data: buffer.toString('base64'), mimeType: 'image/png' }],
+                    };
+                }
+            }
+            // Use RPC for fast, reliable screenshots (bypasses Playwright CDP discovery)
+            const result = await (0, wmux_client_1.sendRpc)('browser.screenshot', {
+                ...(surfaceId && { surfaceId }),
+                ...(fullPage && { fullPage }),
+            });
+            return {
+                content: [
+                    {
+                        type: 'image',
+                        data: result.data,
+                        mimeType: 'image/png',
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+    // -----------------------------------------------------------------------
+    // browser_evaluate
+    // -----------------------------------------------------------------------
+    server.tool('browser_evaluate', 'Evaluate a JavaScript expression in the browser page context. Uses userGesture mode for actions requiring user activation.', {
+        expression: zod_1.z.string().describe('The JavaScript expression to evaluate.'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ expression, surfaceId }) => {
+        try {
+            const warnings = (0, security_1.detectDangerousPatterns)(expression);
+            if (warnings.length > 0) {
+                console.warn(`[browser_evaluate] Dangerous patterns detected: ${warnings.join(', ')}`);
+            }
+            let result;
+            // Try Playwright first for gesture-aware evaluation
+            const page = await engine.getPage(surfaceId).catch(() => null);
+            if (page) {
+                result = await (0, anti_detection_1.evaluateWithGesture)(page, expression);
+            }
+            else {
+                // Fallback: RPC evaluation via main process webContents
+                const rpcResult = await (0, wmux_client_1.sendRpc)('browser.evaluate', {
+                    expression,
+                    ...(surfaceId && { surfaceId }),
+                });
+                result = rpcResult.value;
+            }
+            const text = typeof result === 'string' ? result : (JSON.stringify(result, null, 2) ?? 'undefined');
+            if (warnings.length > 0) {
+                const warningText = `\u26A0 Security warning: expression contains potentially dangerous patterns: ${warnings.join(', ')}. Exercise caution with untrusted input.\n\n`;
+                return {
+                    content: [{ type: 'text', text: warningText + text }],
+                };
+            }
+            return {
+                content: [{ type: 'text', text: text ?? 'undefined' }],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+    // -----------------------------------------------------------------------
+    // browser_console
+    // -----------------------------------------------------------------------
+    server.tool('browser_console', 'Retrieve console messages collected from the browser page. Messages are accumulated over time; use clear=true to reset.', {
+        level: zod_1.z
+            .enum(['error', 'warn', 'info', 'all'])
+            .optional()
+            .describe('Filter by message level. Defaults to "all".'),
+        clear: zod_1.z
+            .boolean()
+            .optional()
+            .describe('Clear collected messages after returning them.'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ level, clear, surfaceId }) => {
+        try {
+            const page = await engine.getPage(surfaceId);
+            if (!page) {
+                throw new Error('No browser page available. Call browser_open with a URL first to establish a CDP connection (required even if a browser panel is already visible).');
+            }
+            const key = surfaceKey(surfaceId);
+            ensureConsoleListener(page, key);
+            const entries = consoleMessages.get(key) ?? [];
+            const filterLevel = level ?? 'all';
+            const filtered = filterLevel === 'all'
+                ? entries
+                : entries.filter((e) => {
+                    if (filterLevel === 'info') {
+                        return e.level === 'log' || e.level === 'info';
+                    }
+                    return e.level === filterLevel;
+                });
+            const text = filtered.length === 0
+                ? 'No console messages collected.'
+                : filtered.map((e) => `[${e.level}] ${e.text}`).join('\n');
+            if (clear) {
+                consoleMessages.set(key, []);
+            }
+            return {
+                content: [{ type: 'text', text }],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+    // -----------------------------------------------------------------------
+    // browser_network
+    // -----------------------------------------------------------------------
+    server.tool('browser_network', 'Retrieve network requests collected from the browser page. Requests are accumulated over time. Use a URL glob pattern to filter.', {
+        filter: zod_1.z
+            .string()
+            .optional()
+            .describe('URL glob pattern to filter requests (e.g. "*api*", "*.json").'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ filter, surfaceId }) => {
+        try {
+            const page = await engine.getPage(surfaceId);
+            if (!page) {
+                throw new Error('No browser page available. Call browser_open with a URL first to establish a CDP connection (required even if a browser panel is already visible).');
+            }
+            const key = surfaceKey(surfaceId);
+            ensureNetworkListener(page, key);
+            const entries = networkRequests.get(key) ?? [];
+            const filtered = filter
+                ? entries.filter((e) => matchesGlob(e.url, filter))
+                : entries;
+            const summary = filtered.map((e) => ({
+                url: e.url,
+                method: e.method,
+                status: e.status ?? '(pending)',
+            }));
+            const text = summary.length === 0
+                ? 'No network requests collected.'
+                : JSON.stringify(summary, null, 2);
+            return {
+                content: [{ type: 'text', text }],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+    // -----------------------------------------------------------------------
+    // browser_response_body
+    // -----------------------------------------------------------------------
+    server.tool('browser_response_body', 'Retrieve the response body for a previously captured network request matching a URL pattern.', {
+        urlPattern: zod_1.z
+            .string()
+            .describe('URL glob pattern to match (e.g. "*api/users*").'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ urlPattern, surfaceId }) => {
+        try {
+            const page = await engine.getPage(surfaceId);
+            if (!page) {
+                throw new Error('No browser page available. Call browser_open with a URL first to establish a CDP connection (required even if a browser panel is already visible).');
+            }
+            const key = surfaceKey(surfaceId);
+            ensureNetworkListener(page, key);
+            const entries = networkRequests.get(key) ?? [];
+            // Find the last matching entry with a captured body
+            let matchedEntry;
+            for (let i = entries.length - 1; i >= 0; i--) {
+                if (matchesGlob(entries[i].url, urlPattern) && entries[i].response?.body !== undefined) {
+                    matchedEntry = entries[i];
+                    break;
+                }
+            }
+            if (!matchedEntry || !matchedEntry.response?.body) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `No response body found for pattern "${urlPattern}". Ensure the request has been made and the response was captured.`,
+                        },
+                    ],
+                };
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: matchedEntry.response.body,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+    // -----------------------------------------------------------------------
+    // browser_highlight
+    // -----------------------------------------------------------------------
+    server.tool('browser_highlight', 'Visually highlight an element on the page by its ref number. Adds a red outline around the element.', {
+        ref: zod_1.z.string().describe('Ref number of the element to highlight (from browser_snapshot).'),
+        surfaceId: optionalSurfaceId,
+    }, async ({ ref, surfaceId }) => {
+        try {
+            const page = await engine.getPage(surfaceId);
+            if (!page) {
+                throw new Error('No browser page available. Call browser_open with a URL first to establish a CDP connection (required even if a browser panel is already visible).');
+            }
+            const el = await (0, snapshot_1.resolveRef)(page, ref);
+            if (!el) {
+                throw new Error(`Could not resolve ref="${ref}" to an element.`);
+            }
+            await el.evaluate((element) => {
+                element.style.outline = '3px solid red';
+                element.style.outlineOffset = '2px';
+            });
+            return {
+                content: [{ type: 'text', text: 'Element highlighted' }],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true,
+            };
+        }
+    });
+}