@mseep/clawdcursor 1.5.5

package/dist/tools/desktop.js

@@ -0,0 +1,346 @@
+"use strict";
+/**
+ * Desktop tools — screenshot, mouse, keyboard, screen info.
+ *
+ * Coordinate system: All mouse tools accept IMAGE-SPACE coordinates
+ * (matching the 1280px-wide screenshots from desktop_screenshot).
+ * The server auto-scales to Windows LOGICAL coordinates via mouseScaleFactor.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDesktopTools = getDesktopTools;
+const os = __importStar(require("os"));
+/** Dangerous key combos that are blocked */
+const BLOCKED_KEYS = ['alt+f4', 'ctrl+alt+delete', 'ctrl+alt+del'];
+const IS_MAC = os.platform() === 'darwin';
+/**
+ * Best-effort active-window label for a tool's result text.
+ *
+ * `getActiveWindow()` goes through the platform a11y bridge (the persistent
+ * PowerShell/UIA process on Windows, AX on macOS, AT-SPI on Linux). That call
+ * is only ever used here to ANNOTATE the result ("Key pressed: X in [app]") —
+ * it is NOT load-bearing for the actual key/type action. So it must never be
+ * able to block or hang the action: if the bridge is slow, recovering, or
+ * wedged, we fall back to "(unknown)" after a short timeout and the keystroke
+ * still goes through. OS-agnostic — applies to every platform's bridge.
+ */
+async function activeWindowLabel(ctx) {
+    try {
+        const active = await Promise.race([
+            ctx.a11y.getActiveWindow(),
+            new Promise(resolve => setTimeout(() => resolve(null), 1500)),
+        ]);
+        return active ? `[${active.processName}] "${active.title}"` : '(unknown)';
+    }
+    catch {
+        return '(unknown)';
+    }
+}
+function getDesktopTools() {
+    return [
+        // ── PERCEPTION ──
+        {
+            name: 'desktop_screenshot',
+            description: 'LAST RESORT — take a screenshot only when the accessibility tree and OCR are both insufficient (custom canvas, icon-only UI, pixel-level verification). Prefer read_screen first, then ocr_read_screen; escalate to screenshot only when those fail. Returns the image resized to 1280px wide.',
+            parameters: {},
+            category: 'perception',
+            compactGroup: 'computer',
+            safetyTier: 0,
+            handler: async (_params, ctx) => {
+                await ctx.ensureInitialized();
+                const frame = await ctx.desktop.captureForLLM();
+                const base64 = frame.buffer.toString('base64');
+                return {
+                    text: `Screenshot: ${frame.llmWidth}x${frame.llmHeight}px (real: ${frame.width}x${frame.height}, scale: ${frame.scaleFactor.toFixed(2)}x). Mouse tools accept these image-space coordinates.`,
+                    image: { data: base64, mimeType: 'image/jpeg' },
+                };
+            },
+        },
+        {
+            name: 'desktop_screenshot_region',
+            description: 'Take a zoomed screenshot of a specific screen region for detailed inspection. Coordinates are in image-space (from desktop_screenshot).',
+            parameters: {
+                x: { type: 'number', description: 'Left edge X in image-space coordinates', required: true },
+                y: { type: 'number', description: 'Top edge Y in image-space coordinates', required: true },
+                width: { type: 'number', description: 'Width in image-space pixels', required: true },
+                height: { type: 'number', description: 'Height in image-space pixels', required: true },
+            },
+            category: 'perception',
+            compactGroup: 'computer',
+            safetyTier: 0,
+            handler: async ({ x, y, width, height }, ctx) => {
+                await ctx.ensureInitialized();
+                const sf = ctx.getScreenshotScaleFactor();
+                const frame = await ctx.desktop.captureRegionForLLM(Math.round(x * sf), Math.round(y * sf), Math.round(width * sf), Math.round(height * sf));
+                const base64 = frame.buffer.toString('base64');
+                return {
+                    text: `Region: (${x},${y}) ${width}x${height} image-space → zoomed to ${frame.llmWidth}x${frame.llmHeight}px.`,
+                    image: { data: base64, mimeType: 'image/jpeg' },
+                };
+            },
+        },
+        {
+            name: 'get_screen_size',
+            description: 'Get the screen dimensions and scale factor.',
+            parameters: {},
+            category: 'perception',
+            compactGroup: 'window',
+            safetyTier: 0,
+            handler: async (_params, ctx) => {
+                await ctx.ensureInitialized();
+                const size = ctx.desktop.getScreenSize();
+                const msf = ctx.getMouseScaleFactor();
+                const ssf = ctx.getScreenshotScaleFactor();
+                return {
+                    text: JSON.stringify({
+                        physicalWidth: size.width,
+                        physicalHeight: size.height,
+                        screenshotScaleFactor: ssf,
+                        mouseScaleFactor: msf,
+                        imageWidth: Math.round(size.width / ssf),
+                        imageHeight: Math.round(size.height / ssf),
+                    }),
+                };
+            },
+        },
+        // ── MOUSE ──
+        {
+            name: 'mouse_click',
+            description: 'Click the left mouse button at the given image-space coordinates.',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate in image-space', required: true },
+                y: { type: 'number', description: 'Y coordinate in image-space', required: true },
+            },
+            category: 'mouse',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ x, y }, ctx) => {
+                await ctx.ensureInitialized();
+                const sf = ctx.getMouseScaleFactor();
+                const rx = Math.round(x * sf), ry = Math.round(y * sf);
+                await ctx.desktop.mouseClick(rx, ry);
+                ctx.a11y.invalidateCache();
+                ctx.uiMaps?.invalidate();
+                return { text: `Clicked at (${x}, ${y}) → logical (${rx}, ${ry})` };
+            },
+        },
+        {
+            name: 'mouse_double_click',
+            description: 'Double-click the left mouse button at the given image-space coordinates.',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate in image-space', required: true },
+                y: { type: 'number', description: 'Y coordinate in image-space', required: true },
+            },
+            category: 'mouse',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ x, y }, ctx) => {
+                await ctx.ensureInitialized();
+                const sf = ctx.getMouseScaleFactor();
+                await ctx.desktop.mouseDoubleClick(Math.round(x * sf), Math.round(y * sf));
+                ctx.a11y.invalidateCache();
+                ctx.uiMaps?.invalidate();
+                return { text: `Double-clicked at (${x}, ${y})` };
+            },
+        },
+        {
+            name: 'mouse_right_click',
+            description: 'Right-click at the given image-space coordinates (opens context menu).',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate in image-space', required: true },
+                y: { type: 'number', description: 'Y coordinate in image-space', required: true },
+            },
+            category: 'mouse',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ x, y }, ctx) => {
+                await ctx.ensureInitialized();
+                const sf = ctx.getMouseScaleFactor();
+                await ctx.desktop.mouseRightClick(Math.round(x * sf), Math.round(y * sf));
+                ctx.a11y.invalidateCache();
+                ctx.uiMaps?.invalidate();
+                return { text: `Right-clicked at (${x}, ${y})` };
+            },
+        },
+        {
+            name: 'mouse_hover',
+            description: 'Move the mouse to the given image-space coordinates without clicking. Useful for revealing tooltips or hover menus.',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate in image-space', required: true },
+                y: { type: 'number', description: 'Y coordinate in image-space', required: true },
+            },
+            category: 'mouse',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ x, y }, ctx) => {
+                await ctx.ensureInitialized();
+                const sf = ctx.getMouseScaleFactor();
+                await ctx.desktop.mouseMove(Math.round(x * sf), Math.round(y * sf));
+                return { text: `Mouse moved to (${x}, ${y})` };
+            },
+        },
+        {
+            name: 'mouse_scroll',
+            description: 'Scroll the mouse wheel at the given image-space coordinates.',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate in image-space', required: true },
+                y: { type: 'number', description: 'Y coordinate in image-space', required: true },
+                direction: { type: 'string', description: 'Scroll direction', required: true, enum: ['up', 'down'] },
+                amount: { type: 'number', description: 'Scroll amount in wheel ticks (default: 3)', required: false, default: 3 },
+            },
+            category: 'mouse',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ x, y, direction, amount }, ctx) => {
+                await ctx.ensureInitialized();
+                const sf = ctx.getMouseScaleFactor();
+                const ticks = amount ?? 3;
+                const delta = direction === 'down' ? ticks : -ticks;
+                await ctx.desktop.mouseScroll(Math.round(x * sf), Math.round(y * sf), delta);
+                return { text: `Scrolled ${direction} ${ticks} ticks at (${x}, ${y})` };
+            },
+        },
+        {
+            name: 'mouse_drag',
+            description: 'Drag from one image-space coordinate to another (click-hold-move-release). Useful for selecting text, moving objects, or resizing.',
+            parameters: {
+                startX: { type: 'number', description: 'Start X in image-space', required: true },
+                startY: { type: 'number', description: 'Start Y in image-space', required: true },
+                endX: { type: 'number', description: 'End X in image-space', required: true },
+                endY: { type: 'number', description: 'End Y in image-space', required: true },
+                x1: { type: 'number', description: 'Alias for startX', required: false },
+                y1: { type: 'number', description: 'Alias for startY', required: false },
+                x2: { type: 'number', description: 'Alias for endX', required: false },
+                y2: { type: 'number', description: 'Alias for endY', required: false },
+            },
+            category: 'mouse',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ startX, startY, endX, endY, x1, y1, x2, y2 }, ctx) => {
+                await ctx.ensureInitialized();
+                const sx = startX ?? x1;
+                const sy = startY ?? y1;
+                const ex = endX ?? x2;
+                const ey = endY ?? y2;
+                const sf = ctx.getMouseScaleFactor();
+                await ctx.desktop.mouseDrag(Math.round(sx * sf), Math.round(sy * sf), Math.round(ex * sf), Math.round(ey * sf));
+                ctx.a11y.invalidateCache();
+                ctx.uiMaps?.invalidate();
+                return { text: `Dragged (${sx},${sy}) → (${ex},${ey})` };
+            },
+        },
+        // ── KEYBOARD ──
+        {
+            name: 'type_text',
+            description: 'Type text into the currently focused element. Internally uses clipboard paste for reliability (no dropped chars). The user clipboard is saved before and restored after, so calling type_text never clobbers any text the caller had previously placed on the clipboard.',
+            parameters: {
+                text: { type: 'string', description: 'The text to type', required: true },
+            },
+            category: 'keyboard',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ text }, ctx) => {
+                await ctx.ensureInitialized();
+                const activeInfo = await activeWindowLabel(ctx);
+                // Preserve the user's clipboard contents around the paste-as-type
+                // operation. Without this, callers who do
+                //   write_clipboard("important sentence")
+                //   type_text("\nheader\n")
+                //   key_press("ctrl+v")
+                // get the header text re-pasted instead of the sentence — type_text
+                // silently overwrote the clipboard. Save/restore makes type_text
+                // transparent to the clipboard.
+                let saved = null;
+                try {
+                    saved = await ctx.a11y.readClipboard();
+                }
+                catch { /* clipboard unreadable — leave saved=null, restore becomes a no-op below */ }
+                await ctx.a11y.writeClipboard(text);
+                await new Promise(r => setTimeout(r, 50));
+                // Paste combo is platform-specific
+                await ctx.desktop.keyPress(IS_MAC ? 'super+v' : 'ctrl+v');
+                await new Promise(r => setTimeout(r, 100));
+                // Restore clipboard. Best-effort — if the read failed (no clipboard
+                // available) or the restore throws, we don't surface the error;
+                // type_text's contract is about typing, not clipboard ops.
+                if (saved !== null) {
+                    try {
+                        await ctx.a11y.writeClipboard(saved);
+                    }
+                    catch { /* best-effort */ }
+                }
+                ctx.a11y.invalidateCache();
+                ctx.uiMaps?.invalidate();
+                return { text: `Typed ${text.length} chars into ${activeInfo}` };
+            },
+        },
+        {
+            name: 'key_press',
+            description: 'Press a keyboard key or key combination. Use "+" for a chord (e.g. "ctrl+s", "shift+enter", "alt+tab"). Separate multiple presses with SPACES to send them in sequence (e.g. "Down Down End", "ctrl+a Delete"). Single keys: "Return", "Tab", "Escape", "Backspace", "Delete", "F1"-"F12", "Left/Right/Up/Down".',
+            parameters: {
+                key: { type: 'string', description: 'Key/combo to press (e.g. "Return", "ctrl+a", "F5"). Space-separate combos for a sequence ("Down Down End").', required: true },
+            },
+            category: 'keyboard',
+            compactGroup: 'computer',
+            safetyTier: 1,
+            handler: async ({ key }, ctx) => {
+                await ctx.ensureInitialized();
+                // Defense-in-depth: a missing/mistyped arg used to reach `.toLowerCase()`
+                // on `undefined` and throw an opaque crash. Fail with an actionable
+                // message instead (the `computer` compound names this field `combo`,
+                // with `key` accepted as an alias).
+                if (typeof key !== 'string' || key.trim() === '') {
+                    return { text: 'key_press: "key" is required — the key or combo to press, e.g. "Return" or "ctrl+a". (On the `computer` compound the field is `combo`; `key` is also accepted.)', isError: true };
+                }
+                // "+" joins a chord; whitespace separates combos pressed in sequence.
+                const combos = key.trim().split(/\s+/);
+                for (const combo of combos) {
+                    const lower = combo.toLowerCase().replace(/\s+/g, '');
+                    if (BLOCKED_KEYS.some(b => lower === b)) {
+                        return { text: `BLOCKED: "${combo}" is a dangerous key combo.`, isError: true };
+                    }
+                }
+                const activeInfo = await activeWindowLabel(ctx);
+                for (const combo of combos) {
+                    await ctx.desktop.keyPress(combo);
+                }
+                ctx.a11y.invalidateCache();
+                ctx.uiMaps?.invalidate();
+                return { text: `Key pressed: ${key} in ${activeInfo}` };
+            },
+        },
+    ];
+}
+//# sourceMappingURL=desktop.js.map

package/dist/tools/desktop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"desktop.js","sourceRoot":"","sources":["../../src/tools/desktop.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCH,0CAmSC;AAjUD,uCAAyB;AAGzB,4CAA4C;AAC5C,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,iBAAiB,EAAE,cAAc,CAAC,CAAC;AACnE,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,EAAE,KAAK,QAAQ,CAAC;AAE1C;;;;;;;;;;GAUG;AACH,KAAK,UAAU,iBAAiB,CAAC,GAAgB;IAC/C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;YAC1B,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;SACpE,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,WAAW,MAAM,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;IAC5E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,WAAW,CAAC;IACrB,CAAC;AACH,CAAC;AAED,SAAgB,eAAe;IAC7B,OAAO;QACL,mBAAmB;QAEnB;YACE,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,gSAAgS;YAC7S,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,YAAY;YACtB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE;gBAC9B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;gBAChD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAC/C,OAAO;oBACL,IAAI,EAAE,eAAe,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,aAAa,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,uDAAuD;oBAC7L,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE;iBAChD,CAAC;YACJ,CAAC;SACF;QAED;YACE,IAAI,EAAE,2BAA2B;YACjC,WAAW,EAAE,yIAAyI;YACtJ,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wCAAwC,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC5F,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uCAAuC,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC3F,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACrF,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE,QAAQ,EAAE,IAAI,EAAE;aACxF;YACD,QAAQ,EAAE,YAAY;YACtB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC9C,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,wBAAwB,EAAE,CAAC;gBAC1C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,mBAAmB,CACjD,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EACtC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAChD,CAAC;gBACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAC/C,OAAO;oBACL,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,KAAK,KAAK,IAAI,MAAM,4BAA4B,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,KAAK;oBAC9G,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE;iBAChD,CAAC;YACJ,CAAC;SACF;QAED;YACE,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,6CAA6C;YAC1D,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,YAAY;YACtB,YAAY,EAAE,QAAQ;YACtB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE;gBAC9B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;gBACzC,MAAM,GAAG,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,GAAG,CAAC,wBAAwB,EAAE,CAAC;gBAC3C,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,aAAa,EAAE,IAAI,CAAC,KAAK;wBACzB,cAAc,EAAE,IAAI,CAAC,MAAM;wBAC3B,qBAAqB,EAAE,GAAG;wBAC1B,gBAAgB,EAAE,GAAG;wBACrB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;wBACxC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;qBAC3C,CAAC;iBACH,CAAC;YACJ,CAAC;SACF;QAED,cAAc;QAEd;YACE,IAAI,EAAE,aAAa;YACnB,WAAW,EAAE,mEAAmE;YAChF,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;aAClF;YACD,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC/B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACrC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBACvD,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3B,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBACzB,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACtE,CAAC;SACF;QAED;YACE,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,0EAA0E;YACvF,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;aAClF;YACD,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC/B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC3E,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3B,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBACzB,OAAO,EAAE,IAAI,EAAE,sBAAsB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACpD,CAAC;SACF;QAED;YACE,IAAI,EAAE,mBAAmB;YACzB,WAAW,EAAE,wEAAwE;YACrF,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;aAClF;YACD,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC/B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC1E,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3B,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBACzB,OAAO,EAAE,IAAI,EAAE,qBAAqB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACnD,CAAC;SACF;QAED;YACE,IAAI,EAAE,aAAa;YACnB,WAAW,EAAE,qHAAqH;YAClI,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;aAClF;YACD,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC/B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,mBAAmB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACjD,CAAC;SACF;QAED;YACE,IAAI,EAAE,cAAc;YACpB,WAAW,EAAE,8DAA8D;YAC3E,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;gBACpG,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2CAA2C,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE;aAClH;YACD,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE;gBAClD,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,CAAC;gBAC1B,MAAM,KAAK,GAAG,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;gBACpD,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC7E,OAAO,EAAE,IAAI,EAAE,YAAY,SAAS,IAAI,KAAK,cAAc,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC1E,CAAC;SACF;QAED;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,oIAAoI;YACjJ,UAAU,EAAE;gBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACjF,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC7E,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC7E,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACxE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACxE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACtE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,KAAK,EAAE;aACvE;YACD,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;gBACrE,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;gBACxB,MAAM,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;gBACxB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;gBACtB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;gBACtB,MAAM,EAAE,GAAG,GAAG,CAAC,mBAAmB,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,CACzB,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,EACxC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CACzC,CAAC;gBACF,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3B,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBACzB,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;YAC3D,CAAC;SACF;QAED,iBAAiB;QAEjB;YACE,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,0QAA0Q;YACvR,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE;aAC1E;YACD,QAAQ,EAAE,UAAU;YACpB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC/B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAEhD,kEAAkE;gBAClE,0CAA0C;gBAC1C,0CAA0C;gBAC1C,4BAA4B;gBAC5B,wBAAwB;gBACxB,oEAAoE;gBACpE,iEAAiE;gBACjE,gCAAgC;gBAChC,IAAI,KAAK,GAAkB,IAAI,CAAC;gBAChC,IAAI,CAAC;oBAAC,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,4EAA4E,CAAC,CAAC;gBAEtI,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBACpC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC1C,mCAAmC;gBACnC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBAE3C,oEAAoE;gBACpE,gEAAgE;gBAChE,2DAA2D;gBAC3D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACnB,IAAI,CAAC;wBAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC3E,CAAC;gBAED,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3B,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBACzB,OAAO,EAAE,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,eAAe,UAAU,EAAE,EAAE,CAAC;YACnE,CAAC;SACF;QAED;YACE,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,kTAAkT;YAC/T,UAAU,EAAE;gBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6GAA6G,EAAE,QAAQ,EAAE,IAAI,EAAE;aACpK;YACD,QAAQ,EAAE,UAAU;YACpB,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC9B,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC9B,0EAA0E;gBAC1E,oEAAoE;gBACpE,qEAAqE;gBACrE,oCAAoC;gBACpC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;oBACjD,OAAO,EAAE,IAAI,EAAE,iKAAiK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBACpM,CAAC;gBACD,sEAAsE;gBACtE,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBACtD,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;wBACxC,OAAO,EAAE,IAAI,EAAE,aAAa,KAAK,6BAA6B,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;oBAClF,CAAC;gBACH,CAAC;gBACD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAChD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACpC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3B,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;gBACzB,OAAO,EAAE,IAAI,EAAE,gBAAgB,GAAG,OAAO,UAAU,EAAE,EAAE,CAAC;YAC1D,CAAC;SACF;KACF,CAAC;AACJ,CAAC"}

package/dist/tools/electron_bridge.d.ts

@@ -0,0 +1,41 @@
+/**
+ * v0.8.2 — Electron / WebView2 / WKWebView detection + CDP bridging hints.
+ *
+ * Modern "native" apps on Windows and macOS are frequently Electron or
+ * WebView2 wrappers around Chromium. Their accessibility trees often
+ * contain only the window chrome — the entire document body is inside
+ * the web view and is NOT reachable via UIA or AX directly.
+ *
+ * Examples on Windows (WebView2 / Electron):
+ *   - New Outlook (olk)
+ *   - Microsoft Teams
+ *   - Discord
+ *   - Slack
+ *   - VS Code
+ *   - GitHub Desktop
+ *   - Notion
+ *
+ * The clean fix is to bridge to their embedded Chromium via CDP. That
+ * requires the app to be launched with a remote-debugging port. We
+ * cannot attach to an already-running app that wasn't launched that
+ * way, but we CAN:
+ *
+ *   1. Detect which running processes are Electron/WebView2 shells by
+ *      process-name heuristics + module snooping (on Windows).
+ *   2. Scan the common Electron remote-debug port range for already-
+ *      enabled /json endpoints.
+ *   3. Tell the agent HOW to relaunch the app with debugging enabled
+ *      so a bridge can attach.
+ *
+ * This file ships two tools:
+ *   - `detect_webview_apps`  — enumerate candidates + their CDP status
+ *   - `attach_webview_cdp`   — discover the debugger URL and connect
+ *                              the existing cdp_driver to it
+ *
+ * Linux TODO: the same heuristics apply (look for
+ * /proc/{pid}/cmdline containing `--type=renderer` or `.asar`), but
+ * AT-SPI already gives us a richer tree on Linux Electron apps than
+ * UIA does on Windows. Follow-up.
+ */
+import type { ToolDefinition } from './types';
+export declare function getElectronBridgeTools(): ToolDefinition[];