@aexol/spectral 0.9.212 → 0.9.215

package/dist/extensions/desktop-control/types.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Shared types for the desktop-control (computer-use) extension.
+ *
+ * The extension injects real mouse/keyboard input into the user's desktop.
+ * Every mutating action is wrapped by a safety layer in index.ts:
+ * screenshot BEFORE -> execute -> screenshot AFTER, with an interactive
+ * confirmation gate, headless deny-by-default, dry-run mode and an optional
+ * app allowlist.
+ */
+export type MouseButton = "left" | "right" | "middle";
+/** Kinds of native actions an adapter can support. `hotkey` covers both desktop_press_key and desktop_hotkey. */
+export type ActionKind = "mouse_move" | "mouse_click" | "mouse_drag" | "scroll" | "type_text" | "hotkey" | "focus_window";
+/**
+ * Normalized action description, platform-agnostic.
+ * Scroll semantics: dy > 0 scrolls DOWN, dx > 0 scrolls RIGHT (units: lines).
+ */
+export type DesktopAction = {
+    kind: "mouse_move";
+    x: number;
+    y: number;
+} | {
+    kind: "mouse_click";
+    x: number;
+    y: number;
+    button: MouseButton;
+    clickCount: number;
+} | {
+    kind: "mouse_drag";
+    fromX: number;
+    fromY: number;
+    toX: number;
+    toY: number;
+} | {
+    kind: "scroll";
+    x?: number;
+    y?: number;
+    dx: number;
+    dy: number;
+} | {
+    kind: "type_text";
+    text: string;
+} | {
+    kind: "hotkey";
+    tokens: string[];
+} | {
+    kind: "focus_window";
+    title: string;
+    owner?: string;
+    windowId?: string | number;
+};
+/** Result of a permission/capability probe. */
+export interface PermissionCheck {
+    ok: boolean;
+    error?: string;
+}
+/**
+ * Per-OS backend. Adapters only build and run native commands; all safety
+ * policy lives in safety.ts / index.ts. Builders are exported separately so
+ * unit tests can assert generated osascript / .ps1 / xdotool argv without
+ * executing anything.
+ */
+export interface DesktopAdapter {
+    platform: string;
+    capabilities: ActionKind[];
+    /** Optional TCC/permission probe (macOS accessibility, etc.). */
+    checkPermissions?(): Promise<PermissionCheck>;
+    /** Name (or "proc title") of the frontmost app, or null if unavailable. */
+    frontmostApp(): Promise<string | null>;
+    /** Human-readable command that WOULD run (used by dry-run plans). */
+    buildPlan(action: DesktopAction): string;
+    /**
+     * Execute the action natively. May return a short read-back string with the
+     * post-action state (macOS: "x,y" cursor position from CGEventGetLocation)
+     * so the safety layer can surface it and detect silent injection failures;
+     * null/undefined when the adapter has nothing to report.
+     */
+    execute(action: DesktopAction): Promise<string | null | void>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/extensions/desktop-control/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/extensions/desktop-control/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtD,iHAAiH;AACjH,MAAM,MAAM,UAAU,GAClB,YAAY,GACZ,aAAa,GACb,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,cAAc,CAAC;AAEnB;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACtF;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAClE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,CAAC;AAExF,+CAA+C;AAC/C,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,iEAAiE;IACjE,gBAAgB,CAAC,IAAI,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9C,2EAA2E;IAC3E,YAAY,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACvC,qEAAqE;IACrE,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAAC;IACzC;;;;;OAKG;IACH,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;CAC/D"}

package/dist/extensions/desktop-control/types.js

@@ -0,0 +1,10 @@
+/**
+ * Shared types for the desktop-control (computer-use) extension.
+ *
+ * The extension injects real mouse/keyboard input into the user's desktop.
+ * Every mutating action is wrapped by a safety layer in index.ts:
+ * screenshot BEFORE -> execute -> screenshot AFTER, with an interactive
+ * confirmation gate, headless deny-by-default, dry-run mode and an optional
+ * app allowlist.
+ */
+export {};

package/dist/extensions/desktop-control/windows.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Windows adapter for desktop-control.
+ *
+ * Input injection via PowerShell scripts (written to temp .ps1 files and run
+ * with cross-spawn, same pattern as the desktop-screenshot extension) that
+ * Add-Type a small user32 P/Invoke class:
+ *   - SetCursorPos for absolute cursor placement
+ *   - SendInput for clicks (MOUSEEVENTF_*), scrolling (WHEEL/HWHEEL) and keys
+ *     (VK codes; type_text uses KEYEVENTF_UNICODE so layout/IME is irrelevant)
+ *   - SetForegroundWindow/ShowWindow/IsIconic for window focus
+ *
+ * NOTE: each invocation pays a one-time Add-Type JIT (~1-2s). UIPI means a
+ * non-elevated spectral cannot inject into elevated (admin) windows.
+ */
+import type { DesktopAction, DesktopAdapter } from "./types.js";
+/** argv for running a temp .ps1 (mirrors desktop-screenshot's runPowerShell). */
+export declare function buildPowerShellArgs(psPath: string): string[];
+export declare function buildMousePS(action: Extract<DesktopAction, {
+    kind: "mouse_move" | "mouse_click" | "mouse_drag" | "scroll";
+}>): string;
+export declare function resolveChordTokens(tokens: string[]): {
+    key: number;
+    modifiers: number[];
+};
+export declare function buildHotkeyPS(key: number, modifiers: number[]): string;
+/** One SendUnicode call per char (KEYEVENTF_UNICODE down+up); \n -> CR (Enter). */
+export declare function buildTypeTextPS(text: string): string;
+export declare function buildFocusPS(hwnd: string | number): string;
+export declare function buildFrontmostPS(): string;
+export declare const win32Adapter: DesktopAdapter;
+//# sourceMappingURL=windows.d.ts.map

package/dist/extensions/desktop-control/windows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"windows.d.ts","sourceRoot":"","sources":["../../../src/extensions/desktop-control/windows.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAwDhE,iFAAiF;AACjF,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAE5D;AAUD,wBAAgB,YAAY,CAC1B,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE;IAAE,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,YAAY,GAAG,QAAQ,CAAA;CAAE,CAAC,GAC/F,MAAM,CA2CR;AAYD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,CAOzF;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAStE;AAED,mFAAmF;AACnF,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASpD;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAS1D;AAED,wBAAgB,gBAAgB,IAAI,MAAM,CAUzC;AAgCD,eAAO,MAAM,YAAY,EAAE,cAyD1B,CAAC"}

package/dist/extensions/desktop-control/windows.js

@@ -0,0 +1,276 @@
+/**
+ * Windows adapter for desktop-control.
+ *
+ * Input injection via PowerShell scripts (written to temp .ps1 files and run
+ * with cross-spawn, same pattern as the desktop-screenshot extension) that
+ * Add-Type a small user32 P/Invoke class:
+ *   - SetCursorPos for absolute cursor placement
+ *   - SendInput for clicks (MOUSEEVENTF_*), scrolling (WHEEL/HWHEEL) and keys
+ *     (VK codes; type_text uses KEYEVENTF_UNICODE so layout/IME is irrelevant)
+ *   - SetForegroundWindow/ShowWindow/IsIconic for window focus
+ *
+ * NOTE: each invocation pays a one-time Add-Type JIT (~1-2s). UIPI means a
+ * non-elevated spectral cannot inject into elevated (admin) windows.
+ */
+import { unlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import crossSpawn from "cross-spawn";
+import { WINDOWS_MODIFIER_VK, WINDOWS_VK, isModifierToken } from "./keys.js";
+const TIMEOUT_MS = 15_000;
+const WHEEL_DELTA = 120; // per line
+const CSHARP_PRELUDE = `using System;
+using System.Runtime.InteropServices;
+public class Win32Input {
+    [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
+    [DllImport("user32.dll", SetLastError = true)] public static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize);
+    [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
+    [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder lpString, int nMaxCount);
+    [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId);
+    [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
+    [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
+    [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr hWnd);
+    [StructLayout(LayoutKind.Sequential)] public struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
+    [StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
+    [StructLayout(LayoutKind.Explicit)] public struct INPUTUNION { [FieldOffset(0)] public MOUSEINPUT mi; [FieldOffset(0)] public KEYBDINPUT ki; }
+    [StructLayout(LayoutKind.Sequential)] public struct INPUT { public uint type; public INPUTUNION u; }
+    public const uint INPUT_MOUSE = 0, INPUT_KEYBOARD = 1;
+    public const uint KEYEVENTF_KEYUP = 0x0002, KEYEVENTF_UNICODE = 0x0004;
+    public const uint LEFT_DOWN = 0x0002, LEFT_UP = 0x0004, RIGHT_DOWN = 0x0008, RIGHT_UP = 0x0010, MIDDLE_DOWN = 0x0020, MIDDLE_UP = 0x0040, WHEEL = 0x0800, HWHEEL = 0x1000;
+    public static void SendMouse(uint flags, int dx, int dy, int data) {
+        var input = new INPUT(); input.type = INPUT_MOUSE;
+        input.u.mi.dx = dx; input.u.mi.dy = dy; input.u.mi.mouseData = unchecked((uint)data);
+        input.u.mi.dwFlags = flags; input.u.mi.time = 0; input.u.mi.dwExtraInfo = IntPtr.Zero;
+        var arr = new INPUT[1]; arr[0] = input;
+        SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
+    }
+    public static void SendKey(ushort vk, bool isDown) {
+        var input = new INPUT(); input.type = INPUT_KEYBOARD;
+        input.u.ki.wVk = vk; input.u.ki.wScan = 0; input.u.ki.dwFlags = isDown ? 0u : KEYEVENTF_KEYUP;
+        input.u.ki.time = 0; input.u.ki.dwExtraInfo = IntPtr.Zero;
+        var arr = new INPUT[1]; arr[0] = input;
+        SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
+    }
+    public static void SendUnicode(int code) {
+        var down = new INPUT(); down.type = INPUT_KEYBOARD;
+        down.u.ki.wVk = 0; down.u.ki.wScan = (ushort)code; down.u.ki.dwFlags = KEYEVENTF_UNICODE;
+        down.u.ki.time = 0; down.u.ki.dwExtraInfo = IntPtr.Zero;
+        var up = new INPUT(); up.type = INPUT_KEYBOARD;
+        up.u.ki.wVk = 0; up.u.ki.wScan = (ushort)code; up.u.ki.dwFlags = KEYEVENTF_UNICODE | KEYEVENTF_KEYUP;
+        up.u.ki.time = 0; up.u.ki.dwExtraInfo = IntPtr.Zero;
+        var arr = new INPUT[2]; arr[0] = down; arr[1] = up;
+        SendInput(2, arr, Marshal.SizeOf(typeof(INPUT)));
+    }
+}`;
+const PS_PRELUDE = ['Add-Type -AssemblyName System.Windows.Forms', 'Add-Type @"', CSHARP_PRELUDE, '"@'];
+// ---------------------------------------------------------------------------
+// Builders (exported for unit tests — pure, never executed by tests)
+// ---------------------------------------------------------------------------
+/** argv for running a temp .ps1 (mirrors desktop-screenshot's runPowerShell). */
+export function buildPowerShellArgs(psPath) {
+    return ["-NoProfile", "-Sta", "-ExecutionPolicy", "Bypass", "-File", psPath];
+}
+const BTN_FLAGS = {
+    left: { down: "LEFT_DOWN", up: "LEFT_UP" },
+    right: { down: "RIGHT_DOWN", up: "RIGHT_UP" },
+    middle: { down: "MIDDLE_DOWN", up: "MIDDLE_UP" },
+};
+const DRAG_STEPS = 10;
+export function buildMousePS(action) {
+    const lines = [];
+    switch (action.kind) {
+        case "mouse_move":
+            lines.push(`[Win32Input]::SetCursorPos(${action.x}, ${action.y})`);
+            break;
+        case "mouse_click": {
+            const b = BTN_FLAGS[action.button];
+            lines.push(`[Win32Input]::SetCursorPos(${action.x}, ${action.y})`);
+            lines.push("Start-Sleep -Milliseconds 80");
+            for (let i = 0; i < action.clickCount; i++) {
+                if (i > 0)
+                    lines.push("Start-Sleep -Milliseconds 50");
+                lines.push(`[Win32Input]::SendMouse([Win32Input]::${b.down}, 0, 0, 0)`);
+                lines.push(`[Win32Input]::SendMouse([Win32Input]::${b.up}, 0, 0, 0)`);
+            }
+            break;
+        }
+        case "mouse_drag":
+            lines.push(`[Win32Input]::SetCursorPos(${action.fromX}, ${action.fromY})`);
+            lines.push("Start-Sleep -Milliseconds 80");
+            lines.push("[Win32Input]::SendMouse([Win32Input]::LEFT_DOWN, 0, 0, 0)");
+            for (let i = 1; i <= DRAG_STEPS; i++) {
+                const t = i / DRAG_STEPS;
+                const x = Math.round(action.fromX + (action.toX - action.fromX) * t);
+                const y = Math.round(action.fromY + (action.toY - action.fromY) * t);
+                lines.push(`[Win32Input]::SetCursorPos(${x}, ${y})`);
+                lines.push("Start-Sleep -Milliseconds 15");
+            }
+            lines.push("[Win32Input]::SendMouse([Win32Input]::LEFT_UP, 0, 0, 0)");
+            break;
+        case "scroll": {
+            if (action.x !== undefined) {
+                lines.push(`[Win32Input]::SetCursorPos(${action.x}, ${action.y})`);
+                lines.push("Start-Sleep -Milliseconds 50");
+            }
+            // Windows wheel delta: positive = up, so invert dy. dx positive = right.
+            if (action.dy !== 0)
+                lines.push(`[Win32Input]::SendMouse([Win32Input]::WHEEL, 0, 0, ${-action.dy * WHEEL_DELTA})`);
+            if (action.dx !== 0)
+                lines.push(`[Win32Input]::SendMouse([Win32Input]::HWHEEL, 0, 0, ${action.dx * WHEEL_DELTA})`);
+            break;
+        }
+    }
+    lines.push("Write-Output 'OK'");
+    return [...PS_PRELUDE, ...lines].join("\r\n");
+}
+function resolveVk(name) {
+    const vk = WINDOWS_VK[name];
+    if (vk === undefined) {
+        throw new Error(`unsupported key "${name}" for Windows VK injection (supported: letters, digits, f1-f12, return/tab/space/escape/backspace/delete, arrows, home/end/pageup/pagedown, punctuation)`);
+    }
+    return vk;
+}
+export function resolveChordTokens(tokens) {
+    const modifiers = [];
+    for (const t of tokens.slice(0, -1)) {
+        if (!isModifierToken(t))
+            throw new Error(`unsupported hotkey token "${t}" (expected modifier)`);
+        modifiers.push(WINDOWS_MODIFIER_VK[t]);
+    }
+    return { key: resolveVk(tokens[tokens.length - 1]), modifiers };
+}
+export function buildHotkeyPS(key, modifiers) {
+    const lines = [];
+    for (const m of modifiers)
+        lines.push(`[Win32Input]::SendKey(0x${m.toString(16)}, $true)`);
+    lines.push(`[Win32Input]::SendKey(0x${key.toString(16)}, $true)`);
+    lines.push("Start-Sleep -Milliseconds 30");
+    lines.push(`[Win32Input]::SendKey(0x${key.toString(16)}, $false)`);
+    for (const m of [...modifiers].reverse())
+        lines.push(`[Win32Input]::SendKey(0x${m.toString(16)}, $false)`);
+    lines.push("Write-Output 'OK'");
+    return [...PS_PRELUDE, ...lines].join("\r\n");
+}
+/** One SendUnicode call per char (KEYEVENTF_UNICODE down+up); \n -> CR (Enter). */
+export function buildTypeTextPS(text) {
+    const lines = [];
+    for (const ch of text.replace(/\r?\n/g, "\r")) {
+        const code = ch.codePointAt(0);
+        if (code === undefined)
+            continue;
+        lines.push(`[Win32Input]::SendUnicode(0x${code.toString(16)})`);
+    }
+    lines.push("Write-Output 'OK'");
+    return [...PS_PRELUDE, ...lines].join("\r\n");
+}
+export function buildFocusPS(hwnd) {
+    const hwndHex = Number(hwnd).toString(16);
+    return [
+        ...PS_PRELUDE,
+        `$hwnd = [IntPtr]::new(0x${hwndHex})`,
+        "if ([Win32Input]::IsIconic($hwnd)) { [Win32Input]::ShowWindow($hwnd, 9) | Out-Null }", // SW_RESTORE
+        "[Win32Input]::SetForegroundWindow($hwnd) | Out-Null",
+        "Write-Output 'OK'",
+    ].join("\r\n");
+}
+export function buildFrontmostPS() {
+    return [
+        ...PS_PRELUDE,
+        "$fg = [Win32Input]::GetForegroundWindow()",
+        "$sb = New-Object System.Text.StringBuilder(512)",
+        "[Win32Input]::GetWindowText($fg, $sb, 512) | Out-Null",
+        "$procId = 0",
+        "[Win32Input]::GetWindowThreadProcessId($fg, [ref]$procId) | Out-Null",
+        "try { $p = Get-Process -Id $procId -ErrorAction Stop; Write-Output ($p.ProcessName + ' ' + $sb.ToString()) } catch { Write-Output $sb.ToString() }",
+    ].join("\r\n");
+}
+// ---------------------------------------------------------------------------
+// Runner + adapter
+// ---------------------------------------------------------------------------
+async function runPowerShellScript(script) {
+    const psPath = join(tmpdir(), `desktop-control-${randomUUID()}.ps1`);
+    // 0600: temp scripts may contain typed text; always unlink after execution.
+    writeFileSync(psPath, script, { encoding: "utf-8", mode: 0o600 });
+    try {
+        return await new Promise((resolve, reject) => {
+            const child = crossSpawn("powershell.exe", buildPowerShellArgs(psPath), {
+                timeout: TIMEOUT_MS,
+                windowsHide: true,
+            });
+            let stdout = "";
+            let stderr = "";
+            child.stdout?.on("data", (chunk) => { stdout += chunk.toString(); });
+            child.stderr?.on("data", (chunk) => { stderr += chunk.toString(); });
+            child.on("error", reject);
+            child.on("close", (code) => {
+                if (code === 0)
+                    return resolve(stdout);
+                const tail = stderr.trim().split("\n").pop() || `exited with code ${code}`;
+                reject(new Error(`powershell.exe ${tail}`));
+            });
+        });
+    }
+    finally {
+        try {
+            unlinkSync(psPath);
+        }
+        catch { /* best effort — already gone */ }
+    }
+}
+export const win32Adapter = {
+    platform: "win32",
+    capabilities: ["mouse_move", "mouse_click", "mouse_drag", "scroll", "type_text", "hotkey", "focus_window"],
+    async frontmostApp() {
+        try {
+            const out = (await runPowerShellScript(buildFrontmostPS())).trim();
+            return out.length > 0 ? out : null;
+        }
+        catch {
+            return null;
+        }
+    },
+    buildPlan(action) {
+        switch (action.kind) {
+            case "mouse_move":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # SetCursorPos(${action.x}, ${action.y})`;
+            case "mouse_click":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # SetCursorPos + SendInput ${action.button} x${action.clickCount} @(${action.x},${action.y})`;
+            case "mouse_drag":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # LEFT_DOWN@(${action.fromX},${action.fromY}) + ${DRAG_STEPS} moves -> LEFT_UP@(${action.toX},${action.toY})`;
+            case "scroll":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # SendInput WHEEL dy=${-action.dy * WHEEL_DELTA} HWHEEL dx=${action.dx * WHEEL_DELTA}`;
+            case "type_text":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # SendInput KEYEVENTF_UNICODE x${action.text.length}`;
+            case "hotkey":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # SendInput VK ${action.tokens.join("+")}`;
+            case "focus_window":
+                return `powershell.exe -NoProfile -Sta -File <tempfile>  # SetForegroundWindow(hwnd for "${action.title}")`;
+        }
+    },
+    async execute(action) {
+        switch (action.kind) {
+            case "mouse_move":
+            case "mouse_click":
+            case "mouse_drag":
+            case "scroll":
+                await runPowerShellScript(buildMousePS(action));
+                return;
+            case "type_text":
+                await runPowerShellScript(buildTypeTextPS(action.text));
+                return;
+            case "hotkey": {
+                const { key, modifiers } = resolveChordTokens(action.tokens);
+                await runPowerShellScript(buildHotkeyPS(key, modifiers));
+                return;
+            }
+            case "focus_window": {
+                if (action.windowId === undefined) {
+                    throw new Error("focus_window requires a resolved window id (use list_windows, then retry with an exact title)");
+                }
+                await runPowerShellScript(buildFocusPS(action.windowId));
+                return;
+            }
+        }
+    },
+};

package/dist/extensions/desktop-screenshot/index.d.ts

@@ -13,5 +13,108 @@
  * extension automatically analyzes with a vision model.
  */
 import type { ExtensionAPI } from "../../sdk/coding-agent/index.js";
+import type { ImageContent } from "../../sdk/ai/index.js";
+export type ScreenshotFormat = "png" | "jpg" | "jpeg";
+export interface ScreenshotOptions {
+    screen?: string | number;
+    format?: ScreenshotFormat;
+    filename?: string;
+}
+export interface WindowInfo {
+    id: number | string;
+    owner: string;
+    title: string;
+}
+export declare function tmpDir(): string;
+export declare function saveScreenshot(buf: Buffer, format: ScreenshotFormat, cwd: string): string;
+interface DarwinScreenRect {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface DarwinScreenGeometry {
+    displayCount: number;
+    /** Primary display top-left origin in the global CGEvent (points) space. */
+    primaryOrigin: {
+        x: number;
+        y: number;
+    };
+    /** Main screen (keyboard focus): logical size + top-left global origin, in points. */
+    main: DarwinScreenRect;
+    backingScaleFactor: number;
+}
+/** Test hook: forget the cached geometry so the next probe hits osascript again. */
+export declare function resetDarwinScreenGeometryCache(): void;
+/**
+ * Logical display geometry via JXA/AppKit, cached for a few seconds (display
+ * arrangements rarely change mid-session). Resolves to `null` when the probe
+ * fails — captures are then left untouched and the tool result says the
+ * scale is unverified. Exported for tests.
+ */
+export declare function darwinScreenGeometry(): Promise<DarwinScreenGeometry | null>;
+export type DarwinScreenshotMode = "screen" | "window";
+export interface DarwinNormalizedScreenshot {
+    buffer: Buffer;
+    /** Coordinate-space metadata appended to the tool result text ("" when nothing to say). */
+    metadata: string;
+    /** Structured fields mirrored into the tool result `details` (additive). */
+    details: {
+        imageWidthPx?: number;
+        imageHeightPx?: number;
+        logicalWidthPt?: number;
+        logicalHeightPt?: number;
+        displayCount?: number;
+        primaryOriginPt?: {
+            x: number;
+            y: number;
+        };
+        scaleApplied?: number;
+        resampled?: boolean;
+    };
+}
+/**
+ * Normalize a macOS capture so image pixels map 1:1 onto the logical screen
+ * points the desktop_* tools accept. Full-screen captures are resampled to
+ * the main screen's logical width when the file is a Retina (backing-scale)
+ * capture; window captures are resampled by the display's backing scale
+ * factor (their coordinates are window-local, which the metadata states).
+ * Never throws: on any probe failure the original buffer comes back with a
+ * caution note instead of a silently wrong scale. Exported for tests.
+ */
+export declare function normalizeDarwinScreenshot(buf: Buffer, format: ScreenshotFormat, mode: DarwinScreenshotMode): Promise<DarwinNormalizedScreenshot>;
+/**
+ * Run a PowerShell script file and return its stdout as a string.
+ *
+ * Uses `cross-spawn` instead of Node's `execFile` because, on Windows,
+ * `execFile` does not resolve commands through PATHEXT and requires a fully
+ * populated PATH. Inside Electron/desktop sandboxes the PATH is often trimmed,
+ * which leads to `spawn powershell.exe ENOENT`. `cross-spawn` resolves the
+ * executable via PATHEXT/PATH like a shell would.
+ *
+ * `-Sta` is passed so the single-threaded apartment required by
+ * System.Windows.Forms / CopyFromScreen is initialized even in sessions that
+ * default to MTA (avoids COM failures).
+ */
+export declare function runPowerShell(psPath: string, opts?: {
+    timeoutMs?: number;
+}): Promise<string>;
+export declare function screenshot(options?: ScreenshotOptions): Promise<Buffer>;
+export declare function screenshotWindow(windowId: number | string, options: ScreenshotOptions, platform: string): Promise<Buffer>;
+export declare function listWindows(): Promise<WindowInfo[]>;
+export declare function formatWindowList(windows: WindowInfo[], maxRows?: number): string;
+/**
+ * Last-resort inline carrier for a screenshot that could not be hosted.
+ *
+ * Returns the image unchanged when its base64 payload already fits
+ * `MAX_INLINE_IMAGE_BASE64_CHARS`, a downscaled/re-encoded copy when resizing
+ * gets it under the cap, and `null` when even that fails (no image resizer
+ * available, or the capture is absurdly large). Callers MUST treat `null` as
+ * "do not attach base64" and report the failure — never as "attach it anyway".
+ *
+ * Exported for tests.
+ */
+export declare function fitInlineScreenshot(image: ImageContent, maxBase64Chars?: number): Promise<ImageContent | null>;
 export default function desktopScreenshotExtension(ext: ExtensionAPI): Promise<void>;
+export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/extensions/desktop-screenshot/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/desktop-screenshot/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAQH,OAAO,KAAK,EAEX,YAAY,EAEZ,MAAM,iCAAiC,CAAC;AA6ezC,wBAA8B,0BAA0B,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAsFzF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/desktop-screenshot/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAQH,OAAO,KAAK,EAEX,YAAY,EAEZ,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAa1D,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;AAItD,MAAM,WAAW,iBAAiB;IACjC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,MAAM,GAAG,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACd;AAYD,wBAAgB,MAAM,IAAI,MAAM,CAI/B;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAUzF;AAkJD,UAAU,gBAAgB;IAAG,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;CAAE;AAEnF,MAAM,WAAW,oBAAoB;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,4EAA4E;IAC5E,aAAa,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,sFAAsF;IACtF,IAAI,EAAE,gBAAgB,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;CAC3B;AAKD,oFAAoF;AACpF,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AA2BD;;;;;GAKG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA4BjF;AAsBD,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEvD,MAAM,WAAW,0BAA0B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,2FAA2F;IAC3F,QAAQ,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,OAAO,EAAE;QACR,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,wBAAsB,yBAAyB,CAC9C,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,oBAAoB,GACxB,OAAO,CAAC,0BAA0B,CAAC,CA8GrC;AAgGD;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAkB5F;AA6ID,wBAAgB,UAAU,CAAC,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAO3E;AAED,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,MAAM,GAAG,MAAM,EACzB,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAOjB;AAED,wBAAgB,WAAW,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAOnD;AAMD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,SAAK,GAAG,MAAM,CAY5E;AAMD;;;;;;;;;;GAUG;AACH,wBAAsB,mBAAmB,CACxC,KAAK,EAAE,YAAY,EACnB,cAAc,GAAE,MAAsC,GACpD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAgB9B;AAMD,wBAA8B,0BAA0B,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA6LzF"}