@aexol/spectral 0.9.212 → 0.9.216

package/dist/extensions/desktop-control/index.js

@@ -0,0 +1,633 @@
+/**
+ * Desktop Control Extension (computer-use with a safety layer)
+ *
+ * Registers snake_case `desktop_*` tools that inject REAL mouse/keyboard
+ * input via native OS commands:
+ *   - macOS:   JXA (osascript -l JavaScript) + CoreGraphics CGEventPost
+ *   - Windows: PowerShell + Add-Type user32 SendInput/SetCursorPos
+ *   - Linux:   xdotool (X11) / wtype (Wayland, keyboard-only)
+ *
+ * Tools are registered ONLY when the platform/session is capable. Every
+ * mutating action is wrapped by the safety layer:
+ *   1. By default there is NO confirmation prompt: enabling the extension
+ *      (enabledNativeExtensions contains "desktop-control") is the explicit
+ *      user consent, so actions run in interactive, headless, serve and RPC
+ *      sessions alike. Opt-in SPECTRAL_DESKTOP_REQUIRE_CONFIRM=1|true restores
+ *      the per-session interactive confirmation (ctx.hasUI; headless
+ *      auto-denies single actions without locking the session).
+ *   2. Optional SPECTRAL_DESKTOP_ALLOWLIST restricts actions to matching
+ *      frontmost apps.
+ *   3. Screenshot BEFORE -> execute -> screenshot AFTER; both images are
+ *      returned as image content so the model verifies the outcome
+ *      (spectral-vision-fallback analyzes them automatically). A pointer
+ *      read-back and a "no visible change" hint expose silent injection
+ *      failures (CGEventPost never errors).
+ *   4. SPECTRAL_DESKTOP_DRY_RUN=1 returns action plans without executing.
+ *
+ * This extension is registered with defaultEnabled: false — it must be
+ * opted in via Settings.enabledNativeExtensions: ["desktop-control"].
+ *
+ * Zero external npm dependencies beyond cross-spawn (Windows PATH is trimmed
+ * under Electron). No native modules.
+ */
+import { listWindows, screenshot, formatWindowList, } from "../desktop-screenshot/index.js";
+import { parseConfirmTimeoutMs } from "../../server/ui/terminal-confirm.js";
+import { createLinuxAdapter } from "./linux.js";
+import { macosAdapter } from "./macos.js";
+import { win32Adapter } from "./windows.js";
+import { DesktopActionPolicy, describeAction, isDesktopControlTool, isDryRun, matchesAllowlist, isRequireConfirm, parseAllowlist, } from "./safety.js";
+import { isModifierToken, parseChord } from "./keys.js";
+const SCREENSHOT_FORMAT = "png";
+// ---------------------------------------------------------------------------
+// Adapter selection
+// ---------------------------------------------------------------------------
+export async function createAdapter(env = process.env) {
+    switch (process.platform) {
+        case "darwin":
+            return { adapter: macosAdapter };
+        case "win32":
+            return { adapter: win32Adapter };
+        case "linux":
+            return createLinuxAdapter(env);
+        default:
+            return { adapter: null, reason: `unsupported platform: ${process.platform}` };
+    }
+}
+function textContent(text) {
+    return { type: "text", text };
+}
+function imageContent(buf) {
+    return { type: "image", data: buf.toString("base64"), mimeType: "image/png" };
+}
+function errorResult(msg, details = {}) {
+    return {
+        content: [textContent(`❌ ${msg}`)],
+        details: { isError: true, error: msg, ...details },
+    };
+}
+function dryRunResult(action, adapter) {
+    return {
+        content: [
+            textContent(`DRY-RUN — nothing was injected into the desktop.\n` +
+                `Would execute: ${describeAction(action)}\n` +
+                `Command: ${adapter.buildPlan(action)}\n` +
+                `Unset SPECTRAL_DESKTOP_DRY_RUN to execute for real.`),
+        ],
+        details: { dryRun: true, action, plan: adapter.buildPlan(action) },
+    };
+}
+/** Parses the macOS pointer read-back ("x,y" cursor position from CGEventGetLocation). */
+function parsePointer(readback) {
+    const m = /^(-?\d+),(-?\d+)$/.exec(readback.trim());
+    if (!m)
+        return null;
+    return { x: Number(m[1]), y: Number(m[2]) };
+}
+/** Cursor position a pointer-targeting action SHOULD leave behind (silent-failure heuristic). */
+function pointerTarget(action) {
+    switch (action.kind) {
+        case "mouse_move":
+            return { x: action.x, y: action.y };
+        case "mouse_click":
+            return { x: action.x, y: action.y };
+        case "mouse_drag":
+            return { x: action.toX, y: action.toY };
+        default:
+            return null;
+    }
+}
+const NO_VISIBLE_CHANGE_HINT = "no visible change — verify BEFORE/AFTER screenshots; check Accessibility permission";
+/**
+ * Execute a mutating action inside the safety envelope:
+ * permissions -> allowlist -> BEFORE screenshot -> execute -> AFTER screenshot.
+ * Returns both screenshots as image content for vision verification.
+ */
+async function runMutating(action, deps) {
+    const { adapter } = deps;
+    const env = deps.env ?? process.env;
+    if (isDryRun(env))
+        return dryRunResult(action, adapter);
+    if (adapter.checkPermissions) {
+        try {
+            const perm = await adapter.checkPermissions();
+            if (!perm.ok)
+                return errorResult(perm.error ?? "permission check failed", { action });
+        }
+        catch (err) {
+            return errorResult(`permission check failed: ${err instanceof Error ? err.message : String(err)}`, { action });
+        }
+    }
+    // Allowlist: mutating actions only run while a permitted app is focused.
+    const allowlist = parseAllowlist(env.SPECTRAL_DESKTOP_ALLOWLIST);
+    let frontBefore = null;
+    if (allowlist.length > 0) {
+        if (action.kind === "focus_window") {
+            const owner = action.owner ?? "";
+            if (!owner || !matchesAllowlist(owner, allowlist)) {
+                return errorResult(`SPECTRAL_DESKTOP_ALLOWLIST blocks focusing "${action.title}" (owner "${owner || "unknown"}" does not match ${allowlist.join(", ")})`, { action, allowlist });
+            }
+        }
+        else {
+            frontBefore = await adapter.frontmostApp().catch(() => null);
+            if (!frontBefore || !matchesAllowlist(frontBefore, allowlist)) {
+                return errorResult(`SPECTRAL_DESKTOP_ALLOWLIST blocks this action: frontmost app is "${frontBefore ?? "unknown"}", expected one of ${allowlist.join(", ")}`, { action, allowlist, frontmost: frontBefore });
+            }
+        }
+    }
+    // BEFORE screenshot — refuse to mutate blind.
+    const capture = deps.capture ?? ((opts) => screenshot({ format: SCREENSHOT_FORMAT, ...opts }));
+    let before;
+    try {
+        before = await capture({ format: SCREENSHOT_FORMAT });
+    }
+    catch (err) {
+        return errorResult(`Refusing to mutate: BEFORE screenshot failed (${err instanceof Error ? err.message : String(err)}).`, { action });
+    }
+    let readback;
+    try {
+        // TOCTOU hardening: the frontmost app may have changed between the
+        // allowlist check above and execution (e.g. while capturing the
+        // BEFORE screenshot). Re-probe immediately before injecting and fail
+        // closed on mismatch. focus_window is exempt: its allowlist check
+        // targets the window owner, and the action itself changes frontmost.
+        if (allowlist.length > 0 && action.kind !== "focus_window") {
+            const frontNow = await adapter.frontmostApp().catch(() => null);
+            if (!frontNow || !matchesAllowlist(frontNow, allowlist)) {
+                const msg = `SPECTRAL_DESKTOP_ALLOWLIST blocked this action right before execution: frontmost app is "${frontNow ?? "unknown"}", expected one of ${allowlist.join(", ")}`;
+                return {
+                    content: [textContent(`❌ ${msg}`), imageContent(before)],
+                    details: {
+                        isError: true,
+                        error: msg,
+                        action,
+                        allowlist,
+                        frontmost: frontNow,
+                        beforeScreenshot: true,
+                    },
+                };
+            }
+        }
+        readback = await adapter.execute(action);
+    }
+    catch (err) {
+        const msg = `Action failed: ${err instanceof Error ? err.message : String(err)}`;
+        return {
+            content: [textContent(`❌ ${msg}`), imageContent(before)],
+            details: { isError: true, error: msg, action, beforeScreenshot: true },
+        };
+    }
+    // AFTER screenshot — verification may fail even though the action ran.
+    let after = null;
+    let afterError = null;
+    try {
+        after = await capture({ format: SCREENSHOT_FORMAT });
+    }
+    catch (err) {
+        afterError = err instanceof Error ? err.message : String(err);
+    }
+    const frontAfter = await adapter.frontmostApp().catch(() => null);
+    // Post-action read-back: CGEventPost never errors, so a cursor that did not
+    // reach the requested coordinates (plus unchanged frontmost) is the only
+    // programmatic signal that the injection silently no-opped. Heuristic only
+    // — never a hard failure.
+    const pointerAfter = typeof readback === "string" ? parsePointer(readback) : null;
+    const target = pointerTarget(action);
+    const pointerMissedTarget = pointerAfter !== null && target !== null &&
+        (Math.abs(pointerAfter.x - target.x) > 1 || Math.abs(pointerAfter.y - target.y) > 1);
+    const pointerLine = pointerAfter ? `\nPointer now at (${pointerAfter.x}, ${pointerAfter.y}).` : "";
+    const hintLine = pointerMissedTarget ? `\n⚠️ ${NO_VISIBLE_CHANGE_HINT}.` : "";
+    const frontLine = frontBefore || frontAfter
+        ? `\nFrontmost: ${frontBefore ?? "?"} -> ${frontAfter ?? "?"}.`
+        : "";
+    const afterLine = after
+        ? "BEFORE and AFTER screenshots are attached — verify the UI state before continuing."
+        : `AFTER screenshot failed (${afterError}) — the action DID run; take a desktop_screenshot to verify.`;
+    return {
+        content: [
+            textContent(`✓ ${describeAction(action)} — executed on ${adapter.platform}.${frontLine}${pointerLine}${hintLine}\n${afterLine}`),
+            imageContent(before),
+            ...(after ? [imageContent(after)] : []),
+        ],
+        details: {
+            ok: true,
+            action,
+            frontmostBefore: frontBefore,
+            frontmostAfter: frontAfter,
+            ...(pointerAfter ? { pointerAfter } : {}),
+            ...(pointerMissedTarget ? { diagnosticHint: NO_VISIBLE_CHANGE_HINT } : {}),
+            afterScreenshotError: afterError,
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Tool definitions
+// ---------------------------------------------------------------------------
+const SAFETY_DOC = "\n\nSafety: real input injection — enabling the extension is the explicit user consent, so actions run without a confirmation prompt. " +
+    "Opt-in SPECTRAL_DESKTOP_REQUIRE_CONFIRM=1 restores a per-session confirmation prompt (denied by default in headless mode). " +
+    'Every action returns BEFORE/AFTER screenshots for verification plus a pointer read-back; a "no visible change" hint flags actions that likely did not land (check Accessibility permission). ' +
+    "Env config: SPECTRAL_DESKTOP_DRY_RUN=1 previews without executing; " +
+    "SPECTRAL_DESKTOP_ALLOWLIST=Safari,Code restricts actions to matching focused apps. " +
+    "Allowlist entries are case-insensitive substrings matched in both directions against the frontmost app, and clicks are coordinate-based, so they may land on non-frontmost overlapping windows — the allowlist is best-effort, not a sandbox.";
+const numberParam = (description) => ({ type: "number", description });
+function actionTool(spec, build, deps) {
+    return {
+        name: spec.name,
+        label: spec.label,
+        description: spec.description + SAFETY_DOC,
+        promptSnippet: `\`${spec.name}\` — ${spec.label.toLowerCase()} (desktop input injection, safety-gated)`,
+        executionMode: "sequential",
+        parameters: {
+            type: "object",
+            properties: spec.parameters,
+        },
+        async execute(_toolCallId, _params) {
+            const built = build(_params);
+            if ("error" in built)
+                return errorResult(built.error, { tool: spec.name });
+            return runMutating(built.action, deps);
+        },
+    };
+}
+function requireNumber(params, key) {
+    const v = params[key];
+    if (typeof v !== "number" || !Number.isFinite(v))
+        return { error: `Parameter "${key}" must be a finite number.` };
+    return { value: v };
+}
+/** Max |dy|/|dx| in lines; keeps adapters in safe integer ranges (Windows: 100*120=12000 fits int32; Linux xdotool --repeat bounded). */
+const SCROLL_LINE_LIMIT = 100;
+/** Clamp (never error) scroll magnitude so wheel deltas cannot overflow int32. */
+function clampScrollLines(v) {
+    return Math.max(-SCROLL_LINE_LIMIT, Math.min(SCROLL_LINE_LIMIT, v));
+}
+function optionalNumber(params, key) {
+    const v = params[key];
+    if (v === undefined)
+        return { value: undefined };
+    if (typeof v !== "number" || !Number.isFinite(v))
+        return { error: `Parameter "${key}" must be a finite number.` };
+    return { value: v };
+}
+const BUTTONS = ["left", "right", "middle"];
+/** Build the 8 desktop tools supported by the adapter's capabilities. */
+export function createDesktopTools(deps) {
+    const caps = new Set(deps.adapter.capabilities);
+    const tools = [];
+    const has = (k) => caps.has(k);
+    if (has("mouse_move")) {
+        tools.push(actionTool({
+            name: "desktop_mouse_move",
+            kind: "mouse_move",
+            label: "Desktop Mouse Move",
+            description: "Move the mouse cursor to absolute screen coordinates (x, y).",
+            parameters: {
+                x: numberParam("Absolute X screen coordinate."),
+                y: numberParam("Absolute Y screen coordinate."),
+            },
+        }, (params) => {
+            const x = requireNumber(params, "x");
+            if ("error" in x)
+                return x;
+            const y = requireNumber(params, "y");
+            if ("error" in y)
+                return y;
+            return { action: { kind: "mouse_move", x: x.value, y: y.value } };
+        }, deps));
+    }
+    if (has("mouse_click")) {
+        tools.push(actionTool({
+            name: "desktop_mouse_click",
+            kind: "mouse_click",
+            label: "Desktop Mouse Click",
+            description: "Click the mouse at absolute screen coordinates. Moves the cursor there first. " +
+                "Use clickCount=2 for double-click. Use list_windows + desktop_screenshot to locate targets before clicking.",
+            parameters: {
+                x: numberParam("Absolute X screen coordinate."),
+                y: numberParam("Absolute Y screen coordinate."),
+                button: { type: "string", enum: [...BUTTONS], description: "Mouse button. Default: left." },
+                clickCount: numberParam("1=single, 2=double, 3=triple. Default: 1."),
+            },
+        }, (params) => {
+            const x = requireNumber(params, "x");
+            if ("error" in x)
+                return x;
+            const y = requireNumber(params, "y");
+            if ("error" in y)
+                return y;
+            const button = params.button === undefined ? "left" : params.button;
+            if (!BUTTONS.includes(button)) {
+                return { error: `Parameter "button" must be one of ${BUTTONS.join(", ")}.` };
+            }
+            let clickCount = 1;
+            if (params.clickCount !== undefined) {
+                if (typeof params.clickCount !== "number" || !Number.isInteger(params.clickCount) || params.clickCount < 1 || params.clickCount > 3) {
+                    return { error: 'Parameter "clickCount" must be an integer between 1 and 3.' };
+                }
+                clickCount = params.clickCount;
+            }
+            return {
+                action: {
+                    kind: "mouse_click",
+                    x: x.value,
+                    y: y.value,
+                    button: button,
+                    clickCount,
+                },
+            };
+        }, deps));
+    }
+    if (has("mouse_drag")) {
+        tools.push(actionTool({
+            name: "desktop_mouse_drag",
+            kind: "mouse_drag",
+            label: "Desktop Mouse Drag",
+            description: "Press the left button at (fromX, fromY), drag to (toX, toY), release.",
+            parameters: {
+                fromX: numberParam("X coordinate to press at."),
+                fromY: numberParam("Y coordinate to press at."),
+                toX: numberParam("X coordinate to release at."),
+                toY: numberParam("Y coordinate to release at."),
+            },
+        }, (params) => {
+            const coords = [];
+            for (const key of ["fromX", "fromY", "toX", "toY"]) {
+                const v = requireNumber(params, key);
+                if ("error" in v)
+                    return v;
+                coords.push(v.value);
+            }
+            return { action: { kind: "mouse_drag", fromX: coords[0], fromY: coords[1], toX: coords[2], toY: coords[3] } };
+        }, deps));
+    }
+    if (has("scroll")) {
+        tools.push(actionTool({
+            name: "desktop_scroll",
+            kind: "scroll",
+            label: "Desktop Scroll",
+            description: "Scroll the wheel. dy is vertical lines (positive = down, negative = up), dx is horizontal lines (positive = right, negative = left). " +
+                "Optionally move the cursor to (x, y) first.",
+            parameters: {
+                dy: numberParam("Vertical lines to scroll. Positive = down, negative = up."),
+                dx: numberParam("Horizontal lines to scroll. Positive = right, negative = left. Optional."),
+                x: numberParam("Optional X coordinate to move the cursor to first."),
+                y: numberParam("Optional Y coordinate to move the cursor to first."),
+            },
+        }, (params) => {
+            const dy = optionalNumber(params, "dy");
+            if ("error" in dy)
+                return dy;
+            const dx = optionalNumber(params, "dx");
+            if ("error" in dx)
+                return dx;
+            if (dy.value === undefined && dx.value === undefined) {
+                return { error: 'Provide "dy" (vertical lines, +down/-up) and/or "dx" (horizontal, +right/-left).' };
+            }
+            const x = optionalNumber(params, "x");
+            if ("error" in x)
+                return x;
+            const y = optionalNumber(params, "y");
+            if ("error" in y)
+                return y;
+            const hasX = x.value !== undefined;
+            const hasY = y.value !== undefined;
+            if (hasX !== hasY) {
+                return { error: 'Provide both "x" and "y", or neither.' };
+            }
+            return {
+                action: {
+                    kind: "scroll",
+                    dx: clampScrollLines(dx.value ?? 0),
+                    dy: clampScrollLines(dy.value ?? 0),
+                    ...(x.value !== undefined && y.value !== undefined ? { x: x.value, y: y.value } : {}),
+                },
+            };
+        }, deps));
+    }
+    if (has("type_text")) {
+        tools.push(actionTool({
+            name: "desktop_type_text",
+            kind: "type_text",
+            label: "Desktop Type Text",
+            description: "Type text into the currently focused window as if on a keyboard. Newlines become Return presses. " +
+                "Focus the right window first (desktop_focus_window or a click).",
+            parameters: {
+                text: { type: "string", description: "Text to type (unicode supported; sent as keyboard events)." },
+            },
+        }, (params) => {
+            const text = params.text;
+            if (typeof text !== "string" || text.length === 0)
+                return { error: 'Parameter "text" must be a non-empty string.' };
+            if (text.length > 5000)
+                return { error: 'Parameter "text" is too long (max 5000 chars); split into chunks.' };
+            return { action: { kind: "type_text", text } };
+        }, deps));
+    }
+    if (has("hotkey")) {
+        tools.push(actionTool({
+            name: "desktop_press_key",
+            kind: "hotkey",
+            label: "Desktop Press Key",
+            description: "Press a single key or chord, e.g. \"enter\", \"esc\", \"cmd+c\", \"ctrl+shift+t\". " +
+                "Modifier aliases: cmd/command/meta/super/win, ctrl/control, alt/option, shift.",
+            parameters: {
+                key: { type: "string", description: 'Key or chord, e.g. "return", "cmd+c", "ctrl+alt+delete".' },
+            },
+        }, (params) => {
+            const raw = params.key;
+            if (typeof raw !== "string" || raw.trim().length === 0)
+                return { error: 'Parameter "key" must be a non-empty string.' };
+            const chord = parseChord(raw);
+            if (!chord)
+                return { error: `Cannot parse key "${raw}". Use "mod+key" chords like "cmd+c" or a single key like "enter".` };
+            return { action: { kind: "hotkey", tokens: [...chord.modifiers, chord.key] } };
+        }, deps));
+        tools.push(actionTool({
+            name: "desktop_hotkey",
+            kind: "hotkey",
+            label: "Desktop Hotkey",
+            description: 'Press a hotkey combination from separate tokens, e.g. ["ctrl", "shift", "t"] or ["cmd", "c"]. ' +
+                "Exactly one non-modifier token is required. " +
+                "Modifier aliases: cmd/command/meta/super/win, ctrl/control, alt/option, shift.",
+            parameters: {
+                keys: {
+                    type: "array",
+                    items: { type: "string" },
+                    description: 'Chord tokens, e.g. ["ctrl", "shift", "esc"].',
+                },
+            },
+        }, (params) => {
+            const keys = params.keys;
+            if (!Array.isArray(keys) || keys.length === 0 || !keys.every((k) => typeof k === "string" && k.trim().length > 0)) {
+                return { error: 'Parameter "keys" must be a non-empty array of strings, e.g. ["ctrl", "c"].' };
+            }
+            const tokens = [];
+            let keyToken = null;
+            for (const entry of keys) {
+                const chord = parseChord(entry);
+                if (!chord)
+                    return { error: `Cannot parse key token "${entry}".` };
+                if (chord.modifiers.length > 0) {
+                    return { error: `Token "${entry}" contains modifiers — pass each modifier as its own array entry (e.g. ["ctrl", "c"]).` };
+                }
+                if (isModifierToken(chord.key)) {
+                    if (!tokens.includes(chord.key))
+                        tokens.push(chord.key);
+                }
+                else if (!keyToken) {
+                    keyToken = chord.key;
+                }
+                else {
+                    return { error: `Multiple keys given (${keyToken}, ${chord.key}) — a hotkey supports exactly one key plus modifiers.` };
+                }
+            }
+            if (!keyToken)
+                return { error: 'Parameter "keys" must include exactly one non-modifier key.' };
+            tokens.push(keyToken);
+            return { action: { kind: "hotkey", tokens } };
+        }, deps));
+    }
+    if (has("focus_window")) {
+        tools.push({
+            name: "desktop_focus_window",
+            label: "Desktop Focus Window",
+            description: "Bring an application window to the foreground by title (case-insensitive substring match from list_windows). " +
+                SAFETY_DOC,
+            promptSnippet: "`desktop_focus_window` — bring a window to the foreground (desktop input injection, safety-gated)",
+            executionMode: "sequential",
+            parameters: {
+                type: "object",
+                properties: {
+                    title: { type: "string", description: "Window title (substring, case-insensitive; from list_windows)." },
+                },
+            },
+            async execute(_toolCallId, _params) {
+                const params = _params;
+                const title = params.title;
+                if (typeof title !== "string" || title.trim().length === 0) {
+                    return errorResult('Parameter "title" must be a non-empty string.', { tool: "desktop_focus_window" });
+                }
+                const listWindowsFn = deps.listWindowsFn ?? listWindows;
+                let windows;
+                try {
+                    windows = await listWindowsFn();
+                }
+                catch (err) {
+                    return errorResult(`Cannot list windows: ${err instanceof Error ? err.message : String(err)}`, { tool: "desktop_focus_window" });
+                }
+                const search = title.toLowerCase();
+                const match = windows.find((w) => w.title.toLowerCase().includes(search))
+                    ?? windows.find((w) => w.owner.toLowerCase().includes(search));
+                if (!match) {
+                    return errorResult(`No window matching "${title}" found.`, { windowList: formatWindowList(windows, 25) });
+                }
+                return runMutating({ kind: "focus_window", title: match.title, owner: match.owner, windowId: match.id }, deps);
+            },
+        });
+    }
+    return tools;
+}
+// ---------------------------------------------------------------------------
+// Extension entry
+// ---------------------------------------------------------------------------
+export default async function desktopControlExtension(ext) {
+    const { adapter, reason } = await createAdapter();
+    if (!adapter) {
+        process.stderr.write(`[desktop-control] Not available: ${reason}\n`);
+        return;
+    }
+    const deps = { adapter };
+    const policy = deps.policy ?? new DesktopActionPolicy();
+    const tools = createDesktopTools(deps);
+    for (const tool of tools)
+        ext.registerTool(tool);
+    ext.on("tool_call", async (event, ctx) => {
+        if (!isDesktopControlTool(event.toolName))
+            return;
+        if (isDryRun())
+            return; // dry-run executes nothing, no gate needed
+        if (!isRequireConfirm())
+            return; // default: enabling the extension IS the consent — no prompt, no block
+        const decision = await policy.authorize({
+            dryRun: false,
+            hasUI: ctx.hasUI,
+            actionSummary: summarizeInput(event.toolName, event.input),
+            confirm: (title, message) => confirmWithOutcome(ctx, title, message),
+        });
+        if (!decision.allowed) {
+            return { block: true, reason: decision.reason };
+        }
+    });
+    process.stderr.write(`[desktop-control] Registered ${tools.map((t) => t.name).join(", ")} (platform: ${adapter.platform}${isDryRun() ? ", DRY-RUN" : ""}${isRequireConfirm() ? ", opt-in confirm" : ", no confirm prompt"}).\n`);
+}
+/**
+ * Run `ctx.ui.confirm` and translate the dialog outcome into the rich
+ * ConfirmResult contract used by DesktopActionPolicy:
+ *   - "deny" (a human answered No in an interactive dialog) -> explicit user
+ *     denial, which locks the session;
+ *   - "timeout" / "no_tty" / "cancelled" -> auto-deny for the current action
+ *     only (no user decision was made, so the session must not be locked);
+ *   - modes that never report an outcome: a plain `false` stays a user denial.
+ *
+ * A per-dialog timeout (SPECTRAL_CONFIRM_TIMEOUT_MS, default 5 min) is passed
+ * explicitly so an unresponsive RPC client cannot hang the dialog forever.
+ *
+ * Only used in opt-in confirm mode (SPECTRAL_DESKTOP_REQUIRE_CONFIRM=1|true);
+ * the default flow never reaches the confirmation dialog.
+ */
+async function confirmWithOutcome(ctx, title, message) {
+    const timeoutMs = parseConfirmTimeoutMs();
+    let outcome;
+    try {
+        const ok = await ctx.ui.confirm(title, message, {
+            timeout: timeoutMs,
+            onOutcome: (o) => {
+                outcome = o;
+            },
+        });
+        if (ok)
+            return { allowed: true };
+        switch (outcome) {
+            case "timeout":
+                return { allowed: false, failure: { kind: "timeout", timeoutMs } };
+            case "no_tty":
+                return { allowed: false, failure: { kind: "no_tty" } };
+            case "cancelled":
+                return { allowed: false, failure: { kind: "cancelled" } };
+            default:
+                // "deny" (explicit human refusal) or unreported outcome.
+                return false;
+        }
+    }
+    catch {
+        // Confirmation infrastructure failed (e.g. RPC channel closed): fail
+        // closed for this action without a user decision — do not lock.
+        return { allowed: false, failure: { kind: "cancelled" } };
+    }
+}
+/** Best-effort summary of raw tool input for the confirmation dialog. */
+function summarizeInput(toolName, input) {
+    const brief = (v) => {
+        const s = String(v ?? "");
+        return s.length > 60 ? s.slice(0, 59) + "…" : s;
+    };
+    switch (toolName) {
+        case "desktop_mouse_move":
+            return `move mouse to (${brief(input.x)}, ${brief(input.y)})`;
+        case "desktop_mouse_click":
+            return `click ${brief(input.button ?? "left")} at (${brief(input.x)}, ${brief(input.y)})`;
+        case "desktop_mouse_drag":
+            return `drag from (${brief(input.fromX)}, ${brief(input.fromY)}) to (${brief(input.toX)}, ${brief(input.toY)})`;
+        case "desktop_scroll":
+            return `scroll dy=${brief(input.dy)} dx=${brief(input.dx)}`;
+        case "desktop_type_text":
+            return `type "${brief(input.text)}"`;
+        case "desktop_press_key":
+            return `press ${brief(input.key)}`;
+        case "desktop_hotkey":
+            return `press hotkey ${Array.isArray(input.keys) ? input.keys.join("+") : brief(input.keys)}`;
+        case "desktop_focus_window":
+            return `focus window "${brief(input.title)}"`;
+        default:
+            return toolName;
+    }
+}

package/dist/extensions/desktop-control/keys.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Key name normalization + platform key tables.
+ *
+ * Chord notation: "cmd+c", "ctrl+shift+t", "enter", "alt+F4".
+ * Modifier aliases: cmd/command/meta/super/win -> "meta", option/alt -> "alt",
+ * control/ctrl -> "ctrl", plus the unicode glyphs ⌘ ⌥ ⇧ ⌃.
+ */
+export type ModifierToken = "ctrl" | "shift" | "alt" | "meta";
+/** macOS virtual keycodes for common keys (kVK_* from Events.h). */
+export declare const MACOS_KEYCODES: Record<string, number>;
+/** Shifted ASCII char -> base key (typed with FLAG_SHIFT). */
+export declare const MACOS_SHIFTED: Record<string, string>;
+/** CGEventFlagMask values. */
+export declare const MACOS_MODIFIER_FLAGS: Record<ModifierToken, number>;
+/** Windows virtual-key codes. */
+export declare const WINDOWS_VK: Record<string, number>;
+export declare const WINDOWS_MODIFIER_VK: Record<ModifierToken, number>;
+/** X keysym names for xdotool/wtype. */
+export declare const XDOTOOL_KEYS: Record<string, string>;
+export declare const XDOTOOL_MODIFIERS: Record<ModifierToken, string>;
+export declare function asModifier(raw: string): ModifierToken | null;
+/** Normalize a key name; returns the input lowercased when unknown (callers decide what to do). */
+export declare function normalizeKey(raw: string): string;
+export declare function isModifierToken(raw: string): boolean;
+/**
+ * Parse a chord like "cmd+shift+t" into modifiers + final key.
+ * Returns null for empty input or multiple non-modifier keys.
+ */
+export declare function parseChord(raw: string): {
+    modifiers: ModifierToken[];
+    key: string;
+} | null;
+//# sourceMappingURL=keys.d.ts.map

package/dist/extensions/desktop-control/keys.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keys.d.ts","sourceRoot":"","sources":["../../../src/extensions/desktop-control/keys.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;AAuD9D,oEAAoE;AACpE,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAajD,CAAC;AAEF,8DAA8D;AAC9D,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAIhD,CAAC;AAOF,8BAA8B;AAC9B,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAK9D,CAAC;AAEF,iCAAiC;AACjC,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAc7C,CAAC;AAEF,eAAO,MAAM,mBAAmB,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAK7D,CAAC;AAEF,wCAAwC;AACxC,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ/C,CAAC;AAEF,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAE3D,CAAC;AAEF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAE5D;AAED,mGAAmG;AACnG,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG;IAAE,SAAS,EAAE,aAAa,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAiB1F"}