humanish 0.105.0 → 0.106.0

package/dist/substrates/e2b/desktop-cdp.js

@@ -0,0 +1,211 @@
+// Chromium DevTools on a hosted E2B desktop, reached from inside the sandbox: browser state reads
+// and mobile emulation.
+import { chromeCdpProbeCommand, parseChromeCdpProbeOutput, } from "./cdp-probe.js";
+import { failureTail, toErrorMessage } from "../../evidence/redaction.js";
+import { readDetachedLog, startDetachedProcess } from "../detached.js";
+import { e2bShell } from "./shell.js";
+/**
+ * The URL / title / page-text / scroll observer behind stopWhen and task criteria. One probe per
+ * observation, run on the sandbox's python3 (see src/substrates/e2b/cdp-probe.ts for why not
+ * node: #514).
+ *
+ * "active": follow the participant to whatever tab they are driving now — never pin the state
+ * observer to the launch tab (a verification link that opened in a NEW tab left a pinned observer
+ * reading the old tab forever).
+ *
+ * `onUnavailable` fires ONCE, on the first probe that could not read the page, with the reason.
+ * The observer still degrades to `{}` for the loop; the callback is how a lane says out loud that
+ * url/text criteria are not being measured, instead of letting the funnel report 0/N (#514).
+ */
+export function makeChromeBrowserStateObserver(desktop, requestTimeoutMs, endpoint, options = {}) {
+    const { targetId, onUnavailable, drift } = options;
+    let reported = false;
+    let drifted = false;
+    const checkedTargets = new Set(drift === undefined ? [] : [drift.emulatedTargetId]);
+    const unavailable = (reason) => {
+        if (!reported) {
+            reported = true;
+            onUnavailable?.(reason);
+        }
+        return {};
+    };
+    const shell = e2bShell(desktop);
+    const checkLaterTarget = async (newTargetId) => {
+        if (drift === undefined || checkedTargets.has(newTargetId))
+            return;
+        checkedTargets.add(newTargetId);
+        // A read that fails or cannot be taken leaves the target's width unknown, which is drift.
+        const read = await shell
+            .run(chromeCdpProbeCommand({
+            ...endpoint,
+            targetId: newTargetId,
+            prefer: "pinned",
+            mode: "fidelity",
+        }), { requestTimeoutMs, timeoutMs: 5_000 })
+            .catch(() => undefined);
+        const fidelity = read === undefined || read.exitCode !== 0
+            ? undefined
+            : parseChromeCdpProbeOutput(read.stdout).fidelity;
+        if (fidelity !== undefined && fidelity.innerWidth === drift.expectedWidth) {
+            drift.onCovered?.(newTargetId, {
+                innerWidth: fidelity.innerWidth,
+                devicePixelRatio: fidelity.devicePixelRatio,
+                maxTouchPoints: fidelity.maxTouchPoints,
+            });
+            if (drift.expectTouch === true && fidelity.maxTouchPoints === 0 && !drifted) {
+                // The viewport followed; touch did not (yet): the holder reloads a later tab once after its
+                // first navigation commits, and this observation may have landed before that reload.
+                drifted = true;
+                drift.onDrift(`a later page target reports the ${fidelity.innerWidth} px viewport but navigator.maxTouchPoints 0 on its first observation; touch reaches a document only when it loads under the override`);
+            }
+            return;
+        }
+        if (drifted)
+            return;
+        drifted = true;
+        drift.onDrift(fidelity === undefined
+            ? "the participant drove a page target other than the emulated launch tab and that page's own read-back could not be taken; whether it laid out at the phone width is not known"
+            : `the participant drove a page target other than the emulated launch tab and that page reports a ${fidelity.innerWidth} px viewport where ${drift.expectedWidth} px was requested (DPR ${fidelity.devicePixelRatio}); the mobile user agent and touch events are browser-wide, the viewport override was not re-applied to it`);
+    };
+    return async () => {
+        let result;
+        try {
+            result = await shell.run(chromeCdpProbeCommand({
+                ...endpoint,
+                ...(targetId === undefined ? {} : { targetId }),
+                prefer: "active",
+                mode: "state",
+            }), { requestTimeoutMs, timeoutMs: 5_000 });
+        }
+        catch (error) {
+            // The executor discards a rejected probe, so a timeout is reported here or nowhere.
+            return unavailable(`probe failed: ${failureTail(toErrorMessage(error))}`);
+        }
+        if (result.exitCode !== 0) {
+            return unavailable(`probe exited ${result.exitCode}: ${failureTail(result.stderr || result.stdout)}`);
+        }
+        const parsed = parseChromeCdpProbeOutput(result.stdout);
+        if (parsed.unavailable !== undefined)
+            return unavailable(parsed.unavailable);
+        if (parsed.targetId !== undefined)
+            await checkLaterTarget(parsed.targetId);
+        return {
+            ...(parsed.url === undefined ? {} : { url: parsed.url }),
+            ...(parsed.title === undefined ? {} : { title: parsed.title }),
+            ...(parsed.text === undefined ? {} : { text: parsed.text }),
+            ...(parsed.scrollY === undefined ? {} : { scrollY: parsed.scrollY }),
+        };
+    };
+}
+/** The user agent a mobile-emulated lane presents unless the lab sets its own. */
+export const DEFAULT_MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1";
+/**
+ * Apply mobile emulation (#221) to the lane's launch page and read back what the page reports.
+ * Fails CLOSED: a request that cannot be applied throws, because a desktop run labelled mobile is
+ * the over-trust this feature exists to prevent. A read-back that cannot be taken is a warning
+ * (the emulation was applied; only the proof is missing).
+ */
+export async function applyMobileEmulation(desktop, requestTimeoutMs, endpoint, request, options = {}) {
+    const { targetId } = options;
+    const command = (mode) => chromeCdpProbeCommand({
+        ...endpoint,
+        ...(targetId === undefined ? {} : { targetId }),
+        prefer: "pinned",
+        mode,
+        emulation: request,
+    });
+    const shell = e2bShell(desktop);
+    const read = async () => {
+        const result = await shell.run(command("fidelity"), {
+            requestTimeoutMs,
+            timeoutMs: 15_000,
+        });
+        if (result.exitCode !== 0) {
+            return {
+                unavailable: `probe exited ${result.exitCode}: ${failureTail(result.stderr || result.stdout)}`,
+            };
+        }
+        return parseChromeCdpProbeOutput(result.stdout);
+    };
+    // The UA / touch / DPR overrides are bound to the DevTools session that set them and lapse the
+    // moment its socket closes (measured: only the viewport width survived a one-shot apply). So the
+    // applier stays attached for the lane's whole life as a detached process; the sandbox teardown
+    // ends it. Its first stdout line says what was applied.
+    const holderName = `mobile-emulation-${Date.now().toString(36)}`;
+    await startDetachedProcess(shell, {
+        name: holderName,
+        command: command("hold"),
+        requestTimeoutMs,
+    });
+    let announced;
+    for (let attempt = 0; attempt < 30 && announced === undefined; attempt += 1) {
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        const log = await readDetachedLog(shell, holderName, requestTimeoutMs).catch(() => "");
+        const line = log.split("\n").find((candidate) => candidate.trim().startsWith("{"));
+        if (line !== undefined)
+            announced = parseChromeCdpProbeOutput(line);
+    }
+    if (announced === undefined) {
+        throw new Error("mobile emulation could not be applied: the in-sandbox applier printed nothing within 15 s");
+    }
+    if (announced.unavailable !== undefined) {
+        throw new Error(`mobile emulation could not be applied (${announced.unavailable}); applied before failing: ${(announced.applied ?? []).join(", ") || "nothing"}`);
+    }
+    const applied = announced;
+    // Viewport/touch read-back proves context settings, not gesture equivalence. Two hosted
+    // replicas and a native-X conversion-toggle control reproduced reset click counts (#676).
+    const warnings = request.touch
+        ? [
+            "Mobile emulation uses desktop pointer-to-touch conversion, which can differ for repeated taps. Confirm gesture failures with direct or native touch input before attributing them to the app.",
+        ]
+        : [];
+    // The reload inside the applier takes a moment; the read-back is retried until the page reports
+    // the requested viewport and user agent, so a slow page does not read as "no proof".
+    let readBack = await read();
+    for (let attempt = 0; attempt < 20 &&
+        (readBack.fidelity === undefined ||
+            readBack.fidelity.innerWidth !== request.width ||
+            !readBack.fidelity.userAgent.includes(request.userAgent.slice(0, 24))); attempt += 1) {
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        readBack = await read();
+    }
+    const fidelityRead = readBack;
+    const requested = {
+        width: request.width,
+        height: request.height,
+        deviceScaleFactor: request.deviceScaleFactor,
+        touch: request.touch,
+        userAgent: request.userAgent,
+    };
+    const emulatedTargetId = applied.targetId ?? fidelityRead.targetId;
+    if (fidelityRead.fidelity === undefined) {
+        warnings.push(`Mobile emulation was applied but the page's own report could not be read (${fidelityRead.unavailable ?? "no fidelity read"}); desktopGeometry.fidelity carries the request without a resolved block.`);
+        return {
+            fidelity: { tier: "mobile-emulated", requested, applied: applied.applied ?? [] },
+            warnings,
+            holderName,
+            ...(emulatedTargetId === undefined ? {} : { targetId: emulatedTargetId }),
+        };
+    }
+    const resolved = { ...fidelityRead.fidelity, source: "cdp" };
+    if (resolved.innerWidth !== request.width) {
+        warnings.push(`Mobile emulation requested a ${request.width} px viewport; the page reports ${resolved.innerWidth} px.`);
+    }
+    if (resolved.devicePixelRatio !== request.deviceScaleFactor) {
+        warnings.push(`Mobile emulation requested devicePixelRatio ${request.deviceScaleFactor}; the page reports ${resolved.devicePixelRatio}.`);
+    }
+    if (request.touch && resolved.maxTouchPoints === 0) {
+        warnings.push("Mobile emulation requested touch; the page reports navigator.maxTouchPoints 0.");
+    }
+    if (!resolved.userAgent.includes("Mobile") &&
+        !resolved.userAgent.includes("Android") &&
+        !resolved.userAgent.includes("iPhone")) {
+        warnings.push("Mobile emulation requested a mobile user agent; the page reports a desktop one.");
+    }
+    return {
+        fidelity: { tier: "mobile-emulated", requested, applied: applied.applied ?? [], resolved },
+        warnings,
+        holderName,
+        ...(emulatedTargetId === undefined ? {} : { targetId: emulatedTargetId }),
+    };
+}

package/dist/substrates/e2b/desktop-executor.d.ts

@@ -0,0 +1,92 @@
+import type { CuaExecutor, CuaObservation } from "../../actors/computer-use/loop.js";
+/**
+ * The minimal slice of the @e2b/desktop Sandbox the executor depends on: a structural subset of
+ * the SDK class (checked against 2.4.0), so a Sandbox satisfies it with no adapter. Methods are typed to return `Promise<void> | void` (and the
+ * screenshot bytes likewise) so a synchronous fake also satisfies the port; the
+ * executor awaits every call, which is correct for both sync and async returns.
+ */
+export interface E2BDesktopLike {
+    /** Optional command surface used only for best-effort substrate fallbacks. */
+    commands?: {
+        run(command: string, options?: {
+            requestTimeoutMs?: number;
+            timeoutMs?: number;
+        }): Promise<{
+            exitCode?: number;
+            stderr?: string;
+            stdout?: string;
+        }>;
+    };
+    /** Optional file surface used to transfer typed text without shell-quoting it. */
+    files?: {
+        write(path: string, data: string | ArrayBuffer, options?: {
+            requestTimeoutMs?: number;
+            useOctetStream?: boolean;
+        }): Promise<unknown>;
+    };
+    /** Capture the current desktop frame as PNG bytes (default/'bytes' overload). */
+    screenshot(): Promise<Uint8Array | Buffer> | Uint8Array | Buffer;
+    /** Left click, optionally moving to (x, y) first. */
+    leftClick(x?: number, y?: number): Promise<void> | void;
+    /** Right click, optionally moving to (x, y) first. */
+    rightClick(x?: number, y?: number): Promise<void> | void;
+    /** Middle click, optionally moving to (x, y) first. */
+    middleClick(x?: number, y?: number): Promise<void> | void;
+    /** Double left click, optionally moving to (x, y) first. */
+    doubleClick(x?: number, y?: number): Promise<void> | void;
+    /** Move the mouse to the given coordinates. */
+    moveMouse(x: number, y: number): Promise<void> | void;
+    /** Optional fresh pointer read; older/custom desktops retain coordinate-bearing clicks. */
+    getCursorPosition?(): Promise<{
+        x: number;
+        y: number;
+    }> | {
+        x: number;
+        y: number;
+    };
+    /** Scroll the mouse wheel vertically by amount ticks in a direction. */
+    scroll(direction?: "up" | "down", amount?: number): Promise<void> | void;
+    /** Write text at the current cursor position (the SDK's typing method). */
+    write(text: string): Promise<void> | void;
+    /** Press a key or chord (the SDK's key method); accepts the keys array. */
+    press(key: string | string[]): Promise<void> | void;
+    /** Drag from one coordinate tuple to another. */
+    drag(from: [number, number], to: [number, number]): Promise<void> | void;
+    /** Wait for the given number of milliseconds. */
+    wait(ms: number): Promise<void> | void;
+}
+export interface E2BDesktopExecutorOptions {
+    /** Fallback wait when a wait action carries no ms. Default 500. */
+    defaultWaitMs?: number;
+    /**
+     * Pixels of CuaAction scroll dy per one SDK scroll tick. The executor maps
+     * abs(dy) / scrollAmountPerTick to the SDK's integer `amount` (floored at 1 for
+     * any nonzero scroll). Default 100.
+     */
+    scrollAmountPerTick?: number;
+    /**
+     * Optional runtime-only browser state probe. Used for deterministic stopWhen guards. The loop
+     * never persists raw URL/title/text; it only uses them in memory to decide whether to stop.
+     */
+    observeBrowserState?: () => Promise<Pick<CuaObservation, "url" | "title" | "text" | "scrollY">>;
+}
+/** The stage of a `type` action that failed. */
+export type CuaTypePhase = "desktop-write" | "text-tempfile" | "text-command";
+/**
+ * A `type` action that failed. Typed text can be a credential (a login step typing a subject-env
+ * password), so the message names the phase and, for xdotool, its exit code only: no text, no
+ * temp-file path, no substrate output. The loop records `.name` + `.message` into the actor trace.
+ */
+export declare class CuaTypeError extends Error {
+    readonly phase: CuaTypePhase;
+    readonly exitCode?: number;
+    constructor(phase: CuaTypePhase, exitCode?: number);
+}
+/**
+ * Create a CuaExecutor backed by an E2B desktop (or any structural E2BDesktopLike,
+ * e.g. a CI fake). observe() captures a frame and computes its perceptual
+ * signature; execute() dispatches one CuaAction to the matching desktop method.
+ * Every desktop call is awaited so sync and async implementations both work. No
+ * screenshot or action is ever logged.
+ */
+export declare function createE2BDesktopExecutor(desktop: E2BDesktopLike, options?: E2BDesktopExecutorOptions): CuaExecutor;

package/dist/substrates/e2b/desktop-executor.js

@@ -0,0 +1,302 @@
+import { shellQuote } from "../shell.js";
+import { perceptualSignature } from "../../evidence/frame-signature.js";
+import { commandFailureInfo } from "../command-failure.js";
+import { CuaExecutorError } from "../../actors/computer-use/executor-error.js";
+import { xdotoolHeldModifiers } from "../../guest/desktop-keys.js";
+const DEFAULT_WAIT_MS = 500;
+const DEFAULT_SCROLL_AMOUNT_PER_TICK = 100;
+const TYPE_COMMAND_TIMEOUT_MS = 15_000;
+const HELD_KEYS_TIMEOUT_MS = 15_000;
+const CURSOR_READ_TIMEOUT_MS = 500;
+/**
+ * The stock image's xdotool waits ~15s on a synchronized move to its current position (#681).
+ * Only an exact integer match can omit that move; fractional actions keep the SDK's own
+ * coordinate conversion. Read every time: another action or sequential role may move it.
+ * The SDK's getCursorPosition (2.3 through 2.4.0) takes no timeout or signal. Bound our wait, observe late
+ * rejection, and leave the underlying read-only request alone; it cannot dispatch a click.
+ */
+function cursorAlreadyAt(desktop, x, y, signal) {
+    if (!Number.isSafeInteger(x) || !Number.isSafeInteger(y) || x < 0 || y < 0)
+        return Promise.resolve(false);
+    let readPosition;
+    try {
+        readPosition = desktop.getCursorPosition;
+    }
+    catch {
+        return Promise.resolve(false);
+    }
+    if (typeof readPosition !== "function" || signal?.aborted)
+        return Promise.resolve(false);
+    return new Promise((resolve) => {
+        let settled = false;
+        const finish = (matches) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", onAbort);
+            resolve(matches);
+        };
+        const onAbort = () => finish(false);
+        const timer = setTimeout(() => finish(false), CURSOR_READ_TIMEOUT_MS);
+        signal?.addEventListener("abort", onAbort, { once: true });
+        Promise.resolve()
+            .then(() => (settled ? undefined : readPosition.call(desktop)))
+            .then((position) => {
+            if (settled)
+                return;
+            if (position === null || typeof position !== "object") {
+                finish(false);
+                return;
+            }
+            // A custom port can return malformed values or throwing accessors; all are uncertainty.
+            try {
+                const point = position;
+                finish(Number.isSafeInteger(point.x) &&
+                    Number.isSafeInteger(point.y) &&
+                    point.x === x &&
+                    point.y === y);
+            }
+            catch {
+                finish(false);
+            }
+        }, () => finish(false));
+    });
+}
+/** Coerce screenshot bytes (Uint8Array or Buffer) to a Node Buffer without copying when possible. */
+function toBuffer(bytes) {
+    return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
+}
+/**
+ * A `type` action that failed. Typed text can be a credential (a login step typing a subject-env
+ * password), so the message names the phase and, for xdotool, its exit code only: no text, no
+ * temp-file path, no substrate output. The loop records `.name` + `.message` into the actor trace.
+ */
+export class CuaTypeError extends Error {
+    phase;
+    exitCode;
+    constructor(phase, exitCode) {
+        super(`type failed at ${phase}${exitCode === undefined ? "" : ` (exit ${exitCode})`}`);
+        this.name = "CuaTypeError";
+        this.phase = phase;
+        if (exitCode !== undefined)
+            this.exitCode = exitCode;
+    }
+}
+/** The SDK write's per-character delay, kept so typing keeps its pace. */
+const XDOTOOL_TYPE_DELAY_MS = 75;
+const TYPE_DIRECTORY = /^\/tmp\/humanish-type-[A-Za-z0-9]+$/;
+function canTypeText(desktop) {
+    return desktop.commands !== undefined && desktop.files !== undefined;
+}
+/**
+ * Type `text` with one xdotool command in the UTF-8 locale, reading it from a file.
+ *
+ * The stock desktop runs commands in the C locale, where xdotool types the characters before the
+ * first non-ASCII one and then fails; the SDK write splits text into 25-unit chunks and fails the
+ * same way partway through. Either would leave a prefix typed, so there is one attempt and no
+ * retry. The text is written to a 0600 file in a directory `mktemp -d` makes, passed to xdotool by
+ * path (never through the shell), and the directory is removed on every exit.
+ */
+async function typeText(desktop, text) {
+    const { commands, files } = desktop;
+    const quick = { requestTimeoutMs: TYPE_COMMAND_TIMEOUT_MS, timeoutMs: TYPE_COMMAND_TIMEOUT_MS };
+    let directory;
+    try {
+        const made = await commands.run('umask 077 && d=$(mktemp -d /tmp/humanish-type-XXXXXXXX) && : > "$d/text" && printf \'%s\' "$d"', quick);
+        directory = made.stdout?.trim();
+    }
+    catch {
+        throw new CuaTypeError("text-tempfile");
+    }
+    if (directory === undefined || !TYPE_DIRECTORY.test(directory))
+        throw new CuaTypeError("text-tempfile");
+    const file = `${directory}/text`;
+    try {
+        try {
+            await files.write(file, text, { requestTimeoutMs: TYPE_COMMAND_TIMEOUT_MS });
+        }
+        catch {
+            throw new CuaTypeError("text-tempfile");
+        }
+        // xdotool waits the delay after every character, so a long text needs a longer budget.
+        const timeoutMs = TYPE_COMMAND_TIMEOUT_MS + [...text].length * XDOTOOL_TYPE_DELAY_MS;
+        let result;
+        try {
+            result = await commands.run(`DISPLAY="\${DISPLAY:-:0}" LC_ALL=C.UTF-8 xdotool type --delay ${XDOTOOL_TYPE_DELAY_MS} --file ${shellQuote(file)}`, { requestTimeoutMs: timeoutMs, timeoutMs });
+        }
+        catch (error) {
+            // The SDK throws on a non-zero exit; only its exit code is kept.
+            throw new CuaTypeError("text-command", commandFailureInfo(error).exitCode);
+        }
+        // A structural fake may return a non-zero exit instead of throwing, as the SDK does.
+        if (result?.exitCode !== undefined && result.exitCode !== 0)
+            throw new CuaTypeError("text-command", result.exitCode);
+    }
+    finally {
+        await commands.run(`rm -rf -- ${shellQuote(directory)}`, quick).catch(() => undefined);
+    }
+}
+/**
+ * Create a CuaExecutor backed by an E2B desktop (or any structural E2BDesktopLike,
+ * e.g. a CI fake). observe() captures a frame and computes its perceptual
+ * signature; execute() dispatches one CuaAction to the matching desktop method.
+ * Every desktop call is awaited so sync and async implementations both work. No
+ * screenshot or action is ever logged.
+ */
+export function createE2BDesktopExecutor(desktop, options = {}) {
+    const defaultWaitMs = options.defaultWaitMs ?? DEFAULT_WAIT_MS;
+    const scrollAmountPerTick = options.scrollAmountPerTick ?? DEFAULT_SCROLL_AMOUNT_PER_TICK;
+    return {
+        async observe() {
+            // Probe browser state before the screenshot so a deterministic stopWhen match
+            // and its persisted frame describe the same settled surface as closely as the
+            // desktop substrate allows.
+            const browserState = await options.observeBrowserState?.().catch(() => undefined);
+            const raw = await desktop.screenshot();
+            const screenshot = toBuffer(raw);
+            return {
+                screenshot,
+                stateSignature: perceptualSignature(screenshot),
+                ...(browserState?.url === undefined ? {} : { url: browserState.url }),
+                ...(browserState?.title === undefined ? {} : { title: browserState.title }),
+                ...(browserState?.text === undefined ? {} : { text: browserState.text }),
+                ...(browserState?.scrollY === undefined ? {} : { scrollY: browserState.scrollY }),
+            };
+        },
+        async execute(action, signal) {
+            signal?.throwIfAborted();
+            const held = "heldKeys" in action ? xdotoolHeldModifiers(action.heldKeys) : undefined;
+            // A pointer action that sends no input presses no keys either, as on the guest desktop.
+            if (held === undefined || sendsNoInput(action))
+                return dispatch(action, signal);
+            return withHeldModifiers(desktop, held, () => dispatch(action, signal));
+        },
+    };
+    async function dispatch(action, signal) {
+        switch (action.kind) {
+            case "click": {
+                const button = action.button ?? "left";
+                if (button === "right") {
+                    await desktop.rightClick(action.x, action.y);
+                }
+                else if (button === "middle") {
+                    await desktop.middleClick(action.x, action.y);
+                }
+                else {
+                    const alreadyAtTarget = await cursorAlreadyAt(desktop, action.x, action.y, signal);
+                    signal?.throwIfAborted();
+                    if (alreadyAtTarget)
+                        await desktop.leftClick();
+                    else
+                        await desktop.leftClick(action.x, action.y);
+                }
+                return;
+            }
+            case "double_click": {
+                const alreadyAtTarget = await cursorAlreadyAt(desktop, action.x, action.y, signal);
+                signal?.throwIfAborted();
+                if (alreadyAtTarget)
+                    await desktop.doubleClick();
+                else
+                    await desktop.doubleClick(action.x, action.y);
+                return;
+            }
+            case "move":
+                await desktop.moveMouse(action.x, action.y);
+                return;
+            case "scroll": {
+                // The real SDK scroll is vertical only and takes no position, so move the
+                // cursor to the scroll point first (the action targets a specific spot;
+                // scrolling at the wrong cursor position would scroll the wrong panel).
+                // dx (horizontal) has no SDK target and is ignored; a zero dy is a no-op.
+                if (action.dy === 0)
+                    return;
+                await desktop.moveMouse(action.x, action.y);
+                const direction = action.dy > 0 ? "down" : "up";
+                const amount = Math.max(1, Math.round(Math.abs(action.dy) / scrollAmountPerTick));
+                await desktop.scroll(direction, amount);
+                return;
+            }
+            case "type": {
+                if (canTypeText(desktop)) {
+                    await typeText(desktop, action.text);
+                    return;
+                }
+                // A desktop without command and file surfaces keeps the SDK write, with no retry: a
+                // failed write may already have typed part of the text. Its error could echo the
+                // command, and so the text, so it is replaced.
+                try {
+                    await desktop.write(action.text);
+                }
+                catch {
+                    throw new CuaTypeError("desktop-write");
+                }
+                return;
+            }
+            case "keypress":
+                // The SDK press() accepts a string[] directly; pass the keys through so
+                // a chord (e.g. ["Control", "a"]) is pressed together, not in sequence.
+                await desktop.press(action.keys);
+                return;
+            case "drag": {
+                // The SDK drag takes two endpoints, not an N-point path: drag from the
+                // first to the last point. 0 points is a safe no-op; 1 point has no
+                // distinct endpoint, so it is also a no-op (no spurious click/move).
+                const path = action.path;
+                if (path.length < 2)
+                    return;
+                const from = path[0];
+                const to = path[path.length - 1];
+                if (from === undefined || to === undefined)
+                    return;
+                await desktop.drag([from.x, from.y], [to.x, to.y]);
+                return;
+            }
+            case "wait":
+                await desktop.wait(action.ms ?? defaultWaitMs);
+                return;
+            case "screenshot":
+                // No-op: the loop calls observe() separately to capture each frame, so
+                // capturing here would double-capture. Leave the desktop untouched.
+                return;
+            case "speak":
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+        }
+    }
+}
+function sendsNoInput(action) {
+    return ((action.kind === "scroll" && action.dy === 0) ||
+        (action.kind === "drag" && action.path.length < 2));
+}
+/**
+ * Hold `chord` (xdotool modifier names) down for `run`, then release it whether `run` succeeded
+ * or not. The SDK has no key-down method, so this uses the sandbox's command channel, where the
+ * SDK's own input methods also run xdotool. A release that fails leaves the desktop in an unknown
+ * state, so it ends the session.
+ */
+async function withHeldModifiers(desktop, chord, run) {
+    const commands = desktop.commands;
+    if (!commands)
+        throw new CuaExecutorError("action_rejected", "not_dispatched");
+    const options = { requestTimeoutMs: HELD_KEYS_TIMEOUT_MS, timeoutMs: HELD_KEYS_TIMEOUT_MS };
+    let failed = false;
+    let failure;
+    try {
+        await commands.run(`xdotool keydown ${chord}`, options);
+        await run();
+    }
+    catch (error) {
+        failed = true;
+        failure = error;
+    }
+    // A keydown that failed may still have pressed some keys, so the release always runs.
+    try {
+        await commands.run(`xdotool keyup ${chord}`, options);
+    }
+    catch {
+        throw new CuaExecutorError("execution_failed", "outcome_uncertain");
+    }
+    if (failed)
+        throw failure;
+}