humanish 0.104.3 → 0.106.0

package/dist/guest/desktop-executor.js

@@ -0,0 +1,256 @@
+import { perceptualSignature } from "../evidence/frame-signature.js";
+import { setTimeout as delay } from "node:timers/promises";
+import { BROWSER_CONTROL_LIMITS, validateBrowserControlAction, validateBrowserControlPng, } from "../browser-control/protocol.js";
+import { CuaExecutorError, isCuaExecutorError } from "../actors/computer-use/executor-error.js";
+import { xdotoolChord, xdotoolHeldModifiers } from "./desktop-keys.js";
+/** Headed X11 input and full-frame captures. Browser metadata is deliberately absent. */
+export function createGuestDesktopExecutor(options) {
+    const { width, height, tools, authoritySignal } = options;
+    if (!isValidFrameSize(width, height))
+        throw new CuaExecutorError("invalid_request", "not_dispatched");
+    const context = {
+        width,
+        height,
+        tools,
+        authoritySignal,
+        onTerminal: () => options.onTerminal(),
+    };
+    const state = { busy: false, terminal: false };
+    return {
+        stallRecovery: "fail_closed",
+        observe: () => observeFrame(context, state),
+        execute: (action, callerSignal) => executeAction(context, state, action, callerSignal),
+    };
+}
+/** Whether a frame of this size fits the browser-control limits. */
+export function isValidFrameSize(width, height) {
+    return ([width, height].every((n) => Number.isSafeInteger(n) && n > 0 && n <= BROWSER_CONTROL_LIMITS.dimension) && width * height <= BROWSER_CONTROL_LIMITS.pixels);
+}
+/** Ends the session once. A throw from the owner's onTerminal leaves it ended. */
+function terminate(state, context) {
+    if (state.terminal)
+        return;
+    state.terminal = true;
+    try {
+        context.onTerminal();
+    }
+    catch {
+        /* Terminal state remains closed even if the owner fails. */
+    }
+}
+function assertOpen(state, signal, dispatched = false) {
+    if (state.terminal || signal.aborted)
+        throw new CuaExecutorError("session_revoked", dispatched ? "outcome_uncertain" : "not_dispatched");
+}
+function begin(state, signal) {
+    assertOpen(state, signal);
+    if (state.busy)
+        throw new CuaExecutorError("executor_busy", "not_dispatched");
+    state.busy = true;
+}
+/** Captures one full frame and checks it is a PNG of the executor's size. */
+async function observeFrame(context, state) {
+    const { tools, authoritySignal, width, height } = context;
+    begin(state, authoritySignal);
+    try {
+        const screenshot = await tools.capture(authoritySignal);
+        assertOpen(state, authoritySignal);
+        validateBrowserControlPng(screenshot);
+        if (screenshot.readUInt32BE(16) !== width || screenshot.readUInt32BE(20) !== height) {
+            throw new CuaExecutorError("invalid_response", "not_dispatched");
+        }
+        return { screenshot, stateSignature: perceptualSignature(screenshot) };
+    }
+    catch (error) {
+        terminate(state, context);
+        throw isCuaExecutorError(error)
+            ? error
+            : new CuaExecutorError("execution_failed", "not_dispatched");
+    }
+    finally {
+        state.busy = false;
+    }
+}
+/**
+ * Runs one action: validates and plans all of it first, pastes text, waits, then sends each
+ * native command with the authority checked immediately before it.
+ */
+async function executeAction(context, state, value, callerSignal) {
+    const { tools, authoritySignal } = context;
+    const signal = callerSignal ? AbortSignal.any([authoritySignal, callerSignal]) : authoritySignal;
+    begin(state, signal);
+    const progress = {
+        dispatched: false,
+        preparing: false,
+        keysHeld: undefined,
+        text: undefined,
+    };
+    try {
+        const action = validateBrowserControlAction(value);
+        const commands = planActionInput(context, action); // Validate the entire action before any preparation/input.
+        if (action.kind === "type" && action.text.length) {
+            progress.preparing = true;
+            progress.text = await tools.prepareText(action.text, signal);
+            assertOpen(state, signal);
+            progress.dispatched = true;
+            try {
+                await progress.text.paste();
+            }
+            catch (error) {
+                // This transaction has no earlier input. Its private bridge can
+                // prove that no insertion or native input was sent.
+                if (isCuaExecutorError(error) && error.disposition === "not_dispatched")
+                    progress.dispatched = false;
+                throw error;
+            }
+        }
+        if (action.kind === "wait")
+            await delay(action.ms ?? 250, undefined, { signal });
+        for (const command of commands) {
+            assertOpen(state, signal, progress.dispatched);
+            // No await between authority check and the native port invocation.
+            progress.dispatched = true;
+            // A keydown can press some keys before it fails; a failed keyup is not retried.
+            if (command[0] === "keydown")
+                progress.keysHeld = command[1];
+            if (command[0] === "keyup")
+                progress.keysHeld = undefined;
+            await tools.input(command, signal);
+        }
+        assertOpen(state, signal, progress.dispatched);
+    }
+    catch (error) {
+        // A failed pointer input still releases its modifiers. After revocation nothing more is
+        // sent, as for the mouse button below.
+        if (progress.keysHeld !== undefined && !signal.aborted) {
+            try {
+                await tools.input(["keyup", progress.keysHeld], signal);
+            }
+            catch {
+                // The session ends below either way.
+            }
+        }
+        if (failureEndsSession(progress, error, signal))
+            terminate(state, context);
+        // Do not send mouseup/key-release after revocation: it can itself click/drop. failureError
+        // reads the signal after terminate, because onTerminal can abort it.
+        throw failureError(progress.dispatched, error, signal);
+    }
+    finally {
+        try {
+            await progress.text?.close();
+        }
+        catch {
+            terminate(state, context);
+            // oxlint-disable-next-line no-unsafe-finally -- a text channel that fails to close makes the action outcome unknowable
+            throw new CuaExecutorError("execution_failed", progress.dispatched ? "outcome_uncertain" : "not_dispatched");
+        }
+        finally {
+            state.busy = false;
+        }
+    }
+}
+/**
+ * Whether a failed action ends the session: anything dispatched, an aborted signal, an error the
+ * executor did not raise, or a failed text preparation other than a refusal before dispatch. It
+ * reads signal.aborted only when the earlier conditions do not decide, as the inline check did.
+ */
+export function failureEndsSession(progress, error, signal) {
+    return ((progress.preparing &&
+        !(isCuaExecutorError(error) &&
+            error.code === "action_rejected" &&
+            error.disposition === "not_dispatched")) ||
+        progress.dispatched ||
+        signal.aborted ||
+        !isCuaExecutorError(error));
+}
+/**
+ * The error a failed action throws: outcome_uncertain once anything was dispatched, then
+ * session_revoked when the signal aborted, then the executor's own error or execution_failed. It
+ * reads signal.aborted only for an undispatched failure, as the inline mapping did.
+ */
+export function failureError(dispatched, error, signal) {
+    if (dispatched)
+        return new CuaExecutorError(isCuaExecutorError(error) ? error.code : "execution_failed", "outcome_uncertain");
+    if (signal.aborted)
+        return new CuaExecutorError("session_revoked", "not_dispatched");
+    return isCuaExecutorError(error)
+        ? error
+        : new CuaExecutorError("execution_failed", "not_dispatched");
+}
+/**
+ * The native commands for an action, wrapped in keydown/keyup when it holds modifiers. It throws
+ * the refusal for an action the desktop cannot take, before anything is sent.
+ */
+export function planActionInput(frame, action) {
+    const commands = actionCommands(frame, action);
+    const held = "heldKeys" in action ? xdotoolHeldModifiers(action.heldKeys) : undefined;
+    return held === undefined || commands.length === 0
+        ? commands
+        : [["keydown", held], ...commands, ["keyup", held]];
+}
+/** A pointer position as xdotool arguments. */
+function framePoint(frame, x, y) {
+    const { width, height } = frame;
+    // Reject out-of-frame input instead of clicking a clamped, unintended target.
+    if (x < 0 || y < 0 || x >= width || y >= height)
+        throw new CuaExecutorError("action_rejected", "not_dispatched");
+    return [String(Math.min(width - 1, Math.round(x))), String(Math.min(height - 1, Math.round(y)))];
+}
+/** The native commands for an action, before any held modifiers. */
+function actionCommands(frame, action) {
+    switch (action.kind) {
+        case "move":
+            return [["mousemove", ...framePoint(frame, action.x, action.y)]];
+        case "click":
+            return [
+                ["mousemove", ...framePoint(frame, action.x, action.y)],
+                ["click", { left: "1", middle: "2", right: "3" }[action.button ?? "left"]],
+            ];
+        case "double_click":
+            return [
+                ["mousemove", ...framePoint(frame, action.x, action.y)],
+                ["click", "--repeat", "2", "--delay", "100", "1"],
+            ];
+        case "keypress":
+            return [["key", "--clearmodifiers", xdotoolChord(action.keys)]];
+        case "type": {
+            // Text transports require exact UTF-8; NUL truncation and lone surrogates
+            // must be refused instead of silently changing the requested text.
+            if (action.text.includes("\0") ||
+                Buffer.from(action.text, "utf8").toString("utf8") !== action.text)
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+            return [];
+        }
+        case "drag": {
+            const points = action.path.map((p) => framePoint(frame, p.x, p.y));
+            if (points.length < 2)
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+            return [
+                ["mousemove", ...points[0]],
+                ["mousedown", "1"],
+                ...points.slice(1).map((p) => ["mousemove", ...p]),
+                ["mouseup", "1"],
+            ];
+        }
+        case "scroll": {
+            const origin = framePoint(frame, action.x, action.y);
+            // Native X11 wheel steps do not promise exact pixel scroll distances.
+            const horizontal = Math.ceil(Math.abs(action.dx) / 120), vertical = Math.ceil(Math.abs(action.dy) / 120);
+            if (horizontal + vertical > 100)
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+            return horizontal + vertical === 0
+                ? []
+                : [
+                    ["mousemove", ...origin],
+                    ...Array.from({ length: horizontal }, () => ["click", action.dx > 0 ? "7" : "6"]),
+                    ...Array.from({ length: vertical }, () => ["click", action.dy > 0 ? "5" : "4"]),
+                ];
+        }
+        case "wait":
+        case "screenshot":
+            return [];
+        case "speak":
+            throw new CuaExecutorError("action_rejected", "not_dispatched");
+    }
+}

package/dist/guest/desktop-keys.d.ts

@@ -0,0 +1,8 @@
+/** A keypress chord in xdotool syntax. Unknown or repeated keys are refused before dispatch. */
+export declare function xdotoolChord(keys: readonly string[]): string;
+/**
+ * The xdotool chord for a pointer action's held keys, or undefined when it holds none. Only
+ * modifiers can be held, which is the use the OpenAI computer-use docs describe; an ordinary key
+ * held down could type into the page.
+ */
+export declare function xdotoolHeldModifiers(keys: readonly string[] | undefined): string | undefined;

package/dist/guest/desktop-keys.js

@@ -0,0 +1,89 @@
+import { CuaExecutorError } from "../actors/computer-use/executor-error.js";
+// Key names for the xdotool-driven desktops: the guest's headed display and the E2B sandbox. Both
+// hand these strings to xdotool, which has a command language, so only names from these tables
+// ever reach it. The guest runtime ships this module, so it lives in src/guest/, which CI's
+// guest-desktop job and scripts/guest-desktop-proof.mjs key on.
+/** The modifiers a pointer action may hold, keyed by the upper-cased provider name. */
+const modifierNames = Object.freeze({
+    CTRL: "ctrl",
+    CONTROL: "ctrl",
+    ALT: "alt",
+    SHIFT: "shift",
+    META: "super",
+    SUPER: "super",
+    CMD: "super",
+});
+const keyNames = Object.freeze({
+    ...modifierNames,
+    ENTER: "Return",
+    RETURN: "Return",
+    TAB: "Tab",
+    ESC: "Escape",
+    ESCAPE: "Escape",
+    SPACE: "space",
+    " ": "space",
+    BACKSPACE: "BackSpace",
+    DELETE: "Delete",
+    INSERT: "Insert",
+    HOME: "Home",
+    END: "End",
+    PAGEUP: "Prior",
+    PAGEDOWN: "Next",
+    ARROWUP: "Up",
+    ARROWDOWN: "Down",
+    ARROWLEFT: "Left",
+    ARROWRIGHT: "Right",
+    UP: "Up",
+    DOWN: "Down",
+    LEFT: "Left",
+    RIGHT: "Right",
+    "+": "plus",
+    "-": "minus",
+    "=": "equal",
+    ",": "comma",
+    ".": "period",
+    "/": "slash",
+    "\\": "backslash",
+    ";": "semicolon",
+    "'": "apostrophe",
+    "[": "bracketleft",
+    "]": "bracketright",
+    "`": "grave",
+});
+function lookup(table, key) {
+    const upper = key.toUpperCase();
+    return Object.hasOwn(table, upper) ? table[upper] : undefined;
+}
+function chord(names) {
+    if (new Set(names).size !== names.length)
+        throw new CuaExecutorError("action_rejected", "not_dispatched");
+    return names.join("+");
+}
+/** A keypress chord in xdotool syntax. Unknown or repeated keys are refused before dispatch. */
+export function xdotoolChord(keys) {
+    return chord(keys.map((key) => {
+        if (/^[a-z0-9]$/i.test(key))
+            return key.toLowerCase();
+        if (/^F(?:[1-9]|1[0-2])$/i.test(key))
+            return key.toUpperCase();
+        const name = lookup(keyNames, key);
+        if (!name)
+            throw new CuaExecutorError("action_rejected", "not_dispatched");
+        return name;
+    }));
+}
+/**
+ * The xdotool chord for a pointer action's held keys, or undefined when it holds none. Only
+ * modifiers can be held, which is the use the OpenAI computer-use docs describe; an ordinary key
+ * held down could type into the page.
+ */
+export function xdotoolHeldModifiers(keys) {
+    if (keys === undefined || keys.length === 0)
+        return undefined;
+    return chord(keys.map((key) => {
+        const name = lookup(modifierNames, key);
+        if (!name)
+            throw new CuaExecutorError("action_rejected", "not_dispatched");
+        return name;
+    }));
+}

package/dist/guest/desktop-media.d.ts

@@ -0,0 +1,70 @@
+import type { CuaExecutor } from "../actors/computer-use/loop.js";
+import { type HeardSpeech } from "../actors/computer-use/speech.js";
+import { CuaExecutorError } from "../actors/computer-use/executor-error.js";
+interface GuestDesktopMediaDeclaration {
+    camera?: {
+        source: string;
+    };
+    microphone?: {
+        source: string;
+    };
+}
+export interface DesktopMediaWorkerTransport {
+    start(options: {
+        env: Readonly<Record<string, string>>;
+        data(bytes: Buffer): void;
+        exit(): void;
+    }): Promise<void>;
+    write(bytes: Buffer): Promise<void>;
+    close(): Promise<void>;
+}
+export interface GuestDesktopMediaOptions {
+    media: GuestDesktopMediaDeclaration;
+    env: Readonly<Record<string, string>>;
+    signal: AbortSignal;
+    onTerminal(): void;
+    transport?: DesktopMediaWorkerTransport;
+    workerPath?: string;
+}
+export interface GuestDesktopMedia {
+    readonly env: Readonly<Record<string, string>>;
+    wrap(executor: CuaExecutor): CuaExecutor;
+    close(): Promise<void>;
+}
+/** Speech text the worker may speak or report: non-blank, valid UTF-8, within the limits. */
+export declare function isSpeakableText(value: unknown): value is string;
+/** A heard utterance from the worker, or undefined when any field is out of contract. */
+export declare function parseHeardSpeech(value: unknown): HeardSpeech | undefined;
+type MediaDeclaration = GuestDesktopMediaOptions["media"];
+/** The guest has a synthetic camera and a speech microphone, and a declaration needs at least one. */
+export declare function isSupportedMediaDeclaration(media: MediaDeclaration): boolean;
+/** The worker's env: device switches, plus Pulse routing when the microphone is on. */
+export declare function desktopMediaEnv(env: Readonly<Record<string, string>>, media: MediaDeclaration): Readonly<Record<string, string>>;
+/** Checked before each line is taken: buffered bytes with no newline past the limit can't form a line. */
+export declare function workerBufferOverflow(buffer: Buffer): boolean;
+/** The next newline-terminated line, or why there is none. A partial tail stays incomplete. */
+export declare function nextWorkerLine(buffer: Buffer): {
+    line: Buffer;
+    rest: Buffer;
+} | "incomplete" | "overflow";
+export type WorkerMessage = {
+    kind: "ready";
+} | {
+    kind: "heard";
+    utterance: HeardSpeech;
+} | {
+    kind: "reply";
+    id: string;
+    ok: boolean;
+} | {
+    kind: "terminal";
+};
+/** What one worker message means. Before the first `ready`, anything else breaks the protocol. */
+export declare function classifyWorkerMessage(value: unknown, ready: boolean): WorkerMessage;
+/** Once a speak command is written, its outcome is unknown unless the worker replies. */
+export declare function dispositionAfterWrite(written: boolean): "outcome_uncertain" | "not_dispatched";
+/** A failed speak keeps its own executor error; anything else is an execution failure. */
+export declare function speakFailure(error: unknown, written: boolean): CuaExecutorError;
+/** Starts optional native guest media before Chromium and later decorates its finite executor. */
+export declare function startDesktopMedia(options: GuestDesktopMediaOptions): Promise<GuestDesktopMedia>;
+export {};