humanish 0.138.0 → 0.139.0

package/dist/substrates/modal/commands.js

@@ -0,0 +1,174 @@
+// Commands and files in a Modal sandbox, with the E2B SDK's contracts, which the routes were written
+// against: a non-zero exit throws CommandExitError, a timeout throws TimeoutError, relative paths
+// are under the user's home, and a background command returns a handle to wait on, kill or feed.
+//
+// Modal runs every exec as root and ignores the image's user setting, so each command drops to the
+// sandbox user here, the account E2B runs commands as. Modal has no way to signal one exec, so a
+// background command records its pid in a file and is killed by that pid.
+import { randomUUID } from "node:crypto";
+import { ReadableStream } from "node:stream/web";
+import { shellQuote } from "../shell.js";
+import { MODAL_SANDBOX_USER } from "./image.js";
+/** E2B's commands.run default, kept so a caller that sets no timeout gets the same bound. */
+const DEFAULT_COMMAND_TIMEOUT_MS = 60_000;
+/** Modal takes sandbox and command timeouts in whole seconds; a shorter one is rounded up. */
+export const wholeSeconds = (ms) => Math.ceil(ms / 1000) * 1000;
+/** The argv that runs `command` in a login shell as the sandbox user, in their home. */
+function asSandboxUser(command) {
+    const { uid } = MODAL_SANDBOX_USER;
+    return [
+        "setpriv",
+        `--reuid=${uid}`,
+        `--regid=${uid}`,
+        "--init-groups",
+        "--",
+        "bash",
+        "-lc",
+        command,
+    ];
+}
+const USER_ENV = {
+    HOME: MODAL_SANDBOX_USER.home,
+    USER: MODAL_SANDBOX_USER.name,
+    LOGNAME: MODAL_SANDBOX_USER.name,
+};
+/** The E2B SDK throws this shape on a non-zero exit; src/substrates/command-failure.ts reads it. */
+class ModalCommandExitError extends Error {
+    exitCode;
+    stdout;
+    stderr;
+    constructor(exitCode, stdout, stderr) {
+        super(`Command exited with code ${exitCode}${stderr ? ` and error:\n${stderr}` : ""}`);
+        this.exitCode = exitCode;
+        this.stdout = stdout;
+        this.stderr = stderr;
+        this.name = "CommandExitError";
+    }
+    get error() {
+        return this.stderr;
+    }
+}
+async function collect(stream, onData) {
+    const decoder = new TextDecoder();
+    let text = "";
+    for await (const chunk of stream) {
+        const data = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
+        text += data;
+        if (onData !== undefined && data !== "")
+            await onData(data);
+    }
+    return text;
+}
+/** Modal ends a command at its timeout and reports it as a deadline; E2B names it TimeoutError. */
+function named(error, timeoutMs) {
+    if (!(error instanceof Error) || !/deadline exceeded/i.test(error.message))
+        return error;
+    const timeout = new Error(`Command did not finish within ${timeoutMs} ms.`, { cause: error });
+    timeout.name = "TimeoutError";
+    return timeout;
+}
+/** Run `command` as the sandbox user, to completion or, with `background`, returning a handle. */
+export async function runCommand(sandbox, command, options = {}) {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS;
+    const pidFile = options.background === true ? `/tmp/humanish-exec-${randomUUID()}.pid` : "";
+    const line = options.background === true
+        ? `printf '%s\\n' "$$" > ${pidFile}; exec bash -c ${shellQuote(command)}`
+        : command;
+    let finished;
+    let child;
+    try {
+        child = await sandbox.exec(asSandboxUser(line), {
+            env: { ...USER_ENV, ...options.envs },
+            ...(options.cwd === undefined ? {} : { workdir: options.cwd }),
+            ...(timeoutMs > 0 ? { timeoutMs: wholeSeconds(timeoutMs) } : {}),
+        });
+        // E2B starts a command with no stdin unless asked; a Modal exec keeps the pipe open, and a
+        // reader such as codex exec would wait on it until its timeout.
+        if (options.stdin !== true)
+            await child.closeStdin();
+        const running = child;
+        finished = Promise.all([
+            collect(running.stdout, options.onStdout),
+            collect(running.stderr, options.onStderr),
+            running.wait(),
+        ]).then(([stdout, stderr, exitCode]) => {
+            if (exitCode !== 0)
+                throw new ModalCommandExitError(exitCode, stdout, stderr);
+            return { exitCode, stdout, stderr };
+        });
+    }
+    catch (error) {
+        throw named(error, timeoutMs);
+    }
+    if (options.background !== true)
+        return finished.catch((error) => {
+            throw named(error, timeoutMs);
+        });
+    // The caller decides when to wait; an exit nobody waits for is not an unhandled rejection.
+    finished.catch(() => { });
+    const running = child;
+    // E2B's handle carries the process's pid, which owners such as the recorder check.
+    const pid = await backgroundPid(sandbox, pidFile);
+    return {
+        ...(pid === undefined ? {} : { pid }),
+        wait: () => finished,
+        kill: async () => {
+            const killed = await runCommand(sandbox, `kill -KILL "$(cat ${pidFile})" 2>/dev/null && echo killed || true`, { timeoutMs: 10_000 });
+            return killed.stdout?.trim() === "killed";
+        },
+        sendStdin: (data) => typeof data === "string" ? running.stdin.writeText(data) : running.stdin.writeBytes(data),
+        closeStdin: () => running.stdin.close(),
+    };
+}
+/** How long a background command has to record its pid before its handle reports none. */
+const PID_WAIT_MS = 5_000;
+/** The pid a background command wrote to `pidFile`, or undefined if none arrived in time. */
+async function backgroundPid(sandbox, pidFile) {
+    for (const end = Date.now() + PID_WAIT_MS; Date.now() < end;) {
+        const read = await runCommand(sandbox, `cat ${pidFile} 2>/dev/null || true`, {
+            timeoutMs: 10_000,
+        });
+        const pid = Number(read.stdout?.trim());
+        if (Number.isSafeInteger(pid) && pid > 0)
+            return pid;
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    return undefined;
+}
+/** Relative paths are under the sandbox user's home, as on E2B. */
+export function sandboxPath(path) {
+    return path.startsWith("/") ? path : `${MODAL_SANDBOX_USER.home}/${path}`;
+}
+export async function writeFile(sandbox, path, data) {
+    const target = sandboxPath(path);
+    const child = await sandbox.exec(asSandboxUser('mkdir -p -- "$(dirname -- "$0")" && cat > "$0"').concat(target), { mode: "binary", env: USER_ENV });
+    await child.stdin.writeBytes(typeof data === "string" ? new TextEncoder().encode(data) : new Uint8Array(data));
+    await child.stdin.close();
+    const [stderr, exitCode] = await Promise.all([child.stderr.readBytes(), child.wait()]);
+    if (exitCode !== 0)
+        throw new ModalCommandExitError(exitCode, "", new TextDecoder().decode(stderr));
+    return { path: target };
+}
+/** A file's bytes as a stream, read as the sandbox user. */
+export async function readFile(sandbox, path) {
+    const child = await sandbox.exec(asSandboxUser('cat -- "$0"').concat(sandboxPath(path)), {
+        mode: "binary",
+        env: USER_ENV,
+    });
+    await child.closeStdin();
+    const bytes = async function* () {
+        for await (const chunk of child.stdout)
+            yield typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk;
+        const exitCode = await child.wait();
+        if (exitCode !== 0)
+            throw new ModalCommandExitError(exitCode, "", await child.stderr.readText());
+    };
+    return ReadableStream.from(bytes());
+}
+/** A file's bytes, whole. */
+export async function readBytes(sandbox, path) {
+    const parts = [];
+    for await (const chunk of await readFile(sandbox, path))
+        parts.push(chunk);
+    return Buffer.concat(parts);
+}

package/dist/substrates/modal/desktop.d.ts

@@ -0,0 +1,16 @@
+import type { E2BCommandResult, E2BCommandRunOptions, E2BDesktopSandbox } from "../e2b/sdk.js";
+type Run = (command: string, options?: E2BCommandRunOptions) => Promise<E2BCommandResult>;
+/** The X display and the ports the desktop's stream uses, as @e2b/desktop sets them. */
+export declare const MODAL_DESKTOP_DISPLAY = ":0";
+export declare const MODAL_STREAM_PORT = 6080;
+/** Start Xvfb at the requested size, then an XFCE session, as @e2b/desktop's create does. */
+export declare function startModalDesktop(run: Run, options: {
+    resolution?: [number, number];
+    dpi?: number;
+}): Promise<void>;
+/**
+ * The desktop half of a sandbox handle. `hosts` maps each tunneled port to the host Modal serves it
+ * on; `readBytes` reads a file in the sandbox.
+ */
+export declare function modalDesktopSurface(run: Run, readBytes: (path: string) => Promise<Uint8Array>, hosts: ReadonlyMap<number, string>): Omit<E2BDesktopSandbox, "sandboxId" | "getInfo" | "commands" | "files">;
+export {};

package/dist/substrates/modal/desktop.js

@@ -0,0 +1,195 @@
+// A desktop in a Modal sandbox, with @e2b/desktop's behavior: the same Xvfb and XFCE start, the
+// same xdotool commands for each input, scrot for screenshots and x11vnc behind noVNC for the live
+// stream. The executor and every e2b-desktop module were tuned against that SDK, so this follows it
+// command for command (the Sandbox class of @e2b/desktop 2.4.0) rather than improving on it.
+import { randomBytes } from "node:crypto";
+import { shellQuote } from "../shell.js";
+/** The X display and the ports the desktop's stream uses, as @e2b/desktop sets them. */
+export const MODAL_DESKTOP_DISPLAY = ":0";
+const VNC_PORT = 5900;
+export const MODAL_STREAM_PORT = 6080;
+const KEYS = {
+    alt: "Alt_L",
+    alt_left: "Alt_L",
+    alt_right: "Alt_R",
+    backspace: "BackSpace",
+    break: "Pause",
+    caps_lock: "Caps_Lock",
+    cmd: "Super_L",
+    command: "Super_L",
+    control: "Control_L",
+    control_left: "Control_L",
+    control_right: "Control_R",
+    ctrl: "Control_L",
+    del: "Delete",
+    delete: "Delete",
+    down: "Down",
+    end: "End",
+    enter: "Return",
+    esc: "Escape",
+    escape: "Escape",
+    f1: "F1",
+    f2: "F2",
+    f3: "F3",
+    f4: "F4",
+    f5: "F5",
+    f6: "F6",
+    f7: "F7",
+    f8: "F8",
+    f9: "F9",
+    f10: "F10",
+    f11: "F11",
+    f12: "F12",
+    home: "Home",
+    insert: "Insert",
+    left: "Left",
+    menu: "Menu",
+    meta: "Meta_L",
+    num_lock: "Num_Lock",
+    page_down: "Page_Down",
+    page_up: "Page_Up",
+    pause: "Pause",
+    print: "Print",
+    right: "Right",
+    scroll_lock: "Scroll_Lock",
+    shift: "Shift_L",
+    shift_left: "Shift_L",
+    shift_right: "Shift_R",
+    space: "space",
+    super: "Super_L",
+    super_left: "Super_L",
+    super_right: "Super_R",
+    tab: "Tab",
+    up: "Up",
+    win: "Super_L",
+    windows: "Super_L",
+};
+const mapKey = (key) => KEYS[key.toLowerCase()] ?? key.toLowerCase();
+/** Run `command` every `intervalMs` until it exits 0, for at most `timeoutMs`. */
+async function waitUntil(run, command, timeoutMs, intervalMs = 500) {
+    for (const end = Date.now() + timeoutMs; Date.now() < end;) {
+        try {
+            await run(command);
+            return true;
+        }
+        catch (error) {
+            if (error.name !== "CommandExitError")
+                throw error;
+        }
+        await new Promise((resolve) => setTimeout(resolve, intervalMs));
+    }
+    return false;
+}
+/** Start Xvfb at the requested size, then an XFCE session, as @e2b/desktop's create does. */
+export async function startModalDesktop(run, options) {
+    const [width, height] = options.resolution ?? [1024, 768];
+    const display = MODAL_DESKTOP_DISPLAY;
+    await run(`setsid -f Xvfb ${display} -ac -screen 0 ${width}x${height}x24 -retro -dpi ${options.dpi ?? 96} -nolisten tcp -nolisten unix </dev/null >/tmp/xvfb.log 2>&1`);
+    if (!(await waitUntil(run, `xdpyinfo -display ${display} >/dev/null`, 10_000)))
+        throw new Error("Could not start Xvfb");
+    await run("setsid -f startxfce4 </dev/null >/tmp/xfce4.log 2>&1");
+    if (!(await waitUntil(run, "pgrep -x xfce4-session >/dev/null && pgrep -x xfwm4 >/dev/null && pgrep -x xfdesktop >/dev/null", 60_000, 1_000)))
+        throw new Error("Could not start XFCE");
+}
+/**
+ * The desktop half of a sandbox handle. `hosts` maps each tunneled port to the host Modal serves it
+ * on; `readBytes` reads a file in the sandbox.
+ */
+export function modalDesktopSurface(run, readBytes, hosts) {
+    const getHost = (port) => {
+        const host = hosts.get(port);
+        if (host === undefined)
+            throw new Error(`Port ${port} has no Modal tunnel in this sandbox.`);
+        return host;
+    };
+    const moveMouse = async (x, y) => {
+        await run(`xdotool mousemove --sync ${x} ${y}`);
+    };
+    // The SDK moves only when both coordinates are truthy, so a click at x or y 0 clicks in place.
+    const clickAt = async (x, y, click) => {
+        if (x && y)
+            await moveMouse(x, y);
+        await run(`xdotool click ${click}`);
+    };
+    let streamUrl;
+    let password;
+    return {
+        getHost,
+        async screenshot() {
+            const path = `/tmp/screenshot-${randomBytes(8).toString("hex")}.png`;
+            await run(`scrot --pointer ${path}`);
+            try {
+                return await readBytes(path);
+            }
+            finally {
+                void run(`rm -f ${path}`).catch(() => { });
+            }
+        },
+        wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+        leftClick: (x, y) => clickAt(x, y, "1"),
+        rightClick: (x, y) => clickAt(x, y, "3"),
+        middleClick: (x, y) => clickAt(x, y, "2"),
+        doubleClick: (x, y) => clickAt(x, y, "--repeat 2 1"),
+        moveMouse,
+        async getCursorPosition() {
+            const result = await run("xdotool getmouselocation");
+            const match = (result.stdout ?? "").match(/x:(\d+)\s+y:(\d+)/);
+            if (!match)
+                throw new Error(`Failed to parse cursor position from output: ${result.stdout}`);
+            return { x: Number.parseInt(match[1], 10), y: Number.parseInt(match[2], 10) };
+        },
+        async scroll(direction = "down", amount = 1) {
+            await run(`xdotool click --repeat ${amount} ${direction === "up" ? "4" : "5"}`);
+        },
+        async write(text) {
+            for (let at = 0; at < text.length; at += 25)
+                await run(`xdotool type --delay 75 -- ${shellQuote(text.slice(at, at + 25))}`);
+        },
+        async press(key) {
+            await run(`xdotool key ${Array.isArray(key) ? key.map(mapKey).join("+") : mapKey(key)}`);
+        },
+        async drag(from, to) {
+            await moveMouse(...from);
+            await run("xdotool mousedown 1");
+            await moveMouse(...to);
+            await run("xdotool mouseup 1");
+        },
+        async open(fileOrUrl) {
+            await run(`setsid -f xdg-open ${fileOrUrl} </dev/null >/dev/null 2>&1`);
+        },
+        async launch(application, uri) {
+            await run(`setsid -f gtk-launch ${application} ${uri ?? ""} </dev/null >/dev/null 2>&1`);
+        },
+        stream: {
+            getAuthKey() {
+                if (password === undefined)
+                    throw new Error("Unable to retrieve stream auth key, check if requireAuth is enabled");
+                return password;
+            },
+            getUrl({ autoConnect = true, viewOnly = false, resize = "scale", authKey } = {}) {
+                if (streamUrl === undefined)
+                    throw new Error("Server is not running");
+                const url = new URL(streamUrl);
+                if (autoConnect)
+                    url.searchParams.set("autoconnect", "true");
+                if (viewOnly)
+                    url.searchParams.set("view_only", "true");
+                if (resize)
+                    url.searchParams.set("resize", resize);
+                if (authKey)
+                    url.searchParams.set("password", authKey);
+                return url.toString();
+            },
+            async start({ requireAuth = false, windowId } = {}) {
+                password = requireAuth ? randomBytes(12).toString("base64url") : undefined;
+                streamUrl = new URL(`https://${getHost(MODAL_STREAM_PORT)}/vnc.html`);
+                if (password !== undefined)
+                    await run(`mkdir -p ~/.vnc && x11vnc -storepasswd ${password} ~/.vnc/passwd`);
+                await run(`x11vnc -bg -display ${MODAL_DESKTOP_DISPLAY} -forever -wait 50 -shared -rfbport ${VNC_PORT} ${password === undefined ? "-nopw" : "-usepw"} 2>/tmp/x11vnc_stderr.log${windowId === undefined ? "" : ` -id ${windowId}`}`);
+                await run(`setsid -f websockify --web /usr/share/novnc ${MODAL_STREAM_PORT} localhost:${VNC_PORT} </dev/null >/tmp/novnc.log 2>&1`);
+                if (!(await waitUntil(run, `netstat -tuln | grep ":${MODAL_STREAM_PORT} "`, 10_000)))
+                    throw new Error("Could not start noVNC server");
+            },
+        },
+    };
+}

package/dist/substrates/modal/image.d.ts

@@ -7,3 +7,5 @@ export declare const MODAL_SANDBOX_USER: {
 };
 /** The terminal route's image: a Debian shell with a passwordless-sudo `user`, as on E2B. */
 export declare function modalShellImage(client: ModalClient): ModalImage;
+/** The computer-use route's image: the shell's packages and an XFCE desktop with Chromium. */
+export declare function modalDesktopImage(client: ModalClient): ModalImage;

package/dist/substrates/modal/image.js

@@ -21,15 +21,50 @@ const SHELL_PACKAGES = [
     "unzip",
     "xz-utils",
 ];
+// What the e2b-desktop modules call in a desktop, beside the shell's packages: Xvfb and an XFCE
+// session as @e2b/desktop starts them, xdotool and scrot for input and screenshots, x11vnc and
+// noVNC for the stream, the X utilities the geometry and screen probes run, wmctrl, ffmpeg and
+// PulseAudio for recordings, Chromium, and fonts for Latin, Chinese, Japanese, Korean and emoji
+// text.
+const DESKTOP_PACKAGES = [
+    "chromium",
+    "dbus-x11",
+    "ffmpeg",
+    "fonts-dejavu",
+    "fonts-noto-cjk",
+    "fonts-noto-color-emoji",
+    "net-tools",
+    "novnc",
+    "pulseaudio",
+    "pulseaudio-utils",
+    "scrot",
+    "websockify",
+    "wmctrl",
+    "x11-utils",
+    "x11-xserver-utils",
+    "x11vnc",
+    "xdg-utils",
+    "xdotool",
+    "xfce4",
+    "xfce4-terminal",
+    "xvfb",
+];
 /** The terminal route's image: a Debian shell with a passwordless-sudo `user`, as on E2B. */
 export function modalShellImage(client) {
+    return debianImage(client, SHELL_PACKAGES);
+}
+/** The computer-use route's image: the shell's packages and an XFCE desktop with Chromium. */
+export function modalDesktopImage(client) {
+    return debianImage(client, [...SHELL_PACKAGES, ...DESKTOP_PACKAGES]);
+}
+function debianImage(client, packages) {
     const { name, uid, home } = MODAL_SANDBOX_USER;
     // prose-check: script
     return client.images
         .fromRegistry(DEBIAN_BASE)
         .dockerfileCommands([
         "ENV DEBIAN_FRONTEND=noninteractive",
-        `RUN rm -f /etc/apt/sources.list /etc/apt/sources.list.d/* && ${SNAPSHOT_SOURCES} && apt-get update && apt-get install -y --no-install-recommends ${SHELL_PACKAGES.join(" ")} && rm -rf /var/lib/apt/lists/*`,
+        `RUN rm -f /etc/apt/sources.list /etc/apt/sources.list.d/* && ${SNAPSHOT_SOURCES} && apt-get update && apt-get install -y --no-install-recommends ${packages.join(" ")} && rm -rf /var/lib/apt/lists/*`,
         `RUN useradd --uid ${uid} --create-home --home-dir ${home} --shell /bin/bash ${name} && printf '${name} ALL=(ALL) NOPASSWD:ALL\\n' > /etc/sudoers.d/${name} && chmod 0440 /etc/sudoers.d/${name}`,
         `WORKDIR ${home}`,
     ]);

package/dist/substrates/modal/sandbox.d.ts

@@ -7,6 +7,8 @@ export interface ModalSandboxProfile {
     /** Physical cores; Modal counts one core as two vCPU. */
     cpu: number;
     memoryMiB: number;
+    /** Start an X desktop after create and tunnel its stream, as @e2b/desktop's create does. */
+    desktop?: true;
 }
 /**
  * A sandbox port over one Modal account. `create` maps E2B's create options onto Modal's; `kill`,
@@ -22,5 +24,7 @@ export declare function modalSandboxModule(client: ModalClient, connection: Moda
 export declare function modalSandboxes(client: ModalClient, connection: ModalConnection): Required<E2BAccount>;
 /** The Modal account `env` names, for release by id and reclaim. Throws when it names none. */
 export declare function modalAccount(env: Readonly<Record<string, string | undefined>>, load?: () => Promise<ModalModule>): Promise<Required<E2BAccount>>;
+/** A sandbox port that creates the computer-use route's desktops on the Modal account `env` names. */
+export declare function loadModalDesktopModule(env: Readonly<Record<string, string | undefined>>, load?: () => Promise<ModalModule>): Promise<E2BDesktopModule>;
 /** A sandbox port that creates the terminal route's shell sandboxes on the Modal account `env` names. */
 export declare function loadModalShellModule(env: Readonly<Record<string, string | undefined>>, load?: () => Promise<ModalModule>): Promise<E2BDesktopModule>;