humanish 0.104.3 → 0.106.0

package/dist/guest/chromium-text.js

@@ -0,0 +1,386 @@
+import { randomUUID } from "node:crypto";
+import { CuaExecutorError, isCuaExecutorError } from "../actors/computer-use/executor-error.js";
+const DEADLINE_MS = 5_000;
+// These expressions execute only in this port's isolated world. Actor text is
+// never interpolated into JavaScript, a selector, a world name, or a CDP method.
+const PREPARE = `(() => {
+  const active = () => Object.getOwnPropertyDescriptor(Document.prototype, "activeElement").get.call(document);
+  const element = active();
+  const editable = () => {
+    if (!Document.prototype.hasFocus.call(document) || active() !== element ||
+        !element || !element.isConnected || element.ownerDocument !== document ||
+        element.localName === "iframe" || element.localName === "frame") return false;
+    if (element instanceof HTMLTextAreaElement) return !element.disabled && !element.readOnly;
+    if (element instanceof HTMLInputElement) return !element.disabled && !element.readOnly &&
+      ["text", "search", "tel", "url", "email", "password", "number"].includes(element.type);
+    return element instanceof HTMLElement && element.isContentEditable;
+  };
+  if (!editable()) return false;
+  Object.defineProperty(globalThis, "__humanish_text_preparation__", {
+    configurable: true, value: Object.freeze({ editable })
+  });
+  return true;
+})()`;
+const RECHECK = `(() => {
+  const state = globalThis.__humanish_text_preparation__;
+  return !!state && state.editable() === true;
+})()`;
+/**
+ * Text a preparation accepts: non-empty, no NUL, at most 64 KiB of UTF-8 that round-trips.
+ * Written as the negated refusal so each comparison keeps its original form.
+ */
+export function isAdmissibleText(text) {
+    return !(typeof text !== "string" ||
+        !text ||
+        text.includes("\0") ||
+        Buffer.byteLength(text, "utf8") > 65536 ||
+        Buffer.from(text, "utf8").toString("utf8") !== text);
+}
+/** The owned top frame's id from Page.getFrameTree, or undefined for anything else. */
+export function ownedFrameId(tree) {
+    const frameId = tree.frameTree?.frame?.id;
+    if (typeof frameId !== "string" || !frameId || tree.frameTree.frame.parentId)
+        return undefined;
+    return frameId;
+}
+/** An isolated world's execution context id: a positive safe integer (negated refusal form). */
+export function isExecutionContextId(contextId) {
+    return !(!Number.isSafeInteger(contextId) || contextId <= 0);
+}
+/** The fixed Runtime.evaluate parameters for one probe in the port's isolated world. */
+export function evaluateParams(expression, contextId) {
+    return {
+        expression,
+        contextId,
+        returnByValue: true,
+        awaitPromise: false,
+        userGesture: false,
+        includeCommandLineAPI: false,
+        silent: true,
+    };
+}
+/** A probe passed: no exception, and a boolean true result. */
+export function probeAccepted(reply) {
+    return !reply.exceptionDetails && reply.result.type === "boolean" && reply.result.value === true;
+}
+function targetHandlers(state) {
+    function invalidate() {
+        state.generation++;
+        for (const interrupt of state.interruptions)
+            interrupt("session_revoked");
+    }
+    function dialog() {
+        state.dialogSeen = true;
+        invalidate();
+    }
+    function targetClosed() {
+        state.unusable = true;
+        invalidate();
+    }
+    return { invalidate, dialog, targetClosed };
+}
+function watchTargetEvents(settings, handlers) {
+    const { page, context } = settings;
+    page.on("framenavigated", handlers.invalidate);
+    page.on("frameattached", handlers.invalidate);
+    page.on("framedetached", handlers.invalidate);
+    page.on("dialog", handlers.dialog);
+    page.on("close", handlers.targetClosed);
+    context.on("page", handlers.invalidate);
+    context.on("close", handlers.targetClosed);
+}
+function unwatchTargetEvents(settings, handlers) {
+    const { page, context } = settings;
+    page.off("framenavigated", handlers.invalidate);
+    page.off("frameattached", handlers.invalidate);
+    page.off("framedetached", handlers.invalidate);
+    page.off("dialog", handlers.dialog);
+    page.off("close", handlers.targetClosed);
+    context.off("page", handlers.invalidate);
+    context.off("close", handlers.targetClosed);
+}
+/** The port may act: open, usable, still at `expected`, and the one page of its context. */
+function checkScope(settings, state, expected) {
+    const { context, page } = settings;
+    if (state.closed || state.unusable)
+        throw new CuaExecutorError("executor_closed", "not_dispatched");
+    if (state.generation !== expected)
+        throw new CuaExecutorError("session_revoked", "not_dispatched");
+    const pages = context.pages();
+    if (state.dialogSeen ||
+        page.isClosed() ||
+        page.context() !== context ||
+        pages.length !== 1 ||
+        pages[0] !== page)
+        throw new CuaExecutorError("action_rejected", "not_dispatched");
+}
+/**
+ * One bounded operation: a 5 s deadline, the caller's abort, and any interrupt race every step.
+ * A failure after a dispatching step makes the port unusable and reports outcome_uncertain.
+ */
+async function runOperation(settings, state, signal, expected, body) {
+    let dispatched = false;
+    let stopped;
+    const controller = new AbortController();
+    let rejectStop;
+    const stop = new Promise((_resolve, reject) => {
+        rejectStop = reject;
+    });
+    // An event can interrupt before the first awaited step; consume that rejection.
+    void stop.catch(() => { });
+    const interrupt = (code) => {
+        if (stopped)
+            return;
+        stopped = code;
+        controller.abort();
+        rejectStop(new CuaExecutorError(code, dispatched ? "outcome_uncertain" : "not_dispatched"));
+    };
+    const abort = () => interrupt("cancelled");
+    const timer = setTimeout(() => interrupt("deadline_exceeded"), DEADLINE_MS);
+    state.interruptions.add(interrupt);
+    signal.addEventListener("abort", abort, { once: true });
+    const check = () => {
+        if (signal.aborted)
+            abort();
+        if (stopped)
+            throw new CuaExecutorError(stopped, dispatched ? "outcome_uncertain" : "not_dispatched");
+        checkScope(settings, state, expected);
+    };
+    try {
+        check();
+        return await body({
+            signal: controller.signal,
+            check,
+            async step(call, dispatch = false) {
+                check();
+                if (dispatch)
+                    dispatched = true;
+                const value = await Promise.race([call(), stop]);
+                check();
+                return value;
+            },
+        });
+    }
+    catch (error) {
+        controller.abort();
+        if (dispatched)
+            state.unusable = true;
+        const code = stopped ?? (isCuaExecutorError(error) ? error.code : "transport_failed");
+        throw new CuaExecutorError(code, dispatched ? "outcome_uncertain" : "not_dispatched");
+    }
+    finally {
+        clearTimeout(timer);
+        signal.removeEventListener("abort", abort);
+        state.interruptions.delete(interrupt);
+    }
+}
+/** A port's detach, memoized per session: every caller shares the one bounded attempt. */
+function sessionDetacher(state) {
+    const detachments = new WeakMap();
+    return (session) => {
+        const prior = detachments.get(session);
+        if (prior)
+            return prior;
+        const result = (async () => {
+            let timer;
+            try {
+                await Promise.race([
+                    session.detach(),
+                    new Promise((_resolve, reject) => {
+                        timer = setTimeout(() => reject(new CuaExecutorError("deadline_exceeded", "not_dispatched")), DEADLINE_MS);
+                    }),
+                ]);
+            }
+            catch {
+                state.unusable = true;
+                throw new CuaExecutorError("transport_failed", "not_dispatched");
+            }
+            finally {
+                clearTimeout(timer);
+                state.sessions.delete(session);
+            }
+        })();
+        detachments.set(session, result);
+        // Late acquisition after cancellation also uses this consumed, bounded path.
+        void result.catch(() => { });
+        return result;
+    };
+}
+/** The owner's native window check, as one step of an operation. */
+async function assertOwnerReady(settings, op) {
+    const { assertFocusedWindow } = settings;
+    await op.step(() => assertFocusedWindow(op.signal));
+}
+/** One fixed probe in the isolated world; anything but a boolean true refuses the action. */
+async function probeElement(op, session, contextId, expression) {
+    const result = await op.step(() => session.send("Runtime.evaluate", evaluateParams(expression, contextId)));
+    if (!probeAccepted(result))
+        throw new CuaExecutorError("action_rejected", "not_dispatched");
+}
+/**
+ * Preparation: the owner check, a CDP session (recorded on the handle as soon as it exists, so
+ * disposal detaches a partial acquisition), the owned top frame, an isolated world, and the
+ * PREPARE probe.
+ */
+async function acquireIsolatedWorld(settings, state, detach, prepared, op) {
+    await assertOwnerReady(settings, op);
+    prepared.session = await op.step(() => settings.context.newCDPSession(settings.page).then((acquired) => {
+        state.sessions.add(acquired);
+        if (op.signal.aborted || state.closed)
+            void detach(acquired).catch(() => { });
+        return acquired;
+    }));
+    const tree = await op.step(() => prepared.session.send("Page.getFrameTree"));
+    const frameId = ownedFrameId(tree);
+    if (frameId === undefined)
+        throw new CuaExecutorError("invalid_response", "not_dispatched");
+    const world = await op.step(() => prepared.session.send("Page.createIsolatedWorld", {
+        frameId,
+        worldName: settings.worldName,
+    }));
+    prepared.contextId = world.executionContextId;
+    if (!isExecutionContextId(prepared.contextId))
+        throw new CuaExecutorError("invalid_response", "not_dispatched");
+    await probeElement(op, prepared.session, prepared.contextId, PREPARE);
+}
+/** Disposes a prepared handle once: aborts its lifetime, detaches its session, releases busy. */
+function disposePrepared(state, detach, prepared, self) {
+    if (!prepared.disposal) {
+        prepared.disposed = true;
+        prepared.lifetime.abort();
+        prepared.signal.removeEventListener("abort", prepared.abortLifetime);
+        prepared.disposal = (async () => {
+            try {
+                if (prepared.session)
+                    await detach(prepared.session);
+            }
+            catch (error) {
+                throw new CuaExecutorError(isCuaExecutorError(error) ? error.code : "transport_failed", prepared.dispatchedText ? "outcome_uncertain" : "not_dispatched");
+            }
+            finally {
+                state.busy = false;
+                if (state.activeDisposal === self)
+                    state.activeDisposal = undefined;
+            }
+        })();
+    }
+    return prepared.disposal;
+}
+/**
+ * The handle for one preparation. paste inserts the text once: owner check, RECHECK probe, owner
+ * check, check, then insertText. close is the handle's disposal.
+ */
+function preparedHandle(settings, state, prepared, dispose) {
+    return {
+        async paste() {
+            if (prepared.used || prepared.disposed)
+                throw new CuaExecutorError("action_rejected", "not_dispatched");
+            prepared.used = true; // A failed attempt cannot be retried through this handle.
+            await runOperation(settings, state, prepared.lifetime.signal, prepared.expected, async (op) => {
+                await assertOwnerReady(settings, op);
+                await probeElement(op, prepared.session, prepared.contextId, RECHECK);
+                await assertOwnerReady(settings, op);
+                op.check();
+                await op.step(() => {
+                    prepared.dispatchedText = true;
+                    return prepared.session.send("Input.insertText", { text: prepared.text });
+                }, true);
+            });
+        },
+        close: dispose,
+    };
+}
+/**
+ * Closes the port: interrupts running operations, removes the target listeners, disposes the
+ * active handle and detaches every session. The first cleanup failure is reported.
+ */
+function closePort(settings, state, handlers, detach) {
+    state.closed = true;
+    for (const interrupt of state.interruptions)
+        interrupt("executor_closed");
+    unwatchTargetEvents(settings, handlers);
+    const active = state.activeDisposal?.();
+    return Promise.allSettled([...(active ? [active] : []), ...[...state.sessions].map(detach)]).then((results) => {
+        const failure = results.find((result) => result.status === "rejected");
+        if (failure?.status === "rejected")
+            throw failure.reason;
+    });
+}
+/**
+ * A finite text port for one owner-acquired Chromium page in a one-page context.
+ * It neither selects/activates targets nor handles browser chrome, frames or
+ * dialogs. An observed dialog poisons this port; the owner must handle it and
+ * construct a new port. A dialog already open can make preparation time out.
+ *
+ * Focus checks and insertText are separate protocol calls: page code can change
+ * focus between them. This is not atomic element binding or proof an app applied
+ * text. A lost post-send acknowledgement is uncertain and terminal for this port.
+ * The owner remains responsible for browser/session teardown.
+ */
+export function createGuestChromiumText(options) {
+    const { context, page, assertFocusedWindow } = options;
+    const settings = {
+        context,
+        page,
+        assertFocusedWindow,
+        worldName: `humanish-text-${randomUUID()}`,
+    };
+    const state = {
+        generation: 0,
+        closed: false,
+        unusable: false,
+        dialogSeen: false,
+        busy: false,
+        activeDisposal: undefined,
+        interruptions: new Set(),
+        sessions: new Set(),
+    };
+    const detach = sessionDetacher(state);
+    const handlers = targetHandlers(state);
+    watchTargetEvents(settings, handlers);
+    let closing;
+    return {
+        async assertReady(signal) {
+            await runOperation(settings, state, signal, state.generation, (op) => assertOwnerReady(settings, op));
+        },
+        async prepareText(text, signal) {
+            if (!isAdmissibleText(text))
+                throw new CuaExecutorError("invalid_request", "not_dispatched");
+            checkScope(settings, state, state.generation);
+            if (state.busy)
+                throw new CuaExecutorError("executor_busy", "not_dispatched");
+            state.busy = true;
+            const expected = state.generation;
+            const lifetime = new AbortController();
+            const prepared = {
+                text,
+                expected,
+                lifetime,
+                signal,
+                abortLifetime: () => lifetime.abort(),
+                session: undefined,
+                contextId: undefined,
+                used: false,
+                disposed: false,
+                dispatchedText: false,
+                disposal: undefined,
+            };
+            signal.addEventListener("abort", prepared.abortLifetime, { once: true });
+            if (signal.aborted)
+                prepared.abortLifetime();
+            const dispose = () => disposePrepared(state, detach, prepared, dispose);
+            state.activeDisposal = dispose;
+            try {
+                await runOperation(settings, state, lifetime.signal, prepared.expected, (op) => acquireIsolatedWorld(settings, state, detach, prepared, op));
+            }
+            catch (error) {
+                await dispose().catch(() => { });
+                throw error;
+            }
+            return preparedHandle(settings, state, prepared, dispose);
+        },
+        close() {
+            return (closing ??= closePort(settings, state, handlers, detach));
+        },
+    };
+}

package/dist/guest/desktop-executor.d.ts

@@ -0,0 +1,57 @@
+import type { CuaAction, CuaExecutor } from "../actors/computer-use/loop.js";
+import { CuaExecutorError } from "../actors/computer-use/executor-error.js";
+/** Internal guest port. Implementations recheck signal synchronously before native dispatch. */
+export interface GuestDesktopTools {
+    capture(signal: AbortSignal): Promise<Buffer>;
+    input(args: readonly string[], signal: AbortSignal): Promise<void>;
+    prepareText(text: string, signal: AbortSignal): Promise<{
+        paste(): Promise<void>;
+        close(): Promise<void>;
+    }>;
+}
+export interface GuestDesktopExecutorOptions {
+    width: number;
+    height: number;
+    tools: GuestDesktopTools;
+    authoritySignal: AbortSignal;
+    /** Owner must stop the private display/browser after a partial/uncertain input. */
+    onTerminal: () => void;
+}
+/** The frame an executor addresses, in pixels. */
+interface Frame {
+    readonly width: number;
+    readonly height: number;
+}
+type TextChannel = Awaited<ReturnType<GuestDesktopTools["prepareText"]>>;
+/** What one execute call has done so far. Its failure and cleanup paths read it. */
+interface ActionProgress {
+    /** Native input or a text insertion may have reached the desktop. */
+    dispatched: boolean;
+    /** Text preparation started. */
+    preparing: boolean;
+    /** The modifiers a keydown pressed and no keyup has released yet. */
+    keysHeld: string | undefined;
+    text: TextChannel | undefined;
+}
+/** Headed X11 input and full-frame captures. Browser metadata is deliberately absent. */
+export declare function createGuestDesktopExecutor(options: GuestDesktopExecutorOptions): CuaExecutor;
+/** Whether a frame of this size fits the browser-control limits. */
+export declare function isValidFrameSize(width: number, height: number): boolean;
+/**
+ * Whether a failed action ends the session: anything dispatched, an aborted signal, an error the
+ * executor did not raise, or a failed text preparation other than a refusal before dispatch. It
+ * reads signal.aborted only when the earlier conditions do not decide, as the inline check did.
+ */
+export declare function failureEndsSession(progress: Pick<ActionProgress, "preparing" | "dispatched">, error: unknown, signal: Pick<AbortSignal, "aborted">): boolean;
+/**
+ * The error a failed action throws: outcome_uncertain once anything was dispatched, then
+ * session_revoked when the signal aborted, then the executor's own error or execution_failed. It
+ * reads signal.aborted only for an undispatched failure, as the inline mapping did.
+ */
+export declare function failureError(dispatched: boolean, error: unknown, signal: Pick<AbortSignal, "aborted">): CuaExecutorError;
+/**
+ * The native commands for an action, wrapped in keydown/keyup when it holds modifiers. It throws
+ * the refusal for an action the desktop cannot take, before anything is sent.
+ */
+export declare function planActionInput(frame: Frame, action: CuaAction): string[][];
+export {};