@myagentroam/node 0.11.38 → 0.11.39

package/dist/computer-use/session.js

@@ -0,0 +1,334 @@
+import { randomUUID } from 'node:crypto';
+import { ComputerUseError } from './contracts.js';
+import { PythonWorker } from './worker.js';
+export class ComputerUseDesktopLeaseService {
+    #holder;
+    acquire(runId) {
+        if (this.#holder?.runId === runId)
+            return this.#holder;
+        if (this.#holder !== undefined)
+            throw new ComputerUseError('DESKTOP_BUSY', `The desktop is in use by another Run since ${new Date(this.#holder.acquiredAt).toISOString()}.`);
+        this.#holder = { runId, leaseId: randomUUID(), acquiredAt: Date.now() };
+        return this.#holder;
+    }
+    release(runId) {
+        if (this.#holder?.runId === runId)
+            this.#holder = undefined;
+    }
+    holder() {
+        return this.#holder;
+    }
+}
+export class ComputerUseSessionResources {
+    runtimeExecutable;
+    lease;
+    showControlIndicator;
+    validateDesktop;
+    #worker;
+    #observationId;
+    #runtimeGeneration = 1;
+    #runId;
+    #closed = false;
+    #bindingClose;
+    #closePromise;
+    #cleanupError;
+    #task;
+    #bindingStop = new AbortController();
+    #unsubscribeExecution;
+    constructor(runtimeExecutable, lease, showControlIndicator, validateDesktop) {
+        this.runtimeExecutable = runtimeExecutable;
+        this.lease = lease;
+        this.showControlIndicator = showControlIndicator;
+        this.validateDesktop = validateDesktop;
+    }
+    attach(runId, signal) {
+        if (this.#cleanupError !== undefined)
+            throw this.#cleanupError;
+        if (this.#closed)
+            throw new ComputerUseError('COMPUTER_UNAVAILABLE', 'Computer Use resources are closed.');
+        if (this.#runId !== undefined)
+            throw new ComputerUseError('COMPUTER_SESSION_BUSY', 'Computer Use Session resources are already attached to another Run.');
+        this.#runId = runId;
+        this.#observationId = undefined;
+        this.#task = undefined;
+        this.#bindingStop = new AbortController();
+        const abort = () => {
+            this.#bindingStop.abort();
+            if (this.#task?.settled === undefined)
+                this.#task?.stop.abort();
+        };
+        signal?.addEventListener('abort', abort, { once: true });
+        this.#unsubscribeExecution = () => signal?.removeEventListener('abort', abort);
+        if (signal?.aborted)
+            abort();
+    }
+    async detach(runId) {
+        if (this.#runId !== runId)
+            return;
+        this.#bindingStop.abort();
+        this.#unsubscribeExecution?.();
+        this.#unsubscribeExecution = undefined;
+        if (this.#task?.settled === undefined)
+            this.#task?.stop.abort();
+        await this.#task?.completion;
+        if (this.#cleanupError !== undefined)
+            throw this.#cleanupError;
+        if (this.#closed)
+            return;
+        this.#observationId = undefined;
+        this.#task = undefined;
+        this.lease.release(runId);
+        this.#runId = undefined;
+        this.#bindingClose = undefined;
+    }
+    setBindingClose(close) {
+        this.#bindingClose = close;
+    }
+    async call(input, signal) {
+        const runId = this.#assertBound();
+        if (signal.aborted)
+            throw new ComputerUseError('COMPUTER_UNAVAILABLE', 'Computer Use call was cancelled.');
+        if (this.#cleanupError !== undefined)
+            throw this.#cleanupError;
+        if (this.#task !== undefined && this.#task.settled === undefined)
+            throw new ComputerUseError('COMPUTER_SESSION_BUSY', 'Computer Use is already executing.');
+        if (input.basedOnObservationId !== undefined &&
+            input.basedOnObservationId !== this.#observationId)
+            throw new ComputerUseError('COMPUTER_OBSERVATION_STALE', 'The desktop observation is stale; observe again before acting.');
+        this.lease.acquire(runId);
+        this.#observationId = undefined;
+        const stop = new AbortController();
+        const abort = () => stop.abort();
+        signal.addEventListener('abort', abort, { once: true });
+        const task = {
+            id: `cu_${randomUUID()}`,
+            stop,
+            // Admission is synchronous; preparation and every failure belong to this slot.
+            completion: Promise.resolve()
+                .then(() => this.#execute(input, stop.signal, runId))
+                .then((result) => (task.settled = { result }), (error) => (task.settled = { error }))
+        };
+        this.#task = task;
+        try {
+            return await this.#waitTask(task, input.yieldMs);
+        }
+        finally {
+            signal.removeEventListener('abort', abort);
+        }
+    }
+    async wait(input, signal) {
+        this.#assertBound();
+        if (signal.aborted)
+            throw waitCancelled();
+        const task = this.#task;
+        if (task === undefined || task.id !== input.taskId)
+            throw new ComputerUseError('COMPUTER_TASK_NOT_FOUND', 'The task is not available in this binding.');
+        if (input.action === 'cancel' && task.settled === undefined)
+            task.stop.abort();
+        return this.#waitTask(task, input.yieldMs, signal);
+    }
+    #assertBound() {
+        if (this.#closed || this.#runId === undefined || this.#bindingStop.signal.aborted)
+            throw new ComputerUseError('COMPUTER_UNAVAILABLE', 'Computer Use binding is closed.');
+        return this.#runId;
+    }
+    async #waitTask(task, yieldMs, signal) {
+        let timer;
+        let onAbort;
+        try {
+            if (task.settled === undefined && yieldMs !== 0) {
+                const waits = [task.completion];
+                if (yieldMs !== undefined)
+                    waits.push(new Promise((resolve) => {
+                        timer = setTimeout(resolve, yieldMs);
+                    }));
+                if (signal !== undefined)
+                    waits.push(new Promise((_, reject) => {
+                        onAbort = () => reject(waitCancelled());
+                        signal.addEventListener('abort', onAbort, { once: true });
+                        if (signal.aborted)
+                            onAbort();
+                    }));
+                await Promise.race(waits);
+            }
+            if (this.#task !== task)
+                throw new ComputerUseError('COMPUTER_TASK_NOT_FOUND', 'The task is no longer available in this binding.');
+            if (task.settled === undefined)
+                return { taskId: task.id, status: task.stop.signal.aborted ? 'STOPPING' : 'RUNNING' };
+            if ('error' in task.settled)
+                throw task.settled.error;
+            return { ...task.settled.result, taskId: task.id, status: 'COMPLETED' };
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+            if (onAbort !== undefined)
+                signal?.removeEventListener('abort', onAbort);
+        }
+    }
+    async #execute(input, signal, runId) {
+        this.#assertActive(signal, runId);
+        await this.validateDesktop();
+        this.#assertActive(signal, runId);
+        let indicator;
+        try {
+            indicator = await this.showControlIndicator(this.runtimeExecutable);
+        }
+        catch (error) {
+            if (error instanceof ComputerUseError && error.code === 'COMPUTER_CLEANUP_FAILED') {
+                this.#cleanupError = error;
+                throw error;
+            }
+            throw new ComputerUseError('COMPUTER_CONTROL_INDICATOR_UNAVAILABLE', 'The desktop control indicator could not be displayed.', { cause: error });
+        }
+        let result;
+        let primaryError;
+        let failed = false;
+        try {
+            this.#assertActive(signal, runId);
+            const worker = await this.#ensureWorker();
+            this.#assertActive(signal, runId);
+            let executionStarted = false;
+            try {
+                indicator.assertActive?.();
+                executionStarted = true;
+                const execution = worker.execute(input.code, input.timeoutMs, signal);
+                const executionResult = await (indicator.lost === undefined
+                    ? execution
+                    : Promise.race([
+                        execution,
+                        indicator.lost.then((error) => {
+                            throw error;
+                        })
+                    ]));
+                result = {
+                    outcome: 'SUCCEEDED',
+                    observationId: randomUUID(),
+                    runtimeGeneration: this.#runtimeGeneration,
+                    logs: executionResult.logs,
+                    images: executionResult.images
+                };
+            }
+            catch (error) {
+                if (executionStarted && worker.startedRequest) {
+                    try {
+                        await this.#resetWorker();
+                    }
+                    catch {
+                        // The worker owner records cleanup failure; the sent request remains uncertain.
+                    }
+                    this.#observationId = undefined;
+                    const message = error instanceof Error ? error.message : 'Computer Use execution ended unexpectedly.';
+                    result = {
+                        message,
+                        outcome: 'OUTCOME_UNKNOWN',
+                        observationId: '',
+                        runtimeGeneration: this.#runtimeGeneration,
+                        logs: []
+                    };
+                }
+                else
+                    throw error;
+            }
+        }
+        catch (error) {
+            failed = true;
+            primaryError = error;
+        }
+        try {
+            await indicator.close();
+        }
+        catch (error) {
+            this.#observationId = undefined;
+            this.#cleanupError ??= new ComputerUseError('COMPUTER_CLEANUP_FAILED', 'Computer Use indicator cleanup failed.', { cause: error });
+        }
+        if (failed) {
+            if (this.#cleanupError !== undefined)
+                throw withCleanupFailure(primaryError, this.#cleanupError);
+            throw primaryError;
+        }
+        if (result === undefined)
+            throw new Error('Computer Use execution returned no result.');
+        if (this.#cleanupError !== undefined)
+            return { ...result, observationId: '', cleanupFailed: true };
+        this.#observationId = result.observationId || undefined;
+        return result;
+    }
+    close() {
+        if (this.#closePromise !== undefined)
+            return this.#closePromise;
+        this.#closed = true;
+        this.#observationId = undefined;
+        this.#closePromise = (async () => {
+            let failure;
+            try {
+                await this.#bindingClose?.();
+            }
+            catch (error) {
+                failure = error;
+            }
+            try {
+                await this.#resetWorker();
+            }
+            catch (error) {
+                failure ??= error;
+            }
+            if (failure !== undefined)
+                throw failure;
+            if (this.#runId !== undefined)
+                this.lease.release(this.#runId);
+            this.#runId = undefined;
+            this.#bindingClose = undefined;
+            this.#task = undefined;
+        })();
+        return this.#closePromise;
+    }
+    #assertActive(signal, runId) {
+        if (signal.aborted || this.#closed || this.#runId !== runId)
+            throw new ComputerUseError('COMPUTER_UNAVAILABLE', 'Computer Use binding is closed.');
+    }
+    async #ensureWorker() {
+        if (this.#worker !== undefined)
+            return this.#worker;
+        try {
+            return await PythonWorker.start(this.runtimeExecutable, (worker) => {
+                this.#worker = worker;
+            });
+        }
+        catch (error) {
+            if (error instanceof ComputerUseError && error.code === 'COMPUTER_CLEANUP_FAILED')
+                this.#cleanupError = error;
+            try {
+                await this.#resetWorker();
+            }
+            catch (cleanupError) {
+                throw withCleanupFailure(error, cleanupError);
+            }
+            throw error;
+        }
+    }
+    async #resetWorker() {
+        const worker = this.#worker;
+        if (worker !== undefined) {
+            try {
+                await worker.close();
+            }
+            catch (error) {
+                this.#cleanupError = new ComputerUseError('COMPUTER_CLEANUP_FAILED', 'Computer Use worker cleanup failed.', { cause: error });
+                throw this.#cleanupError;
+            }
+            this.#worker = undefined;
+            this.#runtimeGeneration++;
+        }
+    }
+}
+function waitCancelled() {
+    return new ComputerUseError('COMPUTER_WAIT_CANCELLED', 'Waiting was cancelled; use wait to inspect the task.');
+}
+function withCleanupFailure(primary, cleanup) {
+    const message = `${primary instanceof Error ? primary.message : 'Computer Use execution failed.'} Computer Use cleanup failed; further use is blocked.`;
+    const cause = new AggregateError([primary, cleanup], 'Computer Use execution and cleanup failed');
+    return primary instanceof ComputerUseError
+        ? new ComputerUseError(primary.code, message, { cause })
+        : new Error(message, { cause });
+}

package/dist/computer-use/worker-source.js

@@ -0,0 +1,135 @@
+export const MAX_IMAGE_BYTES = 16 * 1024 * 1024;
+export const MAX_IMAGES = 4;
+export const PYTHON_WORKER_SOURCE = String.raw `
+import base64
+import contextlib
+import io
+import json
+import sys
+import time
+import traceback
+
+import pyautogui
+
+namespace = {"pyautogui": pyautogui, "time": time}
+current_logs = []
+current_images = []
+
+def _emit(message):
+    sys.__stdout__.write(json.dumps(message, ensure_ascii=False, separators=(",", ":")) + "\n")
+    sys.__stdout__.flush()
+
+def log(value):
+    current_logs.append(str(value))
+
+def _image_bytes(image):
+    if hasattr(image, "save"):
+        buffer = io.BytesIO()
+        image.save(buffer, format="PNG")
+        value = buffer.getvalue()
+        if len(value) > ${MAX_IMAGE_BYTES}:
+            raise ValueError("display image exceeds the size limit")
+        return value
+    if isinstance(image, (bytes, bytearray)):
+        value = bytes(image)
+        if len(value) > ${MAX_IMAGE_BYTES}:
+            raise ValueError("display image exceeds the size limit")
+        return value
+    raise TypeError("display expects a PIL image or PNG bytes")
+
+def display(image):
+    if len(current_images) >= ${MAX_IMAGES}:
+        raise ValueError("too many displayed images")
+    current_images.append({
+        "mimeType": "image/png",
+        "dataBase64": base64.b64encode(_image_bytes(image)).decode("ascii")
+    })
+
+def paste_text(text):
+    value = str(text)
+    try:
+        import pyperclip
+        previous = None
+        try:
+            previous = pyperclip.paste()
+        except Exception:
+            pass
+        with open("clipboard.json", "w", encoding="utf-8") as clipboard_file:
+            json.dump({"value": previous}, clipboard_file, ensure_ascii=False)
+        pyperclip.copy(value)
+        pyautogui.hotkey("command" if sys.platform == "darwin" else "ctrl", "v")
+        time.sleep(0.05)
+        if previous is not None:
+            pyperclip.copy(previous)
+        try:
+            import os
+            os.remove("clipboard.json")
+        except OSError:
+            pass
+        return
+    except Exception as error:
+        raise RuntimeError("Unicode clipboard paste is unavailable") from error
+
+namespace.update({"display": display, "log": log, "paste_text": paste_text})
+_emit({"type": "ready"})
+
+for raw in sys.stdin:
+    request = None
+    try:
+        request = json.loads(raw)
+        request_id = request["id"]
+        code = request["code"]
+        current_logs = []
+        current_images = []
+        captured_stdout = io.StringIO()
+        captured_stderr = io.StringIO()
+        with contextlib.redirect_stdout(captured_stdout), contextlib.redirect_stderr(captured_stderr):
+            if code:
+                exec(compile(code, "<computer-use>", "exec"), namespace, namespace)
+            if not current_images:
+                display(pyautogui.screenshot())
+        stdout_value = captured_stdout.getvalue()
+        stderr_value = captured_stderr.getvalue()
+        if stdout_value:
+            current_logs.append(stdout_value.rstrip())
+        if stderr_value:
+            current_logs.append(stderr_value.rstrip())
+        _emit({
+            "type": "result",
+            "id": request_id,
+            "logs": current_logs,
+            "images": current_images
+        })
+    except BaseException:
+        _emit({
+            "type": "error",
+            "id": request.get("id") if isinstance(request, dict) else "",
+            "error": traceback.format_exc(limit=20)
+        })
+`;
+export const PYTHON_CLEANUP_SOURCE = String.raw `
+import json
+import os
+
+import pyautogui
+
+for key in ("ctrl", "shift", "alt", "win", "command"):
+    try:
+        pyautogui.keyUp(key)
+    except Exception:
+        pass
+for button in ("left", "middle", "right"):
+    try:
+        pyautogui.mouseUp(button=button)
+    except Exception:
+        pass
+try:
+    import pyperclip
+    if os.path.exists("clipboard.json"):
+        with open("clipboard.json", "r", encoding="utf-8") as clipboard_file:
+            previous = json.load(clipboard_file).get("value")
+        if previous is not None:
+            pyperclip.copy(previous)
+except Exception:
+    pass
+`;