@ycheung5/pi-computer-use 0.0.0-stage → 0.1.1

package/dist/capture.d.ts

@@ -0,0 +1,74 @@
+import type { WorkerResult } from "./protocol.js";
+export declare const screenshotParameters: {
+    type: string;
+    additionalProperties: boolean;
+    properties: {
+        scope: {
+            type: string;
+            enum: string[];
+            default: string;
+        };
+        browserTab: {
+            type: string;
+            minLength: number;
+        };
+        windowRef: {
+            type: string;
+            minLength: number;
+        };
+        windowTitle: {
+            type: string;
+            minLength: number;
+        };
+        processId: {
+            type: string;
+            minimum: number;
+        };
+        nativeWindowHandle: {
+            type: string;
+            minimum: number;
+        };
+        activate: {
+            type: string;
+            default: boolean;
+        };
+        windowOnly: {
+            type: string;
+            default: boolean;
+            description: string;
+        };
+        maxWidth: {
+            type: string;
+            minimum: number;
+            maximum: number;
+            default: number;
+        };
+    };
+};
+export declare function screenshot(args: Record<string, unknown>, signal?: AbortSignal): Promise<WorkerResult>;
+export declare function screenshotContent(result: WorkerResult): {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+    details: WorkerResult;
+    isError: boolean;
+} | {
+    content: ({
+        type: "text";
+        text: string;
+        data?: undefined;
+        mimeType?: undefined;
+    } | {
+        type: "image";
+        data: string;
+        mimeType: string;
+        text?: undefined;
+    })[];
+    details: {
+        screenshot: {};
+        ok: boolean;
+        action: string;
+    };
+    isError: boolean;
+};

package/dist/capture.js

@@ -0,0 +1,199 @@
+import { access } from "node:fs/promises";
+import { join } from "node:path";
+import { WorkerClient } from "./client.js";
+import { compileNative, nativeDesktop } from "./native.js";
+export const screenshotParameters = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        scope: {
+            type: "string",
+            enum: ["active_window", "desktop"],
+            default: "active_window",
+        },
+        browserTab: { type: "string", minLength: 1 },
+        windowRef: { type: "string", minLength: 1 },
+        windowTitle: { type: "string", minLength: 1 },
+        processId: { type: "integer", minimum: 1 },
+        nativeWindowHandle: { type: "integer", minimum: 1 },
+        activate: { type: "boolean", default: false },
+        windowOnly: {
+            type: "boolean",
+            default: false,
+            description: "Native window scope only: use PrintWindow/WGC; refuse screen-copy and desktop fallbacks.",
+        },
+        maxWidth: { type: "integer", minimum: 0, maximum: 7680, default: 1600 },
+    },
+};
+function createWgc() {
+    return new WorkerClient({
+        launch: async (signal) => {
+            const windows = process.env.WINDIR;
+            if (!windows)
+                throw new Error("WGC_UNAVAILABLE: Windows directory is unknown");
+            const framework = join(windows, "Microsoft.NET", "Framework64", "v4.0.30319");
+            const metadata = join(windows, "System32", "WinMetadata");
+            const graphics = [];
+            for (const name of [
+                "Windows.Graphics.winmd",
+                "Windows.Graphics.Capture.winmd",
+                "Windows.Graphics.DirectX.winmd",
+                "Windows.Graphics.DirectX.Direct3D11.winmd",
+                "Windows.Graphics.Imaging.winmd",
+            ]) {
+                const candidate = join(metadata, name);
+                try {
+                    await access(candidate);
+                    graphics.push(candidate);
+                }
+                catch {
+                    /* Optional split/consolidated metadata differs across Windows versions. */
+                }
+            }
+            if (graphics.length === 0)
+                throw new Error("WGC_UNAVAILABLE: Windows graphics metadata is missing");
+            return compileNative(["wgc.cs", "window-marker.cs"], [
+                "System.Drawing.dll",
+                join(framework, "System.Runtime.dll"),
+                join(framework, "System.Runtime.WindowsRuntime.dll"),
+                join(metadata, "Windows.Foundation.winmd"),
+                ...graphics,
+            ], signal);
+        },
+        timeoutMs: 8000,
+    });
+}
+export async function screenshot(args, signal) {
+    signal?.throwIfAborted();
+    const maxWidth = args.maxWidth ?? 1600;
+    const scope = args.scope ?? "active_window";
+    const allowed = new Set([
+        "windowRef",
+        "maxWidth",
+        "scope",
+        "windowTitle",
+        "processId",
+        "nativeWindowHandle",
+        "activate",
+        "windowOnly",
+    ]);
+    const invalid = (args.windowRef !== undefined &&
+        (typeof args.windowRef !== "string" ||
+            !args.windowRef.trim() ||
+            scope === "desktop")) ||
+        Object.keys(args).some((key) => !allowed.has(key)) ||
+        !Number.isSafeInteger(maxWidth) ||
+        typeof maxWidth !== "number" ||
+        maxWidth < 0 ||
+        maxWidth > 7680 ||
+        (scope !== "active_window" && scope !== "desktop") ||
+        (args.windowOnly !== undefined && typeof args.windowOnly !== "boolean") ||
+        (args.windowOnly === true && scope === "desktop") ||
+        (args.windowTitle !== undefined &&
+            (typeof args.windowTitle !== "string" || !args.windowTitle.trim())) ||
+        [args.processId, args.nativeWindowHandle].some((value) => value !== undefined &&
+            (typeof value !== "number" ||
+                !Number.isSafeInteger(value) ||
+                value < 1)) ||
+        (args.activate !== undefined && typeof args.activate !== "boolean");
+    if (invalid)
+        return {
+            ok: false,
+            action: "screenshot",
+            error: "INVALID_ARGUMENTS: invalid screenshot target, scope, windowOnly or maxWidth (integer 0–7680)",
+        };
+    let nativeCaptureError;
+    let wgcError;
+    try {
+        const result = await nativeDesktop.execute("screenshot", { ...args, scope, maxWidth }, signal);
+        nativeCaptureError = result.nativeCaptureError;
+        const target = result.wgcTarget;
+        delete result.wgcTarget;
+        if ((result.ok ||
+            ((result.error === "SCREEN_REGION_CAPTURE_FAILED" ||
+                (args.windowOnly === true &&
+                    result.error === "WINDOW_CAPTURE_UNAVAILABLE")) &&
+                result.cleanupConfirmed !== false &&
+                result.identityWarning === undefined)) &&
+            target &&
+            typeof target === "object" &&
+            !Array.isArray(target)) {
+            let failure;
+            const wgc = createWgc();
+            try {
+                const composited = await wgc.execute("screenshot", { ...target, maxWidth }, signal);
+                if (result.nativeCaptureError !== undefined)
+                    composited.nativeCaptureError = result.nativeCaptureError;
+                if (composited.ok) {
+                    if (result.windowRefUnavailable !== undefined) {
+                        composited.windowRefUnavailable = result.windowRefUnavailable;
+                        if (composited.screenshot &&
+                            typeof composited.screenshot === "object")
+                            composited.screenshot.windowRefUnavailable = result.windowRefUnavailable;
+                    }
+                    return composited;
+                }
+                wgcError = composited.error;
+                if (!result.screenshot ||
+                    composited.cleanupConfirmed === false ||
+                    /WINDOW_REFERENCE|CANCEL|STOP|CLEANUP_UNCONFIRMED/.test(String(composited.error)))
+                    return composited;
+                failure = String(composited.error);
+            }
+            catch (error) {
+                if (signal?.aborted)
+                    throw error;
+                failure = error instanceof Error ? error.message : String(error);
+                wgcError = failure;
+                if (/WINDOW_REFERENCE|CANCEL|STOP|CLEANUP_UNCONFIRMED/.test(failure))
+                    throw error;
+            }
+            finally {
+                await wgc.dispose();
+            }
+            result.wgcError = failure;
+            if (result.screenshot && typeof result.screenshot === "object") {
+                const capture = result.screenshot;
+                capture.captureWarnings = [
+                    ...(Array.isArray(capture.captureWarnings)
+                        ? capture.captureWarnings
+                        : []),
+                    `WGC_UNAVAILABLE: ${failure}`,
+                ];
+            }
+        }
+        return result;
+    }
+    catch (error) {
+        return {
+            ok: false,
+            action: "screenshot",
+            error: error instanceof Error ? error.message : String(error),
+            ...(nativeCaptureError === undefined ? {} : { nativeCaptureError }),
+            ...(wgcError === undefined ? {} : { wgcError }),
+        };
+    }
+}
+export function screenshotContent(result) {
+    const capture = result.screenshot;
+    if (!capture ||
+        typeof capture !== "object" ||
+        !("base64" in capture) ||
+        typeof capture.base64 !== "string") {
+        return {
+            content: [{ type: "text", text: JSON.stringify(result) }],
+            details: result,
+            isError: !result.ok,
+        };
+    }
+    const { base64, ...metadata } = capture;
+    const details = { ...result, screenshot: metadata };
+    return {
+        content: [
+            { type: "text", text: JSON.stringify(details) },
+            { type: "image", data: base64, mimeType: "image/png" },
+        ],
+        details,
+        isError: !result.ok,
+    };
+}

package/dist/client.d.ts

@@ -0,0 +1,39 @@
+import type { WorkerResult } from "./protocol.js";
+export interface WorkerLaunch {
+    executable: string;
+    args: string[];
+    cwd?: string;
+}
+export interface WorkerClientOptions {
+    launch: (signal?: AbortSignal) => Promise<WorkerLaunch>;
+    timeoutMs?: number;
+    shutdownMs?: number;
+    canTerminate?: () => boolean;
+    onStop?: (operationId: string) => void;
+    onInputState?: (active: boolean, operationId?: string) => void;
+    onExit?: () => void;
+}
+export declare class WorkerClient {
+    private readonly options;
+    private child;
+    private starting;
+    private readonly pending;
+    private nextId;
+    private disposed;
+    private failure;
+    private inputActive;
+    private retirementFailure;
+    private retirement;
+    private shutdownChanged;
+    private readonly lifetime;
+    private queue;
+    private readonly deadlineCleanup;
+    constructor(options: WorkerClientOptions);
+    private start;
+    private fail;
+    execute(action: string, args: Record<string, unknown>, signal?: AbortSignal, operationId?: string): Promise<WorkerResult>;
+    private executeNow;
+    stopOperation(operationId: string): void;
+    retire(reason?: string): Promise<void>;
+    dispose(): Promise<void>;
+}

package/dist/client.js

@@ -0,0 +1,384 @@
+import { spawn } from "node:child_process";
+import { executionDeadline, pressTiming, textTiming } from "./input-timing.js";
+function abortable(operation, signal, retain) {
+    return new Promise((resolve, reject) => {
+        const abort = () => {
+            if (!retain?.())
+                reject(signal.reason);
+        };
+        signal.addEventListener("abort", abort, { once: true });
+        if (signal.aborted)
+            abort();
+        operation
+            .then(resolve, reject)
+            .finally(() => signal.removeEventListener("abort", abort));
+    });
+}
+export class WorkerClient {
+    options;
+    child;
+    starting;
+    pending = new Map();
+    nextId = 0;
+    disposed = false;
+    failure;
+    inputActive = false;
+    retirementFailure;
+    retirement;
+    shutdownChanged;
+    lifetime = new AbortController();
+    queue = Promise.resolve();
+    deadlineCleanup = new Set();
+    constructor(options) {
+        this.options = options;
+    }
+    async start(signal) {
+        if (this.disposed)
+            throw new Error("WORKER_DISPOSED: session has closed");
+        signal?.throwIfAborted();
+        if (this.retirement)
+            await this.retirement;
+        if (this.retirementFailure)
+            throw this.retirementFailure;
+        if (this.inputActive && !this.child)
+            throw new Error("INPUT_CLEANUP_UNCONFIRMED: input owner exited without cleanup acknowledgment");
+        if (this.child && this.failure)
+            await this.retire(this.failure.message);
+        if (this.child)
+            return this.child;
+        const launchSignal = signal
+            ? AbortSignal.any([signal, this.lifetime.signal])
+            : this.lifetime.signal;
+        this.starting ??= abortable(this.options.launch(launchSignal), launchSignal)
+            .then((launch) => {
+            if (this.disposed)
+                throw new Error("WORKER_DISPOSED: session has closed");
+            signal?.throwIfAborted();
+            const child = spawn(launch.executable, launch.args, {
+                stdio: ["pipe", "pipe", "pipe"],
+                windowsHide: true,
+                cwd: launch.cwd,
+            });
+            this.child = child;
+            let output = "";
+            let diagnostic = "";
+            child.stdout.setEncoding("utf8");
+            child.stderr.setEncoding("utf8");
+            child.stderr.on("data", (chunk) => {
+                diagnostic = (diagnostic + chunk).slice(-4096);
+            });
+            child.stdout.on("data", (chunk) => {
+                output += chunk;
+                if (output.length > 32 * 1024 * 1024) {
+                    this.fail(child, new Error("WORKER_PROTOCOL_ERROR: response exceeds 32 MiB"));
+                    if (!this.inputActive && this.options.canTerminate?.() !== false)
+                        child.kill();
+                    return;
+                }
+                for (;;) {
+                    const newline = output.indexOf("\n");
+                    if (newline < 0)
+                        break;
+                    const line = output.slice(0, newline);
+                    output = output.slice(newline + 1);
+                    try {
+                        const response = JSON.parse(line);
+                        if (response.event === "shutdown_state") {
+                            if (this.child === child &&
+                                (response.phase === "started" ||
+                                    response.phase === "complete" ||
+                                    response.phase === "failed"))
+                                this.shutdownChanged?.(response.phase, typeof response.error === "string"
+                                    ? response.error.slice(0, 2048)
+                                    : undefined);
+                            continue;
+                        }
+                        if (response.event === "input_prepare") {
+                            const pending = Number.isSafeInteger(response.requestId)
+                                ? this.pending.get(Number(response.requestId))
+                                : undefined;
+                            if (!pending ||
+                                typeof response.token !== "string" ||
+                                response.token.length < 1 ||
+                                response.token.length > 200 ||
+                                typeof response.operationId !== "string" ||
+                                (pending.operationId !== undefined &&
+                                    pending.operationId !== response.operationId))
+                                continue;
+                            this.inputActive = true;
+                            this.options.onInputState?.(true, response.operationId);
+                            child.stdin.write(`${JSON.stringify({ inputAck: response.token, requestId: response.requestId })}\n`);
+                            continue;
+                        }
+                        if (response.event === "input_state" &&
+                            typeof response.active === "boolean") {
+                            this.inputActive = response.active;
+                            this.options.onInputState?.(response.active, response.operationId);
+                            continue;
+                        }
+                        if (response.event === "stopped" &&
+                            typeof response.operationId === "string") {
+                            this.options.onStop?.(response.operationId);
+                            continue;
+                        }
+                        if (!Number.isSafeInteger(response.id) ||
+                            typeof response.result?.ok !== "boolean" ||
+                            typeof response.result.action !== "string")
+                            throw new Error("Invalid response");
+                        const pending = this.pending.get(response.id);
+                        if (pending) {
+                            this.pending.delete(response.id);
+                            pending.cleanup();
+                            pending.resolve(response.result);
+                        }
+                    }
+                    catch {
+                        this.fail(child, new Error("WORKER_PROTOCOL_ERROR: invalid response"));
+                        if (!this.inputActive && this.options.canTerminate?.() !== false)
+                            child.kill();
+                    }
+                }
+            });
+            child.once("error", (error) => this.fail(child, new Error(`WORKER_START_FAILED: ${error.message}`), child.pid === undefined));
+            child.once("close", (code) => {
+                const current = this.child === child;
+                this.fail(child, new Error(`WORKER_EXITED: code ${code}${diagnostic ? `; ${diagnostic.trim()}` : ""}`), true);
+                if (current)
+                    this.options.onExit?.();
+            });
+            child.stdin.on("error", (error) => this.fail(child, new Error(`WORKER_WRITE_FAILED: ${error.message}`)));
+            return child;
+        })
+            .finally(() => {
+            this.starting = undefined;
+        });
+        return this.starting;
+    }
+    fail(child, error, confirmedExit = false) {
+        if (this.child !== child)
+            return;
+        if (confirmedExit) {
+            this.child = undefined;
+            this.failure = undefined;
+            // Process death is not an acknowledgement that its resources were freed.
+            if (!this.retirementFailure?.message.startsWith("WORKER_CLEANUP_UNCONFIRMED:"))
+                this.retirementFailure = undefined;
+        }
+        else
+            this.failure = error;
+        for (const [id, pending] of this.pending) {
+            this.pending.delete(id);
+            pending.cleanup();
+            pending.reject(error);
+        }
+    }
+    execute(action, args, signal, operationId) {
+        if (this.disposed)
+            return Promise.reject(new Error("WORKER_DISPOSED: session has closed"));
+        const callSignal = signal
+            ? AbortSignal.any([signal, this.lifetime.signal])
+            : this.lifetime.signal;
+        const result = this.queue
+            .catch(() => undefined)
+            .then(() => {
+            callSignal.throwIfAborted();
+            return this.executeNow(action, args, callSignal, operationId);
+        });
+        this.queue = result.catch(() => undefined);
+        return abortable(result, callSignal, () => this.deadlineCleanup.has(callSignal));
+    }
+    async executeNow(action, args, signal, operationId) {
+        let inputPolicy;
+        if (action === "keypress" || action === "click" || action === "type_text") {
+            try {
+                inputPolicy = (action === "type_text" ? textTiming : pressTiming)(args, this.options.timeoutMs ?? 30_000);
+            }
+            catch (error) {
+                return {
+                    ok: false,
+                    action,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        }
+        const child = await this.start(signal);
+        if (this.disposed)
+            throw new Error("WORKER_DISPOSED: session has closed");
+        signal?.throwIfAborted();
+        const id = ++this.nextId;
+        return new Promise((resolve, reject) => {
+            let expired = false;
+            let cleanupTimer;
+            const timeoutError = () => new Error(`WORKER_TIMEOUT: ${action} exceeded the execution deadline; input was not replayed`);
+            const timer = executionDeadline(inputPolicy?.timeoutMs ??
+                Math.max(1, this.options.timeoutMs ?? 30_000, action === "wait" &&
+                    Number.isSafeInteger(args.milliseconds ?? 500) &&
+                    Number(args.milliseconds ?? 500) >= 1 &&
+                    Number(args.milliseconds ?? 500) <= 30000
+                    ? Number(args.milliseconds ?? 500) + 2000
+                    : 0, action === "wait_for" &&
+                    Number.isSafeInteger(args.timeoutMs ?? 5000) &&
+                    Number(args.timeoutMs ?? 5000) >= 100 &&
+                    Number(args.timeoutMs ?? 5000) <= 60000
+                    ? Number(args.timeoutMs ?? 5000) + 10000
+                    : 0), () => {
+                if (inputPolicy) {
+                    expired = true;
+                    if (signal)
+                        this.deadlineCleanup.add(signal);
+                    // Keep the request and its owner until cooperative release is acknowledged.
+                    child.stdin.write(`${JSON.stringify({ cancel: id })}\n`);
+                    cleanupTimer = setTimeout(() => {
+                        this.fail(child, timeoutError());
+                        if (!this.inputActive && this.options.canTerminate?.() !== false)
+                            child.kill();
+                    }, 5000);
+                    return;
+                }
+                this.fail(child, new Error(`WORKER_TIMEOUT: ${action} exceeded the execution deadline; input was not replayed`));
+                if (!this.inputActive && this.options.canTerminate?.() !== false)
+                    child.kill();
+            });
+            const abort = () => {
+                if (expired)
+                    return;
+                const pending = this.pending.get(id);
+                if (!pending)
+                    return;
+                this.pending.delete(id);
+                pending.cleanup();
+                // Health has no held input. Cooperative input stopping is added before input tools.
+                child.stdin.write(`${JSON.stringify({ cancel: id })}\n`);
+                pending.reject(new Error("COMPUTER_USE_CANCELLED"));
+            };
+            this.pending.set(id, {
+                operationId,
+                resolve: (result) => resolve(expired
+                    ? {
+                        ...result,
+                        ok: false,
+                        error: result.cleanupConfirmed === false ||
+                            /CLEANUP_UNCONFIRMED/.test(String(result.error))
+                            ? (result.error ??
+                                "INPUT_CLEANUP_UNCONFIRMED: deadline expired before release was confirmed")
+                            : timeoutError().message,
+                        executionDeadlineExpired: true,
+                        interrupted: true,
+                        timing: inputPolicy,
+                    }
+                    : result),
+                reject,
+                cleanup: () => {
+                    timer();
+                    if (signal)
+                        this.deadlineCleanup.delete(signal);
+                    clearTimeout(cleanupTimer);
+                    signal?.removeEventListener("abort", abort);
+                },
+            });
+            signal?.addEventListener("abort", abort, { once: true });
+            if (signal?.aborted) {
+                abort();
+                return;
+            }
+            child.stdin.write(`${JSON.stringify({ id, action, args, operationId })}\n`, (error) => {
+                if (error)
+                    this.fail(child, new Error(`WORKER_WRITE_FAILED: ${error.message}`));
+            });
+        });
+    }
+    stopOperation(operationId) {
+        // Control messages bypass queued calls, including a busy native provider.
+        this.child?.stdin.write(`${JSON.stringify({ stop: operationId })}\n`);
+    }
+    async retire(reason = "WORKER_RETIRED: operation stopped") {
+        if (this.inputActive || this.options.canTerminate?.() === false)
+            throw new Error("INPUT_CLEANUP_UNCONFIRMED: active input prevents bridge retirement");
+        if (this.retirementFailure)
+            throw this.retirementFailure;
+        if (this.retirement)
+            return this.retirement;
+        const child = this.child;
+        if (!child)
+            return;
+        this.fail(child, new Error(reason));
+        this.retirement = new Promise((resolve, reject) => {
+            let forced;
+            let timer;
+            let started = false;
+            let complete = false;
+            let settled = false;
+            const fail = (error) => {
+                settled = true;
+                clearTimeout(timer);
+                clearTimeout(forced);
+                reject(error);
+            };
+            const expire = () => {
+                if (this.inputActive || this.options.canTerminate?.() === false) {
+                    fail(new Error("INPUT_CLEANUP_UNCONFIRMED: active input prevents bridge retirement"));
+                    return;
+                }
+                child.kill();
+                forced = setTimeout(() => fail(new Error("WORKER_RETIRE_FAILED: child exit was not confirmed; desktop ownership must stay held")), 1000);
+            };
+            const arm = (milliseconds) => {
+                clearTimeout(timer);
+                timer = setTimeout(expire, milliseconds);
+            };
+            this.shutdownChanged = (phase, error) => {
+                if (settled)
+                    return;
+                if (phase === "started" && !started) {
+                    started = true;
+                    // Resource cleanup is an operation, not an idle exit. Give it the
+                    // existing execution budget; fast completion adds no fixed delay.
+                    arm(this.options.timeoutMs ?? 30_000);
+                }
+                else if (started && phase === "complete" && !complete) {
+                    complete = true;
+                    arm(this.options.shutdownMs ?? 1000);
+                }
+                else if (started && phase === "failed") {
+                    fail(new Error(`WORKER_CLEANUP_UNCONFIRMED: ${error ?? "worker resource cleanup failed"}`));
+                }
+            };
+            arm(this.options.shutdownMs ?? 1000);
+            child.once("close", () => {
+                clearTimeout(timer);
+                clearTimeout(forced);
+                if (settled)
+                    return;
+                if (this.inputActive || this.options.canTerminate?.() === false) {
+                    fail(new Error("INPUT_CLEANUP_UNCONFIRMED: input owner exited without release"));
+                    return;
+                }
+                if (started && !complete) {
+                    fail(new Error("WORKER_CLEANUP_UNCONFIRMED: worker exited before shutdown completed"));
+                    return;
+                }
+                settled = true;
+                resolve();
+            });
+            child.stdin.end();
+        });
+        try {
+            await this.retirement;
+        }
+        catch (error) {
+            this.retirementFailure =
+                error instanceof Error ? error : new Error(String(error));
+            throw error;
+        }
+        finally {
+            this.shutdownChanged = undefined;
+            this.retirement = undefined;
+        }
+    }
+    async dispose() {
+        this.disposed = true;
+        this.lifetime.abort(new Error("WORKER_DISPOSED: session has closed"));
+        await this.starting?.catch(() => undefined);
+        await this.retire("WORKER_DISPOSED: session has closed");
+    }
+}