@ycheung5/pi-computer-use 0.0.0-stage → 0.1.1

package/dist/launcher.js

@@ -0,0 +1,85 @@
+import { execFile } from "node:child_process";
+import { access } from "node:fs/promises";
+import { release } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { provisionRuntime } from "./runtime.js";
+const run = promisify(execFile);
+let cachedWindowsEnvironment;
+export async function windowsHostEnvironment(signal) {
+    if (process.platform === "win32") {
+        if (!process.env.LOCALAPPDATA)
+            throw new Error("WINDOWS_DATA_DIRECTORY_MISSING: LOCALAPPDATA is required");
+        return { localData: process.env.LOCALAPPDATA, architecture: process.arch };
+    }
+    if (cachedWindowsEnvironment)
+        return cachedWindowsEnvironment;
+    let environment;
+    try {
+        // Fixed variable names, not expanded values: paths containing shell characters remain data.
+        const result = await run("cmd.exe", ["/u", "/d", "/c", "set LOCALAPPDATA & set PROCESSOR_ARCHITECTURE"], { signal, timeout: 5000, maxBuffer: 65536, encoding: "buffer" });
+        const values = new Map(result.stdout
+            .toString("utf16le")
+            .split(/\r?\n/u)
+            .map((line) => {
+            const equals = line.indexOf("=");
+            return [line.slice(0, equals), line.slice(equals + 1)];
+        }));
+        environment = {
+            localData: values.get("LOCALAPPDATA") ?? "",
+            architecture: values.get("PROCESSOR_ARCHITECTURE") ?? "",
+        };
+        if (!environment.localData || !environment.architecture)
+            throw new Error("Windows environment is incomplete");
+    }
+    catch (error) {
+        signal?.throwIfAborted();
+        throw new Error("WSL_BOOTSTRAP_FAILED: enable Windows executable interop and ensure cmd.exe and wslpath are available", { cause: error });
+    }
+    const translate = async (mode, path) => (await run("wslpath", [mode, path], { signal, timeout: 5000 })).stdout.trim();
+    const localData = await translate("-u", environment.localData);
+    const architecture = environment.architecture === "AMD64"
+        ? "x64"
+        : environment.architecture === "ARM64"
+            ? "arm64"
+            : environment.architecture;
+    cachedWindowsEnvironment = { localData, architecture };
+    return cachedWindowsEnvironment;
+}
+async function wslWorkerLaunch(worker, signal) {
+    const { localData, architecture } = await windowsHostEnvironment(signal);
+    const translate = async (mode, path) => (await run("wslpath", [mode, path], { signal, timeout: 5000 })).stdout.trim();
+    const executable = await provisionRuntime({
+        directory: join(localData, "pi-computer-use", "runtime"),
+        architecture,
+        signal,
+    });
+    const windowsWorker = await translate("-w", worker);
+    return { executable, args: [windowsWorker], cwd: localData };
+}
+export async function nativeWorkerLaunch(signal) {
+    signal?.throwIfAborted();
+    const wsl = process.platform === "linux" && /microsoft.*wsl2/i.test(release());
+    if (process.platform !== "win32" && !wsl) {
+        throw new Error("UNSUPPORTED_HOST: computer use requires native Windows or WSL2 with Windows interop");
+    }
+    const worker = fileURLToPath(new URL("./worker.js", import.meta.url));
+    try {
+        await access(worker);
+    }
+    catch {
+        throw new Error("WORKER_ASSET_MISSING: install the built package, or run npm run build in a checkout");
+    }
+    if (wsl)
+        return wslWorkerLaunch(worker, signal);
+    const localData = process.env.LOCALAPPDATA;
+    if (!localData)
+        throw new Error("WINDOWS_DATA_DIRECTORY_MISSING: LOCALAPPDATA is required for the private runtime cache");
+    const executable = await provisionRuntime({
+        directory: join(localData, "pi-computer-use", "runtime"),
+        architecture: process.arch,
+        signal,
+    });
+    return { executable, args: [worker], cwd: localData };
+}

package/dist/native.d.ts

@@ -0,0 +1,11 @@
+import { WorkerClient, type WorkerLaunch } from "./client.js";
+import type { WorkerResult } from "./protocol.js";
+export declare function compileNative(sourceNames: string[], references?: string[], signal?: AbortSignal): Promise<WorkerLaunch>;
+export declare function retainInput(): void;
+export declare function confirmInputCleanup(): void;
+export declare function runNativeOperation<T>(operationId: string, action: () => T): T;
+declare class NativeDesktopClient extends WorkerClient {
+    execute(action: string, args: Record<string, unknown>, signal?: AbortSignal, operationId?: string | undefined): Promise<WorkerResult>;
+}
+export declare const nativeDesktop: NativeDesktopClient;
+export {};

package/dist/native.js

@@ -0,0 +1,146 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { spawn } from "node:child_process";
+import { createHash, randomUUID } from "node:crypto";
+import { access, mkdir, readFile, rename, rm } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { WorkerClient } from "./client.js";
+export async function compileNative(sourceNames, references = [], signal) {
+    if (process.platform !== "win32")
+        throw new Error("UNSUPPORTED_HOST: native desktop access requires Windows");
+    const sources = sourceNames.map((name) => fileURLToPath(new URL(`../assets/${name}`, import.meta.url)));
+    const digest = createHash("sha256");
+    for (const source of sources) {
+        digest.update(source);
+        try {
+            digest.update(await readFile(source));
+        }
+        catch (error) {
+            throw new Error(`NATIVE_RESOURCE_MISSING: required resource ${source}`, {
+                cause: error,
+            });
+        }
+    }
+    digest.update(references.join("\n"));
+    const hash = digest.digest("hex").slice(0, 20);
+    const data = process.env.LOCALAPPDATA;
+    const windows = process.env.WINDIR;
+    if (!data || !windows)
+        throw new Error("NATIVE_SETUP_FAILED: LOCALAPPDATA and WINDIR are required");
+    const directory = join(data, "pi-computer-use", "native", hash);
+    const executable = join(directory, "windows.exe");
+    try {
+        await access(executable);
+        return { executable, args: [], cwd: directory };
+    }
+    catch {
+        /* Compile this package's native source once per content version. */
+    }
+    const compiler = join(windows, "Microsoft.NET", "Framework64", "v4.0.30319", "csc.exe");
+    try {
+        await access(compiler);
+    }
+    catch {
+        throw new Error("NATIVE_COMPILER_MISSING: Windows .NET Framework64 csc.exe is unavailable");
+    }
+    signal?.throwIfAborted();
+    await mkdir(directory, { recursive: true });
+    const temporary = join(directory, `windows-${randomUUID()}.exe`);
+    try {
+        await new Promise((resolve, reject) => {
+            const child = spawn(compiler, [
+                "/nologo",
+                "/target:exe",
+                `/out:${temporary}`,
+                "/r:System.Web.Extensions.dll",
+                ...references.map((reference) => `/r:${reference}`),
+                ...sources,
+            ], {
+                stdio: ["ignore", "pipe", "pipe"],
+                windowsHide: true,
+                cwd: dirname(compiler),
+                signal,
+            });
+            let diagnostic = "";
+            for (const stream of [child.stdout, child.stderr])
+                stream.on("data", (chunk) => {
+                    diagnostic = (diagnostic + chunk).slice(-4096);
+                });
+            const timer = setTimeout(() => {
+                child.kill();
+                reject(new Error("NATIVE_COMPILE_TIMEOUT"));
+            }, 30_000);
+            child.once("error", (error) => {
+                clearTimeout(timer);
+                reject(error);
+            });
+            child.once("close", (code) => {
+                clearTimeout(timer);
+                code === 0
+                    ? resolve()
+                    : reject(new Error(`NATIVE_COMPILE_FAILED: ${diagnostic.trim()}`));
+            });
+        });
+        signal?.throwIfAborted();
+        try {
+            await rename(temporary, executable);
+        }
+        catch (error) {
+            if (!(error instanceof Error &&
+                "code" in error &&
+                (error.code === "EEXIST" || error.code === "EPERM")))
+                throw error;
+            await access(executable);
+        }
+        return { executable, args: [], cwd: directory };
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+}
+let inputPending = false;
+export function retainInput() {
+    inputPending = true;
+}
+export function confirmInputCleanup() {
+    inputPending = false;
+}
+const operationContext = new AsyncLocalStorage();
+export function runNativeOperation(operationId, action) {
+    return operationContext.run(operationId, action);
+}
+class NativeDesktopClient extends WorkerClient {
+    execute(action, args, signal, operationId = operationContext.getStore()) {
+        return super.execute(action, args, signal, operationId);
+    }
+}
+export const nativeDesktop = new NativeDesktopClient({
+    launch: async (signal) => {
+        const names = [
+            "windows.cs",
+            "window-marker.cs",
+            "window-references.cs",
+            "accessibility.cs",
+            "capture.cs",
+            "controls.cs",
+            "window-management.cs",
+            "pointer.cs",
+            "browser-indicators.cs",
+            "text.cs",
+            "input-timing.cs",
+        ];
+        const framework = join(process.env.WINDIR ?? "", "Microsoft.NET", "Framework64", "v4.0.30319");
+        return compileNative(names, [
+            "System.Drawing.dll",
+            "Microsoft.CSharp.dll",
+            "System.Windows.Forms.dll",
+            ...[
+                "WindowsBase.dll",
+                "UIAutomationClient.dll",
+                "UIAutomationTypes.dll",
+            ].map((name) => join(framework, "WPF", name)),
+        ], signal);
+    },
+    timeoutMs: 40_000,
+    canTerminate: () => !inputPending,
+});

package/dist/ocr.d.ts

@@ -0,0 +1,52 @@
+import type { WorkerResult } from "./protocol.js";
+export declare const ocrParameters: {
+    type: string;
+    additionalProperties: boolean;
+    properties: {
+        windowRef: {
+            type: string;
+            minLength: number;
+        };
+        windowTitle: {
+            type: string;
+            minLength: number;
+        };
+        processId: {
+            type: string;
+            minimum: number;
+        };
+        nativeWindowHandle: {
+            type: string;
+            minimum: number;
+        };
+        activate: {
+            type: string;
+            default: boolean;
+        };
+        scope: {
+            type: string;
+            enum: string[];
+            default: string;
+        };
+        maxWidth: {
+            type: string;
+            minimum: number;
+            maximum: number;
+            default: number;
+        };
+        query: {
+            type: string;
+            minLength: number;
+        };
+        resolveControl: {
+            type: string;
+            default: boolean;
+        };
+        includeScreenshot: {
+            type: string;
+            default: boolean;
+        };
+    };
+};
+export declare function ocrDesktop(args: Record<string, unknown>, signal?: AbortSignal): Promise<WorkerResult>;
+export declare function disposeOcr(): Promise<void>;

package/dist/ocr.js

@@ -0,0 +1,246 @@
+import { join } from "node:path";
+import { screenshot } from "./capture.js";
+import { WorkerClient } from "./client.js";
+import { compileNative, nativeDesktop } from "./native.js";
+export const ocrParameters = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        windowRef: { type: "string", minLength: 1 },
+        windowTitle: { type: "string", minLength: 1 },
+        processId: { type: "integer", minimum: 1 },
+        nativeWindowHandle: { type: "integer", minimum: 1 },
+        activate: { type: "boolean", default: false },
+        scope: {
+            type: "string",
+            enum: ["active_window", "desktop"],
+            default: "active_window",
+        },
+        maxWidth: { type: "integer", minimum: 0, maximum: 7680, default: 1920 },
+        query: { type: "string", minLength: 1 },
+        resolveControl: { type: "boolean", default: false },
+        includeScreenshot: { type: "boolean", default: false },
+    },
+};
+const createEngine = () => new WorkerClient({
+    launch: async (signal) => {
+        const windows = process.env.WINDIR;
+        if (!windows)
+            throw new Error("OCR_UNAVAILABLE: Windows metadata is unavailable");
+        const framework = join(windows, "Microsoft.NET", "Framework64", "v4.0.30319");
+        const metadata = join(windows, "System32", "WinMetadata");
+        return compileNative(["ocr.cs"], [
+            "System.Drawing.dll",
+            "System.Core.dll",
+            ...["System.Runtime.dll", "System.Runtime.WindowsRuntime.dll"].map((name) => join(framework, name)),
+            ...[
+                "Windows.Foundation.winmd",
+                "Windows.Media.winmd",
+                "Windows.Graphics.winmd",
+                "Windows.Globalization.winmd",
+            ].map((name) => join(metadata, name)),
+        ], signal);
+    },
+    timeoutMs: 20_000,
+});
+let engine = createEngine();
+function union(words) {
+    if (words.length === 0)
+        return null;
+    const x = Math.min(...words.map((word) => word.x)), y = Math.min(...words.map((word) => word.y));
+    return {
+        x,
+        y,
+        width: Math.max(...words.map((word) => word.x + word.width)) - x,
+        height: Math.max(...words.map((word) => word.y + word.height)) - y,
+    };
+}
+const normalize = (text) => text.replace(/\s/gu, "").toLowerCase();
+export async function ocrDesktop(args, signal) {
+    const fail = (error) => ({
+        ok: false,
+        action: "ocr",
+        error,
+        conclusive: false,
+    });
+    for (const [key, value] of Object.entries(args)) {
+        const rule = ocrParameters.properties[key];
+        if (!rule)
+            return fail(`INVALID_ARGUMENTS: unknown option ${key}`);
+        if (rule.type === "integer"
+            ? !Number.isSafeInteger(value)
+            : typeof value !== rule.type)
+            return fail(`INVALID_ARGUMENTS: invalid ${key}`);
+    }
+    if ((args.windowRef !== undefined && !String(args.windowRef).trim()) ||
+        (args.query !== undefined && !normalize(String(args.query))) ||
+        (args.windowTitle !== undefined && !String(args.windowTitle).trim()) ||
+        [args.processId, args.nativeWindowHandle].some((value) => value !== undefined && Number(value) < 1) ||
+        (args.scope !== undefined &&
+            args.scope !== "active_window" &&
+            args.scope !== "desktop") ||
+        (args.maxWidth !== undefined &&
+            (Number(args.maxWidth) < 0 || Number(args.maxWidth) > 7680)))
+        return fail("INVALID_ARGUMENTS: invalid OCR target, query, scope or maxWidth");
+    signal?.throwIfAborted();
+    const captureArgs = {
+        maxWidth: args.maxWidth ?? 1920,
+    };
+    for (const key of [
+        "scope",
+        "windowRef",
+        "windowTitle",
+        "processId",
+        "nativeWindowHandle",
+        "activate",
+    ])
+        if (args[key] !== undefined)
+            captureArgs[key] = args[key];
+    const captured = await screenshot(captureArgs, signal);
+    signal?.throwIfAborted();
+    if (!captured.ok)
+        return fail(`OCR_CAPTURE_FAILED: ${captured.error}`);
+    const image = captured.screenshot;
+    if (!image || image.windowCaptureFailed)
+        return fail("OCR_CAPTURE_FAILED: target has no capturable surface; desktop fallback is not target evidence");
+    try {
+        const recognized = await engine.execute("recognize", { png: image.base64 }, signal);
+        signal?.throwIfAborted();
+        if (!recognized.ok)
+            return { ...recognized, action: "ocr", conclusive: false };
+        const origin = image.origin;
+        const scale = Number(image.imageScale), scaleY = Number(image.imageScaleY ?? scale);
+        if (!(scale > 0 && scaleY > 0))
+            return fail("OCR_CAPTURE_FAILED: invalid image mapping");
+        const lines = recognized.lines.map((line) => {
+            const words = line.words.map((word) => ({
+                text: word.text,
+                x: origin.x + word.x / scale,
+                y: origin.y + word.y / scaleY,
+                width: word.width / scale,
+                height: word.height / scaleY,
+            }));
+            return { text: line.text, words, boundingBox: union(words) };
+        });
+        const result = {
+            ok: true,
+            action: "ocr",
+            text: recognized.text,
+            lines,
+            language: recognized.language,
+            availableLanguages: recognized.availableLanguages,
+            source: args.scope === "desktop" ? "desktop" : "window",
+            ...(captured.windowRef === undefined
+                ? {}
+                : { windowRef: captured.windowRef }),
+            ...(captured.windowRefUnavailable === undefined
+                ? {}
+                : { windowRefUnavailable: captured.windowRefUnavailable }),
+            coordinateSpace: "screen",
+            imageBounds: image.bounds,
+            imageScale: scale,
+            imageScaleY: scaleY,
+            captureMethod: image.method,
+            occludedPossible: image.occludedPossible,
+            captureWarnings: image.captureWarnings,
+            conclusive: image.occludedPossible !== true,
+            captureConclusive: image.occludedPossible !== true,
+        };
+        if (args.includeScreenshot)
+            result.screenshot = image;
+        if (args.query !== undefined) {
+            const matched = [];
+            const needle = normalize(String(args.query));
+            for (const line of lines) {
+                const normalizedWords = line.words.map((word) => normalize(word.text));
+                const offset = normalizedWords.join("").indexOf(needle);
+                if (offset < 0)
+                    continue;
+                let position = 0;
+                const words = line.words.filter((_, index) => {
+                    const start = position;
+                    position += normalizedWords[index]?.length ?? 0;
+                    return position > offset && start < offset + needle.length;
+                });
+                const bounds = union(words);
+                if (!bounds)
+                    continue;
+                const x = bounds.x + bounds.width / 2, y = bounds.y + bounds.height / 2;
+                let control = null;
+                let controlError;
+                if (args.resolveControl && !image.occludedPossible) {
+                    const target = {
+                        x: Math.round(x),
+                        y: Math.round(y),
+                        detailLevel: "full",
+                    };
+                    for (const key of [
+                        "windowRef",
+                        "windowTitle",
+                        "processId",
+                        "nativeWindowHandle",
+                    ])
+                        if ((captured[key] ?? args[key]) !== undefined)
+                            target[key] = captured[key] ?? args[key];
+                    try {
+                        const observed = await nativeDesktop.execute("element_info", target, signal);
+                        if (observed.ok &&
+                            observed.element &&
+                            typeof observed.element === "object") {
+                            const element = observed.element;
+                            control = Object.fromEntries([
+                                "controlType",
+                                "name",
+                                "automationId",
+                                "className",
+                                "boundingBox",
+                            ]
+                                .filter((key) => element[key] !== undefined)
+                                .map((key) => [key, element[key]]));
+                        }
+                        else
+                            controlError = String(observed.error);
+                    }
+                    catch (error) {
+                        if (signal?.aborted)
+                            throw error;
+                        controlError = String(error);
+                    }
+                }
+                matched.push({
+                    source: "ocr",
+                    line: line.text,
+                    word: { text: words.map((word) => word.text).join(" "), x, y },
+                    boundingBox: bounds,
+                    control,
+                    ...(controlError ? { controlError } : {}),
+                });
+                if (matched.length === 3)
+                    break;
+            }
+            result.query = args.query;
+            result.matched = matched;
+            if (!matched.length)
+                result.conclusive = false;
+            if (!matched.length)
+                result.note =
+                    "No literal OCR word sequence matched (case-insensitive, ignoring whitespace); recognition is not proof that the text is absent.";
+        }
+        if (image.occludedPossible)
+            result.note =
+                "Capture may contain occluding content; OCR cannot establish target text absence or control identity.";
+        return result;
+    }
+    catch (error) {
+        if (signal?.aborted) {
+            const retired = engine;
+            engine = createEngine();
+            await retired.dispose();
+            throw error;
+        }
+        return fail(`OCR_UNAVAILABLE: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+export async function disposeOcr() {
+    await engine.dispose();
+}

package/dist/ownership.d.ts

@@ -0,0 +1,4 @@
+import type { WorkerResult } from "./protocol.js";
+export declare function acquireDesktop(operationId: string, signal?: AbortSignal): Promise<WorkerResult>;
+export declare function releaseDesktop(operationId: string): Promise<WorkerResult>;
+export declare function disposeOwnership(): Promise<void>;

package/dist/ownership.js

@@ -0,0 +1,24 @@
+import { WorkerClient } from "./client.js";
+import { compileNative } from "./native.js";
+const control = new WorkerClient({
+    launch: (signal) => compileNative(["ownership.cs"], [], signal),
+});
+let held;
+export async function acquireDesktop(operationId, signal) {
+    const result = await control.execute("acquire", { operationId }, signal);
+    if (result.ok)
+        held = operationId;
+    return result;
+}
+export async function releaseDesktop(operationId) {
+    if (held !== operationId)
+        return { ok: true, action: "operation_end", released: false };
+    const result = await control.execute("release", { operationId });
+    if (result.ok)
+        held = undefined;
+    return { ...result, action: "operation_end" };
+}
+export async function disposeOwnership() {
+    await control.dispose();
+    held = undefined;
+}