@ycheung5/pi-computer-use 0.0.0-stage → 0.1.0

package/dist/text.d.ts

@@ -0,0 +1,100 @@
+import type { WorkerResult } from "./protocol.js";
+export declare const textTools: ({
+    action: string;
+    parameters: {
+        type: string;
+        additionalProperties: boolean;
+        properties: {
+            text: {
+                type: string;
+                minLength: number;
+                maxLength: number;
+            };
+            method: {
+                type: string;
+                enum: string[];
+                default: string;
+            };
+            focusMode: {
+                type: string;
+                enum: string[];
+                default: string;
+            };
+            restoreClipboard: {
+                type: string;
+                default: boolean;
+            };
+            windowRef: {
+                type: string;
+                minLength: number;
+            };
+            windowTitle: {
+                type: string;
+            };
+            processId: {
+                type: string;
+                minimum: number;
+            };
+            nativeWindowHandle: {
+                type: string;
+                minimum: number;
+            };
+            activate: {
+                type: string;
+            };
+            browserTab: {
+                type: string;
+                minLength: number;
+            };
+        };
+        required: string[];
+    };
+} | {
+    action: string;
+    parameters: {
+        type: string;
+        additionalProperties: boolean;
+        properties: {
+            keys: {
+                type: string;
+                items: {
+                    type: string;
+                    minLength: number;
+                };
+                minItems: number;
+                maxItems: number;
+            };
+            dispatch: {
+                type: string;
+                enum: string[];
+                default: string;
+            };
+            windowRef: {
+                type: string;
+                minLength: number;
+            };
+            windowTitle: {
+                type: string;
+            };
+            processId: {
+                type: string;
+                minimum: number;
+            };
+            nativeWindowHandle: {
+                type: string;
+                minimum: number;
+            };
+            activate: {
+                type: string;
+            };
+            browserTab: {
+                type: string;
+                minLength: number;
+            };
+        };
+        required: string[];
+    };
+})[];
+export declare function validateText(action: string, args: Record<string, unknown>): WorkerResult | undefined;
+export declare function prepareTextInput(action: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<Record<string, unknown>>;
+export declare function executeText(action: string, args: Record<string, unknown>, cancelEvent: string, operationId: string, listenerProcessId?: number, signal?: AbortSignal): Promise<WorkerResult>;

package/dist/text.js

@@ -0,0 +1,183 @@
+import { compileNative } from "./native.js";
+import { executePointer } from "./pointer.js";
+const target = {
+    windowRef: { type: "string", minLength: 1 },
+    windowTitle: { type: "string" },
+    processId: { type: "integer", minimum: 1 },
+    nativeWindowHandle: { type: "integer", minimum: 1 },
+    activate: { type: "boolean" },
+    browserTab: { type: "string", minLength: 1 },
+};
+export const textTools = [
+    {
+        action: "type_text",
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            properties: {
+                ...target,
+                text: { type: "string", minLength: 1, maxLength: 1000000 },
+                method: {
+                    type: "string",
+                    enum: ["auto", "clipboard", "sendinput", "background"],
+                    default: "auto",
+                },
+                focusMode: {
+                    type: "string",
+                    enum: ["preserve", "main"],
+                    default: "preserve",
+                },
+                restoreClipboard: { type: "boolean", default: true },
+            },
+            required: ["text"],
+        },
+    },
+    {
+        action: "keypress",
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            properties: {
+                ...target,
+                keys: {
+                    type: "array",
+                    items: { type: "string", minLength: 1 },
+                    minItems: 1,
+                    maxItems: 4,
+                },
+                dispatch: {
+                    type: "string",
+                    enum: ["foreground", "background"],
+                    default: "foreground",
+                },
+            },
+            required: ["keys"],
+        },
+    },
+];
+const modifiers = ["ctrl", "control", "alt", "option", "shift"];
+const special = new Set([
+    "enter",
+    "return",
+    "tab",
+    "esc",
+    "escape",
+    "backspace",
+    "delete",
+    "del",
+    "home",
+    "end",
+    "pageup",
+    "pgup",
+    "pagedown",
+    "pgdn",
+    "up",
+    "down",
+    "left",
+    "right",
+    "space",
+    "insert",
+    "ins",
+    "pause",
+    "capslock",
+    "numlock",
+    "scrolllock",
+    ...Array.from({ length: 24 }, (_, i) => `f${i + 1}`),
+]);
+export function validateText(action, args) {
+    const error = (message) => ({
+        ok: false,
+        action,
+        error: message,
+    });
+    const bad = (message) => error(`INVALID_ARGUMENTS: ${message}`);
+    const allowed = new Set([
+        ...Object.keys(target),
+        ...(action === "type_text"
+            ? ["text", "method", "focusMode", "restoreClipboard"]
+            : ["keys", "dispatch"]),
+    ]);
+    if (args.windowRef !== undefined &&
+        (typeof args.windowRef !== "string" || !args.windowRef.trim()))
+        return bad("windowRef must be nonempty");
+    for (const key of Object.keys(args))
+        if (!allowed.has(key))
+            return bad(`unknown option ${key}`);
+    for (const key of ["processId", "nativeWindowHandle"])
+        if (args[key] !== undefined &&
+            (!Number.isSafeInteger(args[key]) || Number(args[key]) < 1))
+            return bad(`${key} must be a positive integer`);
+    if (args.windowTitle !== undefined &&
+        (typeof args.windowTitle !== "string" || !args.windowTitle.trim()))
+        return bad("windowTitle must be nonempty");
+    if (args.activate !== undefined && typeof args.activate !== "boolean")
+        return bad("activate must be boolean");
+    if (action === "type_text") {
+        if (typeof args.text !== "string" ||
+            args.text.length === 0 ||
+            args.text.length > 1000000 ||
+            args.text.includes("\0"))
+            return bad("text must be nonempty, without NUL, at most 1000000 UTF-16 units");
+        if (args.method !== undefined &&
+            !["auto", "clipboard", "sendinput", "background"].includes(String(args.method)))
+            return bad("unsupported method");
+        if (args.focusMode !== undefined &&
+            !["preserve", "main"].includes(String(args.focusMode)))
+            return bad("unsupported focusMode");
+        if (args.restoreClipboard !== undefined &&
+            typeof args.restoreClipboard !== "boolean")
+            return bad("restoreClipboard must be boolean");
+    }
+    else {
+        if (!Array.isArray(args.keys) ||
+            args.keys.length < 1 ||
+            args.keys.length > 4 ||
+            args.keys.some((k) => typeof k !== "string" || !k.length))
+            return bad("keys requires 1–4 key names");
+        let main = 0;
+        const seen = new Set();
+        for (const raw of args.keys) {
+            const key = raw.toLowerCase().trim();
+            if (["win", "windows", "cmd", "meta"].includes(key))
+                return error("KEY_BLOCKED: Windows key is blocked by design");
+            if (modifiers.includes(key)) {
+                const canonical = key === "control" ? "ctrl" : key === "option" ? "alt" : key;
+                if (seen.has(canonical))
+                    return bad("duplicate modifier");
+                seen.add(canonical);
+            }
+            else {
+                main++;
+                if (!special.has(key) &&
+                    !(raw.length === 1 &&
+                        raw.charCodeAt(0) >= 32 &&
+                        raw.charCodeAt(0) !== 127))
+                    return bad("unsupported key");
+            }
+        }
+        if (main !== 1)
+            return bad("exactly one non-modifier key is required");
+        if (args.dispatch !== undefined &&
+            !["foreground", "background"].includes(String(args.dispatch)))
+            return bad("unsupported dispatch");
+        if (args.dispatch === "background" &&
+            (args.keys.length !== 1 ||
+                args.keys[0].length !== 1 ||
+                args.keys[0].charCodeAt(0) < 32))
+            return error("BACKGROUND_UNAVAILABLE: only one printable character can be posted");
+    }
+    return undefined;
+}
+export async function prepareTextInput(action, args, signal) {
+    if (action === "type_text" &&
+        ["auto", "clipboard"].includes(String(args.method ?? "auto")))
+        return {
+            ...args,
+            _clipboardExecutable: (await compileNative(["clipboard.cs"], [], signal))
+                .executable,
+        };
+    return args;
+}
+export async function executeText(action, args, cancelEvent, operationId, listenerProcessId, signal) {
+    return executePointer(action, args, cancelEvent, operationId, listenerProcessId, signal);
+}

package/dist/waits.d.ts

@@ -0,0 +1,19 @@
+import type { WorkerResult } from "./protocol.js";
+type Rule = {
+    type: "integer" | "string" | "boolean";
+    default?: unknown;
+    minimum?: number;
+    maximum?: number;
+    minLength?: number;
+    enum?: string[];
+};
+export declare const waitTools: {
+    action: string;
+    parameters: {
+        type: string;
+        additionalProperties: boolean;
+        properties: Record<string, Rule>;
+    };
+}[];
+export declare function waitDesktop(action: string, input: Record<string, unknown>, signal?: AbortSignal): Promise<WorkerResult>;
+export {};

package/dist/waits.js

@@ -0,0 +1,262 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { executeAccessibility } from "./accessibility.js";
+import { retireDesktop } from "./desktop.js";
+import { nativeDesktop } from "./native.js";
+import { ocrDesktop } from "./ocr.js";
+const waitDefinition = {
+    milliseconds: { type: "integer", minimum: 1, maximum: 30000, default: 500 },
+};
+const conditionDefinition = {
+    windowRef: { type: "string", minLength: 1 },
+    windowTitle: { type: "string" },
+    processId: { type: "integer", minimum: 1 },
+    nativeWindowHandle: { type: "integer", minimum: 1 },
+    activate: { type: "boolean", default: false },
+    scope: {
+        type: "string",
+        enum: ["active_window", "desktop"],
+        default: "active_window",
+    },
+    condition: {
+        type: "string",
+        enum: ["window", "element", "text"],
+        default: "window",
+    },
+    query: { type: "string" },
+    controlType: { type: "string" },
+    viewMode: {
+        type: "string",
+        enum: ["control", "content", "raw"],
+        default: "control",
+    },
+    includeOffscreen: { type: "boolean", default: false },
+    treeTimeoutMs: {
+        type: "integer",
+        minimum: 100,
+        maximum: 30000,
+        default: 1000,
+    },
+    maxDepth: { type: "integer", minimum: 0, maximum: 12, default: 8 },
+    maxNodes: { type: "integer", minimum: 1, maximum: 5000, default: 1200 },
+    appear: { type: "boolean", default: true },
+    timeoutMs: { type: "integer", minimum: 100, maximum: 60000, default: 5000 },
+    intervalMs: { type: "integer", minimum: 50, maximum: 5000, default: 250 },
+};
+export const waitTools = [
+    {
+        action: "wait",
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            properties: waitDefinition,
+        },
+    },
+    {
+        action: "wait_for",
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            properties: conditionDefinition,
+        },
+    },
+];
+async function observation(args, signal) {
+    if (args.condition === "window") {
+        // Internal enumeration has a higher cap than the model-facing listing tool.
+        const result = await nativeDesktop.execute("list_windows", {
+            includeInvisible: true,
+            maxWindows: 4096,
+            _metadataOnly: true,
+            ...(args.windowRef === undefined
+                ? {}
+                : {
+                    windowRef: args.windowRef,
+                    ...(args.nativeWindowHandle === undefined
+                        ? {}
+                        : { nativeWindowHandle: args.nativeWindowHandle }),
+                    ...(args.processId === undefined
+                        ? {}
+                        : { processId: args.processId }),
+                    ...(args.windowTitle === undefined
+                        ? {}
+                        : { windowTitle: args.windowTitle }),
+                }),
+        }, signal);
+        if (!result.ok)
+            return { found: false, conclusive: false, error: String(result.error) };
+        if (typeof result.referencePresent === "boolean")
+            return { found: result.referencePresent, conclusive: true };
+        const windows = result.windows;
+        const found = windows.some((window) => (args.nativeWindowHandle === undefined ||
+            window.nativeWindowHandle === args.nativeWindowHandle) &&
+            (args.processId === undefined || window.processId === args.processId) &&
+            (args.windowTitle === undefined ||
+                String(window.name)
+                    .toLowerCase()
+                    .includes(String(args.windowTitle).toLowerCase())));
+        return {
+            found,
+            conclusive: found || result.truncated !== true,
+            truncated: result.truncated === true,
+        };
+    }
+    const target = {
+        scope: args.scope,
+        activate: false,
+    };
+    for (const key of [
+        "windowRef",
+        "windowTitle",
+        "processId",
+        "nativeWindowHandle",
+    ])
+        if (args[key] !== undefined)
+            target[key] = args[key];
+    if (args.condition === "text") {
+        const result = await ocrDesktop({ ...target, query: args.query, resolveControl: false }, signal);
+        const found = result.ok && Array.isArray(result.matched) && result.matched.length > 0;
+        return {
+            found,
+            conclusive: result.ok &&
+                result.captureConclusive === true &&
+                result.occludedPossible !== true &&
+                result.truncated !== true,
+            ...(!result.ok ? { error: String(result.error) } : {}),
+        };
+    }
+    const result = await executeAccessibility("find", {
+        ...target,
+        query: args.query,
+        ...(args.controlType !== undefined
+            ? { controlType: args.controlType }
+            : {}),
+        viewMode: args.viewMode,
+        includeOffscreen: args.includeOffscreen,
+        treeTimeoutMs: args.treeTimeoutMs,
+        maxDepth: args.maxDepth,
+        maxNodes: args.maxNodes,
+        maxResults: 1,
+        ocrFallback: false,
+    }, signal);
+    const found = result.ok && Array.isArray(result.matches) && result.matches.length > 0;
+    return {
+        found,
+        conclusive: result.ok &&
+            (found ||
+                (result.truncated !== true && result.providerError === undefined)),
+        truncated: result.truncated === true,
+        ...(!result.ok ? { error: String(result.error) } : {}),
+    };
+}
+export async function waitDesktop(action, input, signal) {
+    const definition = action === "wait" ? waitDefinition : conditionDefinition;
+    const fail = (message) => ({
+        ok: false,
+        action,
+        error: `INVALID_ARGUMENTS: ${message}`,
+    });
+    if (action !== "wait" && action !== "wait_for")
+        return fail("unknown wait action");
+    const args = {};
+    for (const key of Object.keys(input))
+        if (!(key in definition))
+            return fail(`unknown option ${key}`);
+    for (const [key, rule] of Object.entries(definition)) {
+        const value = Object.hasOwn(input, key) ? input[key] : rule.default;
+        if (value === undefined)
+            continue;
+        if (rule.type === "integer"
+            ? !Number.isSafeInteger(value)
+            : typeof value !== rule.type)
+            return fail(`invalid ${key}`);
+        if (typeof value === "number" &&
+            ((rule.minimum !== undefined && value < rule.minimum) ||
+                (rule.maximum !== undefined && value > rule.maximum)))
+            return fail(`${key} outside bounds`);
+        if (typeof value === "string" &&
+            ((rule.enum && !rule.enum.includes(value)) ||
+                ((key === "query" || key === "windowTitle" || key === "windowRef") &&
+                    !value.trim())))
+            return fail(`invalid ${key}`);
+        args[key] = value;
+    }
+    signal?.throwIfAborted();
+    if (action === "wait") {
+        await delay(Number(args.milliseconds), undefined, { signal });
+        return { ok: true, action, milliseconds: args.milliseconds };
+    }
+    if (args.condition === "window" &&
+        args.windowRef === undefined &&
+        args.nativeWindowHandle === undefined &&
+        args.processId === undefined &&
+        args.windowTitle === undefined)
+        return fail("window condition requires an explicit target");
+    if (args.condition !== "window" && !String(args.query ?? "").trim())
+        return fail("element/text condition requires query");
+    const start = performance.now(), timeout = Number(args.timeoutMs);
+    let last = { found: false, conclusive: false };
+    let attempts = 0;
+    const outcome = (observed) => {
+        return {
+            ok: observed,
+            action,
+            condition: args.condition,
+            appear: args.appear,
+            ...(args.query !== undefined ? { query: args.query } : {}),
+            found: last.found,
+            conclusive: last.conclusive,
+            verification: observed
+                ? "observed"
+                : last.conclusive
+                    ? "timeout"
+                    : "inconclusive",
+            timedOut: !observed,
+            waitedMs: Math.round(performance.now() - start),
+            attempts,
+            ...(last.truncated ? { observationTruncated: true } : {}),
+            ...(last.error ? { observationError: last.error } : {}),
+            ...(args.condition === "text"
+                ? {
+                    verificationNote: "Condition describes literal OCR output from a complete reliable capture, not universal proof of application text.",
+                }
+                : {}),
+        };
+    };
+    for (;;) {
+        if (attempts > 0 && performance.now() - start >= timeout)
+            return outcome(false);
+        signal?.throwIfAborted();
+        const remaining = Math.max(1, Math.ceil(timeout - (performance.now() - start)));
+        if (attempts > 0 && remaining < 100) {
+            await delay(remaining, undefined, { signal });
+            return outcome(false);
+        }
+        const deadline = AbortSignal.timeout(remaining);
+        const pollSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
+        try {
+            last = await observation({
+                ...args,
+                treeTimeoutMs: Math.max(100, Math.min(Number(args.treeTimeoutMs), remaining)),
+            }, pollSignal);
+        }
+        catch (error) {
+            signal?.throwIfAborted();
+            last = {
+                found: false,
+                conclusive: false,
+                error: deadline.aborted
+                    ? "OBSERVATION_TIMEOUT: condition could not be established before its deadline"
+                    : String(error),
+            };
+            if (deadline.aborted)
+                await retireDesktop();
+        }
+        attempts++;
+        signal?.throwIfAborted();
+        const observed = last.conclusive && last.found === args.appear;
+        const waitedMs = Math.round(performance.now() - start);
+        if (observed || waitedMs >= timeout)
+            return outcome(observed);
+        await delay(Math.min(Number(args.intervalMs), Math.max(1, timeout - (performance.now() - start))), undefined, { signal });
+    }
+}

package/dist/window-management.d.ts

@@ -0,0 +1,25 @@
+import type { WorkerResult } from "./protocol.js";
+type Rule = {
+    type: "string" | "integer" | "boolean";
+    minimum?: number;
+    maximum?: number;
+    minLength?: number;
+    maxLength?: number;
+    default?: unknown;
+};
+export declare const windowTools: {
+    action: string;
+    parameters: {
+        properties: Record<string, Rule> | {
+            browserTab: {
+                type: string;
+                minLength: number;
+            };
+        };
+        required?: string[];
+        type: string;
+        additionalProperties: boolean;
+    };
+}[];
+export declare function manageWindow(action: string, input: Record<string, unknown>, signal?: AbortSignal): Promise<WorkerResult>;
+export {};