@ycheung5/pi-computer-use 0.0.0-stage → 0.1.0

package/dist/accessibility.js

@@ -0,0 +1,258 @@
+import { screenshot } from "./capture.js";
+import { nativeDesktop } from "./native.js";
+import { ocrDesktop } from "./ocr.js";
+const target = {
+    windowRef: { type: "string", minLength: 1 },
+    windowTitle: { type: "string" },
+    processId: { type: "integer", minimum: 1 },
+    nativeWindowHandle: { type: "integer", minimum: 1 },
+    activate: { type: "boolean" },
+};
+const view = {
+    viewMode: {
+        type: "string",
+        enum: ["control", "content", "raw"],
+        default: "control",
+    },
+    includeOffscreen: { type: "boolean", default: false },
+};
+const browser = {
+    browserTab: { type: "string", minLength: 1 },
+};
+const detail = {
+    detailLevel: {
+        type: "string",
+        enum: ["compact", "full"],
+        default: "compact",
+    },
+};
+const tree = {
+    scope: {
+        type: "string",
+        enum: ["active_window", "desktop"],
+        default: "active_window",
+    },
+    treeTimeoutMs: {
+        type: "integer",
+        minimum: 100,
+        maximum: 30000,
+        default: 5000,
+    },
+    maxDepth: { type: "integer", minimum: 0, maximum: 12, default: 6 },
+    maxNodes: { type: "integer", minimum: 1, maximum: 3000, default: 500 },
+};
+const definitions = {
+    accessibility_tree: {
+        properties: { ...target, ...browser, ...view, ...detail, ...tree },
+    },
+    snapshot: {
+        properties: {
+            ...target,
+            ...browser,
+            ...view,
+            ...detail,
+            ...tree,
+            maxDepth: { ...tree.maxDepth, default: 5 },
+            maxNodes: { ...tree.maxNodes, maximum: 2000, default: 250 },
+            includeScreenshot: { type: "boolean", default: true },
+            captureWindow: { type: "boolean", default: false },
+            maxWidth: { type: "integer", minimum: 0, maximum: 7680, default: 1600 },
+        },
+    },
+    find: {
+        properties: {
+            ...target,
+            ...browser,
+            ...view,
+            ...tree,
+            query: { type: "string", minLength: 1 },
+            controlType: { type: "string" },
+            ocrFallback: { type: "boolean", default: true },
+            maxDepth: { ...tree.maxDepth, default: 8 },
+            maxNodes: { ...tree.maxNodes, maximum: 5000, default: 1200 },
+            maxResults: { type: "integer", minimum: 1, maximum: 200, default: 25 },
+        },
+        required: ["query"],
+    },
+    element_info: {
+        properties: {
+            ...target,
+            ...view,
+            ...detail,
+            elementId: { type: "string", minLength: 1 },
+            x: { type: "integer" },
+            y: { type: "integer" },
+        },
+    },
+    focus: {
+        properties: {
+            ...target,
+            ...view,
+            elementId: { type: "string", minLength: 1 },
+        },
+        required: ["elementId"],
+    },
+    invoke: {
+        properties: {
+            ...target,
+            ...view,
+            elementId: { type: "string", minLength: 1 },
+            action: {
+                type: "string",
+                enum: ["default", "expand", "collapse", "scroll_into_view"],
+                default: "default",
+            },
+            fallbackClick: { type: "boolean", default: true },
+        },
+        required: ["elementId"],
+    },
+    set_value: {
+        properties: {
+            ...target,
+            ...view,
+            elementId: { type: "string", minLength: 1 },
+            value: { type: "string" },
+            fallbackType: { type: "boolean", default: true },
+            restoreClipboard: { type: "boolean", default: true },
+        },
+        required: ["elementId", "value"],
+    },
+    scroll: {
+        properties: {
+            ...target,
+            ...view,
+            ...browser,
+            elementId: { type: "string", minLength: 1 },
+            x: { type: "integer" },
+            y: { type: "integer" },
+            deltaY: { type: "integer", default: 480 },
+            deltaX: { type: "integer", default: 0 },
+        },
+    },
+};
+const toolDefinitions = Object.entries(definitions).map(([action, definition]) => ({
+    action,
+    parameters: { type: "object", additionalProperties: false, ...definition },
+}));
+export const observationTools = toolDefinitions.filter((tool) => ["snapshot", "accessibility_tree", "find", "element_info"].includes(tool.action));
+export const controlTools = toolDefinitions.filter((tool) => ["focus", "invoke", "set_value", "scroll"].includes(tool.action));
+export async function executeAccessibility(action, args, signal, execute) {
+    const definition = definitions[action];
+    if (!definition)
+        return { ok: false, action, error: "UNKNOWN_ACTION" };
+    const normalized = {};
+    const invalid = (message) => ({
+        ok: false,
+        action,
+        error: `INVALID_ARGUMENTS: ${message}`,
+    });
+    for (const key of Object.keys(args))
+        if (!(key in definition.properties))
+            return invalid(`unknown option ${key}`);
+    for (const [key, rule] of Object.entries(definition.properties)) {
+        const value = Object.hasOwn(args, key) ? args[key] : rule.default;
+        if (value === undefined) {
+            if (definition.required?.includes(key))
+                return invalid(`${key} is required`);
+            continue;
+        }
+        if (rule.type === "integer"
+            ? !Number.isSafeInteger(value)
+            : typeof value !== rule.type)
+            return invalid(`${key} must be ${rule.type}`);
+        if (typeof value === "number" &&
+            ((rule.minimum !== undefined && value < rule.minimum) ||
+                (rule.maximum !== undefined && value > rule.maximum)))
+            return invalid(`${key} is outside supported bounds`);
+        if (typeof value === "string" &&
+            ((rule.enum && !rule.enum.includes(value)) ||
+                (rule.minLength && value.length < rule.minLength)))
+            return invalid(`${key} has an unsupported value`);
+        normalized[key] = value;
+    }
+    if (action === "element_info" &&
+        !normalized.elementId &&
+        (normalized.x === undefined || normalized.y === undefined))
+        return invalid("provide elementId or both x and y");
+    if (action === "scroll" && !normalized.elementId) {
+        if (normalized.x === undefined || normalized.y === undefined)
+            return invalid("provide elementId or both x and y");
+        return {
+            ok: false,
+            action,
+            error: "INPUT_SAFETY_NOT_READY: coordinate scrolling requires guarded physical input",
+            inputRequired: true,
+        };
+    }
+    signal?.throwIfAborted();
+    let result;
+    // Guarded mutation failures retain the cleanup lease; do not translate them
+    // into ordinary observation errors or clear the outer input state.
+    if (execute)
+        return execute(normalized);
+    try {
+        result = await nativeDesktop.execute(action, normalized, signal);
+    }
+    catch (error) {
+        result = {
+            ok: false,
+            action,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+    if (action === "snapshot" && normalized.includeScreenshot) {
+        delete result.screenshotPending;
+        const windowCapture = normalized.captureWindow && normalized.scope !== "desktop";
+        const captureArgs = {
+            scope: windowCapture ? "active_window" : "desktop",
+            maxWidth: normalized.maxWidth,
+        };
+        if (windowCapture) {
+            for (const key of [
+                "windowRef",
+                "windowTitle",
+                "processId",
+                "nativeWindowHandle",
+                "activate",
+            ]) {
+                const value = result[key] ?? normalized[key];
+                if (value !== undefined)
+                    captureArgs[key] = value;
+            }
+        }
+        const captured = await screenshot(captureArgs, signal);
+        if (captured.ok && captured.screenshot)
+            result.screenshot = captured.screenshot;
+        else {
+            result.partial = true;
+            result.screenshotError = String(captured.error ?? "SCREENSHOT_UNAVAILABLE");
+        }
+    }
+    if (action === "find" && result.ocrFallbackPending) {
+        delete result.ocrFallbackPending;
+        const ocrArgs = {
+            query: normalized.query,
+            scope: normalized.scope,
+        };
+        for (const key of [
+            "windowRef",
+            "windowTitle",
+            "processId",
+            "nativeWindowHandle",
+            "activate",
+        ]) {
+            const value = result[key] ?? normalized[key];
+            if (value !== undefined)
+                ocrArgs[key] = value;
+        }
+        const recognized = await ocrDesktop(ocrArgs, signal);
+        result.ocr = recognized;
+        if (recognized.ok)
+            result.ocrCandidates = recognized.matched ?? [];
+        else {
+            result.ocrError = recognized.error;
+            result.conclusive = false;
+        }
+    }
+    return result;
+}

package/dist/browser-actions.d.ts

@@ -0,0 +1,21 @@
+type Rule = {
+    type: string;
+    default?: unknown;
+    enum?: string[];
+    minimum?: number;
+    maximum?: number;
+    minLength?: number;
+    minItems?: number;
+    items?: Rule;
+};
+export declare const browserActionTools: {
+    action: string;
+    parameters: {
+        properties: Record<string, Rule>;
+        required?: string[];
+        type: string;
+        additionalProperties: boolean;
+    };
+}[];
+export declare function normalizeBrowserAction(action: string, args: Record<string, unknown>): Record<string, unknown> | undefined;
+export {};

package/dist/browser-actions.js

@@ -0,0 +1,213 @@
+// Shared reference-compatible tool schemas; browser execution stays in browser.ts.
+const target = {
+    windowRef: { type: "string", minLength: 1 },
+    windowTitle: { type: "string" },
+    processId: { type: "integer", minimum: 1 },
+    nativeWindowHandle: { type: "integer", minimum: 1 },
+    activate: { type: "boolean", default: false },
+};
+const view = {
+    viewMode: {
+        type: "string",
+        enum: ["control", "content", "raw"],
+        default: "control",
+    },
+    includeOffscreen: { type: "boolean", default: false },
+};
+const tab = { browserTab: { type: "string", minLength: 1 } };
+const element = { elementId: { type: "string", minLength: 1 } };
+const point = { x: { type: "integer" }, y: { type: "integer" } };
+const button = {
+    type: "string",
+    enum: ["left", "right", "middle"],
+    default: "left",
+};
+const dispatch = {
+    type: "string",
+    enum: ["auto", "background", "foreground"],
+    default: "auto",
+};
+const definitions = {
+    click: {
+        properties: { ...target, ...view, ...element, ...point, button, dispatch },
+    },
+    double_click: {
+        properties: { ...target, ...view, ...element, ...point, button, dispatch },
+    },
+    move: { properties: { ...target, ...view, ...element, ...point } },
+    scroll: {
+        properties: {
+            ...target,
+            ...view,
+            ...tab,
+            ...element,
+            ...point,
+            deltaY: { type: "integer", default: 480 },
+            deltaX: { type: "integer", default: 0 },
+        },
+    },
+    type_text: {
+        required: ["text"],
+        properties: {
+            ...target,
+            ...tab,
+            text: { type: "string" },
+            method: {
+                type: "string",
+                enum: ["auto", "clipboard", "sendinput", "background"],
+                default: "auto",
+            },
+            focusMode: {
+                type: "string",
+                enum: ["preserve", "main"],
+                default: "preserve",
+            },
+            restoreClipboard: { type: "boolean", default: true },
+        },
+    },
+    keypress: {
+        required: ["keys"],
+        properties: {
+            ...target,
+            ...tab,
+            keys: {
+                type: "array",
+                minItems: 1,
+                items: { type: "string", minLength: 1 },
+            },
+            dispatch: {
+                type: "string",
+                enum: ["foreground", "background"],
+                default: "foreground",
+            },
+        },
+    },
+    focus: {
+        required: ["elementId"],
+        properties: { ...target, ...view, ...element },
+    },
+    invoke: {
+        required: ["elementId"],
+        properties: {
+            ...target,
+            ...view,
+            ...element,
+            action: {
+                type: "string",
+                enum: ["default", "expand", "collapse", "scroll_into_view"],
+                default: "default",
+            },
+            fallbackClick: { type: "boolean", default: true },
+        },
+    },
+    set_value: {
+        required: ["elementId", "value"],
+        properties: {
+            ...target,
+            ...view,
+            ...element,
+            value: { type: "string" },
+            fallbackType: { type: "boolean", default: true },
+            restoreClipboard: { type: "boolean", default: true },
+        },
+    },
+    activate_window: { properties: { ...target, ...tab } },
+    close_window: { properties: { ...target, ...tab } },
+    wait_for: {
+        properties: {
+            ...target,
+            ...tab,
+            ...view,
+            condition: {
+                type: "string",
+                enum: ["window", "element", "text"],
+                default: "window",
+            },
+            query: { type: "string" },
+            controlType: { type: "string" },
+            scope: {
+                type: "string",
+                enum: ["active_window", "desktop"],
+                default: "active_window",
+            },
+            treeTimeoutMs: {
+                type: "integer",
+                minimum: 100,
+                maximum: 30000,
+                default: 1000,
+            },
+            maxDepth: { type: "integer", minimum: 0, maximum: 12, default: 8 },
+            maxNodes: { type: "integer", minimum: 1, maximum: 5000, default: 1200 },
+            appear: { type: "boolean", default: true },
+            timeoutMs: {
+                type: "integer",
+                minimum: 100,
+                maximum: 60000,
+                default: 5000,
+            },
+            intervalMs: { type: "integer", minimum: 50, maximum: 5000, default: 250 },
+        },
+    },
+};
+export const browserActionTools = Object.entries(definitions).map(([action, definition]) => ({
+    action,
+    parameters: { type: "object", additionalProperties: false, ...definition },
+}));
+export function normalizeBrowserAction(action, args) {
+    const definition = definitions[action];
+    if (!definition)
+        return undefined;
+    const properties = { ...definition.properties, ...tab };
+    for (const key of Object.keys(args))
+        if (!Object.hasOwn(properties, key))
+            throw new Error(`INVALID_ARGUMENTS: unknown option ${key}`);
+    const normalized = {};
+    const check = (name, value, rule) => {
+        if (rule.type === "array") {
+            if (!Array.isArray(value) ||
+                value.length < (rule.minItems ?? 0) ||
+                value.length > 10000)
+                throw new Error(`INVALID_ARGUMENTS: ${name} must be an array with supported length`);
+            if (rule.items)
+                for (const child of value)
+                    check(name, child, rule.items);
+        }
+        else if (rule.type === "integer"
+            ? !Number.isSafeInteger(value)
+            : typeof value !== rule.type)
+            throw new Error(`INVALID_ARGUMENTS: ${name} must be ${rule.type}`);
+        if ((typeof value === "number" &&
+            ((rule.minimum !== undefined && value < rule.minimum) ||
+                (rule.maximum !== undefined && value > rule.maximum))) ||
+            (typeof value === "string" &&
+                ((rule.enum && !rule.enum.includes(value)) ||
+                    (rule.minLength !== undefined && value.length < rule.minLength))))
+            throw new Error(`INVALID_ARGUMENTS: ${name} has an unsupported value`);
+    };
+    for (const [key, rule] of Object.entries(properties)) {
+        const value = Object.hasOwn(args, key) ? args[key] : rule.default;
+        if (value === undefined) {
+            if (definition.required?.includes(key))
+                throw new Error(`INVALID_ARGUMENTS: ${key} is required`);
+            continue;
+        }
+        check(key, value, rule);
+        normalized[key] = value;
+    }
+    if (action === "keypress") {
+        const keys = normalized.keys;
+        if (keys.some((key) => /^(win|windows|meta|cmd|command)$/i.test(key)))
+            throw new Error("INVALID_ARGUMENTS: Windows key is blocked");
+        if (keys.filter((key) => !/^(ctrl|control|alt|option|shift)$/i.test(key))
+            .length !== 1)
+            throw new Error("INVALID_ARGUMENTS: keypress needs exactly one non-modifier key");
+    }
+    if (action === "invoke" && normalized.action !== "default")
+        throw new Error("INVALID_ARGUMENTS: non-default invoke actions apply only to UI Automation elements");
+    if (action === "wait_for" &&
+        (normalized.condition === "window" ||
+            typeof normalized.query !== "string" ||
+            !normalized.query.trim()))
+        throw new Error("INVALID_ARGUMENTS: browser wait_for requires element/text condition and query");
+    return normalized;
+}

package/dist/browser-indicators.d.ts

@@ -0,0 +1,23 @@
+export declare function updateBrowserIndicator(input: {
+    mode: "mark" | "hide" | "show" | "clear";
+    point?: {
+        x: number;
+        y: number;
+    };
+}): {
+    active: boolean;
+    tabBadge: boolean;
+    clickMarker: boolean;
+    hidden?: undefined;
+    ariaHidden?: undefined;
+    pointerEvents?: undefined;
+    shadowClosed?: undefined;
+} | {
+    active: boolean;
+    tabBadge: boolean;
+    clickMarker: boolean;
+    hidden: boolean;
+    ariaHidden: boolean;
+    pointerEvents: string;
+    shadowClosed: boolean;
+};

package/dist/browser-indicators.js

@@ -0,0 +1,69 @@
+// Runs only in this extension's isolated browser realm. Keep the function self-contained.
+export function updateBrowserIndicator(input) {
+    const world = globalThis;
+    let cue = world.__piComputerUseCue;
+    if (input.mode === "clear") {
+        cue?.host.remove();
+        cue?.icon.remove();
+        delete world.__piComputerUseCue;
+        return { active: false, tabBadge: false, clickMarker: false };
+    }
+    if (!cue?.host.isConnected &&
+        input.mode === "mark" &&
+        document.documentElement) {
+        cue?.icon.remove();
+        const host = document.createElement("div");
+        host.setAttribute("aria-hidden", "true");
+        host.setAttribute("role", "presentation");
+        host.style.cssText =
+            "all:initial!important;position:fixed!important;left:0!important;top:0!important;width:0!important;height:0!important;overflow:visible!important;pointer-events:none!important;z-index:2147483647!important";
+        const root = host.attachShadow({ mode: "closed" });
+        const marker = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+        marker.setAttribute("viewBox", "0 0 28 34");
+        marker.setAttribute("aria-hidden", "true");
+        marker.style.cssText =
+            "position:fixed;width:28px;height:34px;pointer-events:none;display:none;filter:drop-shadow(0 0 3px #2d9cff)";
+        const arrow = document.createElementNS("http://www.w3.org/2000/svg", "path");
+        arrow.setAttribute("d", "M2 2 L2 25 L8 20 L13 31 L18 28 L13 18 L23 18 Z");
+        arrow.setAttribute("fill", "#2d9cff");
+        arrow.setAttribute("stroke", "white");
+        arrow.setAttribute("stroke-width", "1.5");
+        marker.append(arrow);
+        root.append(marker);
+        document.documentElement.append(host);
+        const icon = document.createElement("link");
+        icon.rel = "icon";
+        icon.type = "image/svg+xml";
+        icon.href =
+            "data:image/svg+xml," +
+                encodeURIComponent('<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="6" fill="#247fd9"/><text x="16" y="23" font-family="sans-serif" font-size="23" text-anchor="middle" fill="white">π</text></svg>');
+        (document.head ?? document.documentElement).append(icon);
+        cue = { host, marker, icon };
+        world.__piComputerUseCue = cue;
+    }
+    if (!cue)
+        return { active: false, tabBadge: false, clickMarker: false };
+    if (input.mode === "hide")
+        cue.host.style.setProperty("display", "none", "important");
+    else if (input.mode === "show")
+        cue.host.style.removeProperty("display");
+    else {
+        cue.host.style.removeProperty("display");
+        if (input.point &&
+            Number.isFinite(input.point.x) &&
+            Number.isFinite(input.point.y)) {
+            cue.marker.style.left = `${input.point.x}px`;
+            cue.marker.style.top = `${input.point.y}px`;
+            cue.marker.style.display = "block";
+        }
+    }
+    return {
+        active: cue.host.isConnected,
+        tabBadge: cue.icon.isConnected,
+        clickMarker: cue.marker.style.display !== "none",
+        hidden: cue.host.style.display === "none",
+        ariaHidden: cue.host.getAttribute("aria-hidden") === "true",
+        pointerEvents: cue.host.style.pointerEvents,
+        shadowClosed: cue.host.shadowRoot === null,
+    };
+}

package/dist/browser-profiles.d.ts

@@ -0,0 +1,10 @@
+export interface BrowserProfile {
+    path: string;
+    kept: boolean;
+    lock: string;
+    token: string;
+}
+export declare function acquireBrowserProfile(app: string, executable: string, kept: boolean): Promise<BrowserProfile>;
+export declare function recordProfileBrowser(profile: BrowserProfile, pid: number | undefined): Promise<void>;
+export declare function releaseBrowserProfile(profile: BrowserProfile, browserPid?: number): Promise<void>;
+export declare function browserIdleMs(): number;