@oharness/engine 0.9.3 → 0.11.0

package/dist/computer/cua-driver.d.ts

@@ -0,0 +1,174 @@
+import type { Logger } from "../util/logger.ts";
+import type { AppInfo, AxAction, AxNode, ComputerInputEvent, ComputerPermissions, ComputerTarget, DisplayInfo, HelperInfo, Rect, Shot, WindowInfo } from "./target.ts";
+/**
+ * A `ComputerTarget` backed by Cua Driver (github.com/trycua/cua, MIT).
+ *
+ * Cua Driver is the platform layer: ScreenCaptureKit and AX with SkyLight
+ * background input on macOS; UIA, MSAA, PostMessage and WGC on Windows. We
+ * drive its MCP server over stdio and keep everything above it — the tools
+ * the model sees, per-app approval, tiers, the arbiter — our own. The model
+ * never talks to Cua Driver directly: only this adapter does, and it calls a
+ * fixed handful of its 58 tools.
+ *
+ * How it is run matters:
+ *
+ * - Embedded (`CUA_DRIVER_EMBEDDED=1`) and spawned by the app itself, so
+ *   macOS charges its TCC requests to OHarness and the person grants
+ *   OHarness, not a second app. The driver never prompts in this mode; the
+ *   host asks (see `requestPermissions`).
+ * - Telemetry and the update check off. Both default on, embedded or not.
+ * - `mcp --direct`: no daemon and no Cua cursor. The desktop app draws its
+ *   own, per conversation, and keeps it out of captures.
+ *
+ * Coordinates: Cua takes pointer x/y in pixels of the window screenshot it
+ * last returned. The engine works in screen units (each screenshot's frame
+ * says where its pixels sit), so pointer events are mapped back through the
+ * last screenshot of that window here. Element frames already come in screen
+ * units.
+ */
+export interface CuaDriverOptions {
+    /** The cua-driver executable. */
+    path: string;
+    /** The host app's bundle id, for Cua's own identity checks. */
+    hostBundleId?: string;
+    /** Where the driver may keep state; telemetry is off either way. */
+    stateDir?: string;
+    /**
+     * The Rego policy the driver enforces on every call: only the tools this
+     * adapter uses, and none that write files (desktop/computer-helper/cua/policy.rego).
+     */
+    policyFile?: string;
+    logger?: Logger;
+    /**
+     * Ask the OS for Screen Recording or Accessibility. Embedded, the driver
+     * never prompts: the host app has to, as itself.
+     */
+    requestPermissions?: (grant?: keyof ComputerPermissions) => Promise<void>;
+    /** Milliseconds since the person's last real input, from the host. */
+    idle?: () => Promise<number>;
+    /**
+     * Who a process is, asked live. Cua's running-app list is a snapshot that
+     * goes stale (see `apps`); this names the pids it is missing. Defaults to
+     * `lsappinfo` on macOS.
+     */
+    appForPid?: (pid: number) => Promise<{
+        name: string;
+        id: string;
+    } | undefined>;
+    /** Whether a pid is still a process. Injectable for tests. */
+    alive?: (pid: number) => boolean;
+    /**
+     * The monitors, from the host. Cua Driver only knows the main one, so
+     * without this the agent sees a single display.
+     */
+    displays?: () => Promise<DisplayInfo[]>;
+    /**
+     * Capture a display, or a region of one, as the host. Cua's desktop
+     * capture is the main display, whole; any other display, and any region of
+     * one, comes from here.
+     */
+    captureDisplay?: (opts: {
+        display?: string;
+        region?: Rect;
+        maxWidth: number;
+    }) => Promise<Shot>;
+    /**
+     * A region of a window at full pixel density, from the host. Cua's own
+     * zoom stops at 500 px wide, which turns a strip across a window into a
+     * picture too small to read; it stays the fallback.
+     */
+    captureWindow?: (opts: {
+        window: string;
+        region: Rect;
+        maxWidth: number;
+    }) => Promise<Shot>;
+}
+export declare class CuaDriverTarget implements ComputerTarget {
+    private readonly client;
+    private readonly opts;
+    private readonly pids;
+    private readonly shots;
+    private readonly refs;
+    /** The field typing goes to, per window, set by `axAction focus`. */
+    private readonly targets;
+    private appIds;
+    private version;
+    private constructor();
+    static start(opts: CuaDriverOptions): Promise<CuaDriverTarget>;
+    private call;
+    private refusal;
+    private pidOf;
+    /** Screen point → pixels of the last screenshot of this window, taking one if needed. */
+    private toWindowPixels;
+    info(): Promise<HelperInfo>;
+    permissions(): Promise<ComputerPermissions>;
+    requestPermissions(grant?: keyof ComputerPermissions): Promise<void>;
+    /**
+     * The running apps, checked against the live windows.
+     *
+     * Cua builds `list_apps` from NSWorkspace's runningApplications, which a
+     * process only keeps current by running its main run loop — and the driver,
+     * embedded and long-lived, does not. The list stays as it was when the
+     * driver started: an app launched since is missing, and one relaunched keeps
+     * its old, dead pid. Windows, from the window server, are live. So an entry
+     * whose process is gone is dropped, and a pid that owns a window but is not
+     * listed is named live — by bundle id, which is what tiers and approvals go
+     * by, so a browser opened after OHarness is still a browser.
+     */
+    apps(): Promise<AppInfo[]>;
+    private reconcile;
+    windows(opts?: {
+        pid?: number;
+    }): Promise<WindowInfo[]>;
+    displays(): Promise<DisplayInfo[]>;
+    screenshot(opts: {
+        window?: string;
+        display?: string;
+        region?: Rect;
+        maxWidth?: number;
+    }): Promise<Shot>;
+    /**
+     * A region of a window through Cua's own zoom: cropped from a full-density
+     * capture, padded by a fifth on each side, at most 500 px wide. It reads
+     * the region in pixels of the last screenshot this connection took of the
+     * window, and a tree read since may have replaced that one, so a refusal
+     * for want of it takes a screenshot the same size and tries once more.
+     */
+    private zoom;
+    axTree(opts: {
+        window: string;
+        depth?: number;
+        maxNodes?: number;
+    }): Promise<{
+        nodes: AxNode[];
+        truncated: boolean;
+    }>;
+    private refInfo;
+    /**
+     * Run an action on a ref, re-finding the element once if its token went
+     * stale. Any snapshot of a window — a screenshot included — supersedes the
+     * tokens of the one before, and the model reasonably reads the tree, looks
+     * at the window, then clicks.
+     */
+    private onRef;
+    axAction(ref: string, action: AxAction, value?: string): Promise<void>;
+    input(event: ComputerInputEvent, opts: {
+        window?: string;
+        foreground: boolean;
+    }): Promise<void>;
+    invokeMenu(window: string, path: string[]): Promise<void>;
+    open(app: string): Promise<{
+        pid?: number;
+    }>;
+    frontmost(): Promise<{
+        pid: number;
+        appId: string;
+        window?: string;
+    }>;
+    activate(pid: number): Promise<void>;
+    idle(): Promise<number>;
+    pointVisible(window: string, x: number, y: number): Promise<boolean>;
+    clipboardRead(): Promise<string>;
+    clipboardWrite(text: string): Promise<void>;
+    close(): Promise<void>;
+}