@oharness/engine 0.9.3 → 0.11.0

package/dist/computer/session.d.ts

@@ -0,0 +1,228 @@
+import type { Logger } from "../util/logger.ts";
+import { ComputerArbiter } from "./arbiter.ts";
+import type { LeaseHolder } from "./arbiter.ts";
+import type { ActionKind, ComputerTier, PolicyOptions } from "./policy.ts";
+import type { ComputerEnableMode } from "./mode.ts";
+import type { AppInfo, AxNode, ComputerPermissions, ComputerTarget, ComputerTargetFactory, DisplayInfo, HelperInfo, Rect, Shot, WindowInfo } from "./target.ts";
+/**
+ * What the page and the overlay are told about each action, so the person can
+ * see where the agent is working without reading the transcript. Presentation
+ * only: nothing here goes to the model.
+ */
+export interface ComputerActionEvent {
+    sessionId: string;
+    label: string;
+    kind: "look" | "click" | "type" | "scroll" | "drag" | "key" | "control" | "release" | "idle";
+    window?: string;
+    app?: string;
+    /** Screen point the action lands on, when it has one. */
+    point?: {
+        x: number;
+        y: number;
+    };
+    /** Whether that point is on screen and uncovered: the overlay draws the cursor only then. */
+    visible?: boolean;
+    /** Frame of the element acted on, for a ref action. */
+    frame?: Rect;
+    /** Where the window is on screen, so a live view can place the point inside it. */
+    windowFrame?: Rect;
+    /** The window's title, for the live view's heading. */
+    title?: string;
+}
+/**
+ * What the windows a turn worked in looked like when it ended, for the person
+ * reading the conversation back. Never model input: the agent works from the
+ * accessibility tree and takes its own screenshots when it needs one.
+ */
+export interface ComputerSnapshotEvent {
+    sessionId: string;
+    shots: {
+        window: string;
+        app: string;
+        title?: string;
+        mimeType: string;
+        data: string;
+    }[];
+}
+export interface ComputerRuntimeOptions {
+    target: ComputerTargetFactory;
+    /**
+     * Capture one window for the person, outside the platform layer. The host
+     * does this, not the target: a target screenshot replaces the snapshot the
+     * agent's element refs belong to, and a picture for the transcript must not
+     * cost the agent its refs. Without it, turns end with no snapshot.
+     */
+    snapshot?: (window: string) => Promise<{
+        mimeType: string;
+        data: string;
+    } | undefined>;
+    /** Pids of the host app itself; its windows are never listed or driven. */
+    selfPids?: () => number[];
+    policy?: () => PolicyOptions;
+    /** `never` keeps the agent in the background: take_control is refused. */
+    takeover?: () => "ask" | "never";
+    /** Whether `computer_enable` asks first, goes straight through, or is gone. Absent is `ask`. */
+    enable?: () => ComputerEnableMode;
+    arbiter?: ComputerArbiter;
+    logger?: Logger;
+    /**
+     * Read the OS grants from the host app itself, live. Preferred over asking
+     * the helper: macOS caches a process's Accessibility answer, so a helper
+     * started before the grant keeps saying no after the person said yes.
+     */
+    permissions?: () => Promise<ComputerPermissions>;
+}
+/**
+ * What every conversation shares: one helper process, one arbiter.
+ *
+ * The helper starts on the first call that needs it — registered eagerly,
+ * launched lazily, the same rule as the browser — and is restarted on the next
+ * call if it dies.
+ */
+export declare class ComputerRuntime {
+    readonly arbiter: ComputerArbiter;
+    private readonly opts;
+    private starting?;
+    private helperInfo?;
+    private readonly observers;
+    private readonly snapshotObservers;
+    private readonly enableObservers;
+    constructor(opts: ComputerRuntimeOptions);
+    target(): Promise<ComputerTarget>;
+    /** Forget a dead helper so the next call starts a fresh one. */
+    reset(): void;
+    info(): Promise<HelperInfo>;
+    private granted?;
+    permissions(): Promise<ComputerPermissions>;
+    requestPermissions(grant?: keyof ComputerPermissions): Promise<void>;
+    selfPids(): Set<number>;
+    policy(): PolicyOptions;
+    takeover(): "ask" | "never";
+    enableMode(): ComputerEnableMode;
+    /**
+     * A conversation's agent switched Computer use on. The + menu tells the
+     * server itself; this is how the server hears of the other way, to save the
+     * conversation and show the chip and the permission card.
+     */
+    onEnabled(listener: (sessionId: string) => void): () => void;
+    emitEnabled(sessionId: string): void;
+    onAction(listener: (event: ComputerActionEvent) => void): () => void;
+    emit(event: ComputerActionEvent): void;
+    canSnapshot(): boolean;
+    snapshot(window: string): Promise<{
+        mimeType: string;
+        data: string;
+    } | undefined>;
+    onSnapshot(listener: (event: ComputerSnapshotEvent) => void): () => void;
+    emitSnapshot(event: ComputerSnapshotEvent): void;
+    close(): Promise<void>;
+}
+/**
+ * One conversation's view of the computer.
+ *
+ * Holds what only makes sense per conversation: the apps and windows it has
+ * seen (so a permission prompt can name an app synchronously), the geometry of
+ * the screenshots it was shown (so its coordinates can be mapped back), and
+ * the window it last worked in.
+ */
+export declare class ComputerUseSession {
+    readonly runtime: ComputerRuntime;
+    private sessionId;
+    private label;
+    private readonly apps;
+    private readonly windowsById;
+    private readonly shots;
+    private readonly secureRefs;
+    /**
+     * The windows this turn worked in, in the order last used, each with the
+     * agent's own screenshot of it while that is still how it looks. Each gets
+     * one picture under the reply when the turn ends: that screenshot when there
+     * is one, else a capture. The screenshot sits folded inside its tool card,
+     * where nobody reading the reply sees it, and reusing it costs no capture —
+     * so a host without Screen Recording still shows what the agent saw.
+     */
+    private readonly unseen;
+    private readonly refFrames;
+    private currentWindow?;
+    constructor(runtime: ComputerRuntime, sessionId?: string);
+    bind(sessionId: string): void;
+    setLabel(label: () => string | undefined): void;
+    holder(): LeaseHolder;
+    get id(): string;
+    listApps(): Promise<AppInfo[]>;
+    listWindows(pid?: number): Promise<WindowInfo[]>;
+    private rememberApp;
+    /**
+     * The app a call names, synchronously, for the permission prompt.
+     *
+     * From what this conversation has already seen; an app it has not listed
+     * yet is named as the model wrote it, which still scopes the approval to
+     * one app.
+     */
+    appIdFor(input: {
+        app?: string;
+        window?: string;
+        ref?: string;
+    }): string | undefined;
+    /** Find the window a call means: by ref, by id, by app, or the last one used. */
+    resolveWindow(input: {
+        app?: string;
+        window?: string;
+        ref?: string;
+    }): Promise<WindowInfo>;
+    tierOf(window: WindowInfo): ComputerTier;
+    /**
+     * Everything that has to hold before touching a window: it is not ours,
+     * its tier allows this kind of action, and no other conversation has it.
+     */
+    guard(window: WindowInfo, kind: ActionKind): void;
+    /**
+     * Stay off a window the person is using right now.
+     *
+     * Their front window with input in the last few seconds is theirs: the
+     * lease on it lapses and the call is refused. Pointer-free actions on other
+     * windows go ahead regardless.
+     */
+    respectUser(window: WindowInfo, target: ComputerTarget): Promise<void>;
+    /** Keys wait until the person has stopped typing, up to a limit. */
+    waitForQuiet(target: ComputerTarget, signal?: AbortSignal, quietMs?: number, maxMs?: number): Promise<void>;
+    rememberShot(key: string, shot: Shot): void;
+    /**
+     * A rectangle `[x0, y0, x1, y1]` in the last screenshot under `key` (a
+     * window id, or `display:<id>`), on screen. Clipped to the screenshot, so a
+     * region drawn a little past its edge still means its edge.
+     */
+    regionToScreen(key: string, what: string, region: readonly number[]): Rect;
+    /** Screen rect → `[x0, y0, x1, y1]` in the last screenshot under `key`, rounded. */
+    screenToRegion(key: string, rect: Rect): number[] | undefined;
+    /** The monitors, primary first; empty when the layer cannot say. */
+    listDisplays(): Promise<DisplayInfo[]>;
+    /** A display by id or by name, as the model wrote it. */
+    resolveDisplay(wanted: string): Promise<DisplayInfo | undefined>;
+    /**
+     * Map a point in the last screenshot of `window` back to the screen.
+     *
+     * The model only ever sees image pixels. Retina, per-monitor DPI and the
+     * downscale to fit the image budget are all folded into the screenshot's own
+     * frame, so none of them reach the model.
+     */
+    toScreen(window: WindowInfo, x: number, y: number): {
+        x: number;
+        y: number;
+    };
+    rememberTree(nodes: AxNode[]): void;
+    isSecure(ref: string): boolean;
+    frameOf(ref: string): Rect | undefined;
+    announce(event: Omit<ComputerActionEvent, "sessionId" | "label">, target?: ComputerTarget): Promise<void>;
+    /** The agent's own screenshot of this window: its state, until the next action on it. */
+    sawWindow(window: string, shot?: {
+        mimeType: string;
+        data: string;
+    }): void;
+    /** The turn ended: let go of windows and control, so others need not wait out the lease. */
+    endTurn(): void;
+    private snapshotTurn;
+    close(): Promise<void>;
+}
+/** Refs look like `w<window>.<n>`; the window id is everything between. */
+export declare function windowOfRef(ref: string): string | undefined;