@oharness/engine 0.9.3 → 0.11.0

package/dist/computer/tools.js

@@ -0,0 +1,664 @@
+import { z } from "zod";
+import { defineTool } from "../tools/define.js";
+import { toolError, toolImage, toolText } from "../tools/types.js";
+import { errorMessage } from "../core/errors.js";
+import { ComputerError } from "./target.js";
+import { COMPUTER_ENABLE_TOOL, computerOn, setComputerOn } from "./mode.js";
+/**
+ * The computer tools.
+ *
+ * Built against one conversation's session, the way the browser tools are: a
+ * ref means nothing without the tree it came from, and the permission prompt
+ * has to name the app a call reaches, which is session state rather than an
+ * argument.
+ *
+ * Shaped like the browser tools the model already knows: read a tree of refs,
+ * act on a ref, fall back to pixels from a screenshot, chain with a batch.
+ */
+/** Screenshots are sent at most this wide; the frame maps coordinates back. */
+const SHOT_MAX_WIDTH = 1280;
+/**
+ * A zoom is sent at most this wide: past it a model downscales the image
+ * anyway, and the point of a zoom is that nothing gets downscaled.
+ */
+const ZOOM_MAX_WIDTH = 1568;
+/** Margin around a zoomed element, in screen units, so its edges are in the picture. */
+const ZOOM_REF_MARGIN = 12;
+/** Nodes in one tree read, so a huge window cannot flood the context. */
+const TREE_MAX_NODES = 400;
+const target = {
+    app: z.string().optional().describe("App name or id (bundle id on macOS, exe name on Windows). Uses its front window."),
+    window: z.string().optional().describe("Window id from computer_list_apps. Omit to use the app's front window, or the last window used."),
+};
+const modifiersParam = z
+    .array(z.enum(["cmd", "ctrl", "alt", "option", "shift", "mod", "win"]))
+    .optional()
+    .describe("Held while acting. `mod` is cmd on macOS and ctrl on Windows.");
+export function createComputerTools(session) {
+    /** Every call is approved per app: `Computer(com.apple.Notes)`, `Computer(excel.exe)`. */
+    const principals = (input) => {
+        const id = session.appIdFor(input);
+        return id ? [`Computer(${id})`] : [];
+    };
+    const listApps = defineTool({
+        name: "computer_list_apps",
+        description: "List the apps running on the user's computer and their open windows, with window ids and what each app allows (full, click-only, look-only). Start here.",
+        parameters: z.object({}),
+        // Names and titles only; reading a window's contents is what asks.
+        readOnly: true,
+        renderTitle: () => "computer_list_apps",
+        async execute(_input, ctx) {
+            return run(session, ctx, async () => {
+                const [apps, windows, displays] = await Promise.all([session.listApps(), session.listWindows(), session.listDisplays()]);
+                const lines = [];
+                // One monitor needs no saying; with more, each window says where it is.
+                const several = displays.length > 1;
+                if (several) {
+                    lines.push("Displays:");
+                    for (const d of displays)
+                        lines.push(`  display ${d.id}: ${d.name} — ${Math.round(d.frame.w)}x${Math.round(d.frame.h)} at (${Math.round(d.frame.x)}, ${Math.round(d.frame.y)})${d.primary ? " [primary]" : ""}`);
+                    lines.push("");
+                }
+                for (const app of apps) {
+                    const own = windows.filter((w) => w.pid === app.pid);
+                    if (!own.length && !app.active)
+                        continue;
+                    const tier = session.tierOf({ appId: app.id, app: app.name });
+                    lines.push(`${app.name} (${app.id})${app.active ? " — frontmost" : ""}${tier === "full" ? "" : ` — ${tier === "read" ? "look only" : tier === "click" ? "click only" : "blocked"}`}`);
+                    for (const w of own) {
+                        const on = several && !w.minimized && w.onScreen !== false ? displayOf(w.frame, displays) : undefined;
+                        lines.push(`  window ${w.id}: ${w.title || "(untitled)"}${w.minimized ? " [minimized]" : w.onScreen === false ? " [not on screen]" : ""}${w.focused ? " [focused]" : ""}${on ? ` [display ${on.id}]` : ""}`);
+                    }
+                }
+                return toolText(lines.join("\n") || "No apps with windows.");
+            });
+        },
+    });
+    const readApp = defineTool({
+        name: "computer_read_app",
+        description: "Read a window's accessibility tree: its buttons, fields, menus and text, each with a `ref` to act on. Prefer this to a screenshot; it is exact and costs far fewer tokens.",
+        parameters: z.object({
+            ...target,
+            depth: z.number().int().min(1).max(40).optional().describe("How deep to walk. Default 25."),
+        }),
+        // Looking changes nothing, so plan mode allows it; the first look at an
+        // app still asks, because a screenshot shows whatever is in it.
+        observes: true,
+        principals,
+        renderTitle: (input) => `computer_read_app(${input.app ?? input.window ?? ""})`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                const t = await session.runtime.target();
+                const window = await session.resolveWindow(input);
+                session.guard(window, "look");
+                const tree = await t.axTree({ window: window.id, depth: input.depth ?? 25, maxNodes: TREE_MAX_NODES });
+                session.rememberTree(tree.nodes);
+                await session.announce({ kind: "look", window: window.id, app: window.app }, t);
+                const header = `${window.app} — window ${window.id}: ${window.title || "(untitled)"}`;
+                const body = tree.nodes.map(formatNode).join("\n");
+                const tail = tree.truncated ? `\n… truncated at ${TREE_MAX_NODES} nodes; read a smaller depth or take a screenshot.` : "";
+                return toolText(`${header}\n${body || "(the accessibility tree is empty — take a screenshot instead)"}${tail}`);
+            });
+        },
+    });
+    const screenshot = defineTool({
+        name: "computer_screenshot",
+        description: "Take a screenshot of one window (default) or a whole display. Returns the image; x/y in other tools are pixels in the latest screenshot of that window.",
+        parameters: z.object({
+            ...target,
+            display: z.string().optional().describe('A display id or name from computer_list_apps ("primary" works too), to capture a whole screen instead of a window.'),
+        }),
+        // Looking changes nothing, so plan mode allows it; the first look at an
+        // app still asks, because a screenshot shows whatever is in it.
+        observes: true,
+        principals,
+        renderTitle: (input) => `computer_screenshot(${input.app ?? input.window ?? input.display ?? ""})`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                const t = await session.runtime.target();
+                if (input.display) {
+                    const display = await session.resolveDisplay(input.display);
+                    const id = display?.id ?? input.display;
+                    const shot = await t.screenshot({ display: id, maxWidth: SHOT_MAX_WIDTH });
+                    session.rememberShot(`display:${id}`, shot);
+                    return toolImage(shot.mimeType ?? "image/jpeg", shot.data, `Display ${id}${display ? ` (${display.name})` : ""}, ${shot.width}x${shot.height} px.`);
+                }
+                const window = await session.resolveWindow(input);
+                session.guard(window, "look");
+                const shot = await t.screenshot({ window: window.id, maxWidth: SHOT_MAX_WIDTH });
+                session.rememberShot(window.id, shot);
+                await session.announce({ kind: "look", window: window.id, app: window.app }, t);
+                session.sawWindow(window.id, { mimeType: shot.mimeType ?? "image/jpeg", data: shot.data });
+                return toolImage(shot.mimeType ?? "image/jpeg", shot.data, `${window.app} — window ${window.id}, ${shot.width}x${shot.height} px.`);
+            });
+        },
+    });
+    const clickParams = z.object({
+        ...target,
+        ref: z.string().optional().describe("Element ref from computer_read_app. Preferred: works without moving the cursor."),
+        x: z.number().optional().describe("Pixel x in the latest screenshot of the window."),
+        y: z.number().optional(),
+        button: z.enum(["left", "right", "middle"]).optional(),
+        double: z.boolean().optional(),
+        modifiers: modifiersParam,
+    });
+    const zoom = defineTool({
+        name: "computer_zoom",
+        description: "Look closely at part of a window (or display) you already have a screenshot of: returns just that region at the screen's full pixel density, so small text and fine detail are legible. Pass `region` as [x0, y0, x1, y1] in pixels of the latest computer_screenshot, or `ref` to zoom on an element from computer_read_app. A reading aid only: x/y for the other tools still refer to the screenshot, never to the zoomed image.",
+        parameters: z.object({
+            ...target,
+            display: z.string().optional().describe("Zoom into the last screenshot of this display instead of a window."),
+            region: z.array(z.number()).length(4).optional().describe("[x0, y0, x1, y1]: left, top, right, bottom, in pixels of the latest screenshot."),
+            ref: z.string().optional().describe("Element ref from computer_read_app; zooms on that element with a small margin."),
+        }),
+        observes: true,
+        principals,
+        renderTitle: (input) => `computer_zoom(${input.ref ?? input.region?.join(",") ?? ""})`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                if (!input.region === !input.ref)
+                    throw new ComputerError("refused", "Pass either region (pixels of the latest screenshot) or ref, not both.");
+                const t = await session.runtime.target();
+                if (input.display) {
+                    if (!input.region)
+                        throw new ComputerError("refused", "On a display, zoom by region: refs belong to windows.");
+                    const display = await session.resolveDisplay(input.display);
+                    const id = display?.id ?? input.display;
+                    const key = `display:${id}`;
+                    const rect = session.regionToScreen(key, `display ${id}`, input.region);
+                    const shot = await t.screenshot({ display: id, region: rect, maxWidth: ZOOM_MAX_WIDTH });
+                    return toolImage(shot.mimeType ?? "image/jpeg", shot.data, zoomCaption(`Display ${id}`, session.screenToRegion(key, shot.frame), shot));
+                }
+                const window = await session.resolveWindow(input);
+                session.guard(window, "look");
+                let rect;
+                if (input.ref) {
+                    const frame = session.frameOf(input.ref);
+                    if (!frame)
+                        throw new ComputerError("stale_ref", `Unknown ref ${input.ref}, or it has no place on screen. Read the window again with computer_read_app.`);
+                    rect = within({ x: frame.x - ZOOM_REF_MARGIN, y: frame.y - ZOOM_REF_MARGIN, w: frame.w + 2 * ZOOM_REF_MARGIN, h: frame.h + 2 * ZOOM_REF_MARGIN }, window.frame);
+                }
+                else {
+                    rect = session.regionToScreen(window.id, window.app, input.region);
+                }
+                const shot = await t.screenshot({ window: window.id, region: rect, maxWidth: ZOOM_MAX_WIDTH });
+                // Not remembered as the window's screenshot: coordinates keep meaning
+                // pixels of the full one.
+                await session.announce({ kind: "look", window: window.id, app: window.app }, t);
+                return toolImage(shot.mimeType ?? "image/jpeg", shot.data, zoomCaption(window.app, session.screenToRegion(window.id, shot.frame), shot));
+            });
+        },
+    });
+    const click = defineTool({
+        name: "computer_click",
+        description: "Click an element by ref, or a point by x/y from a screenshot. Right-click opens a context menu.",
+        parameters: clickParams,
+        principals,
+        renderTitle: (input) => `computer_click(${input.ref ?? `${input.x},${input.y}`})`,
+        async execute(input, ctx) {
+            return run(session, ctx, () => doClick(session, input, ctx));
+        },
+    });
+    const typeText = defineTool({
+        name: "computer_type",
+        description: "Type text. With a ref, into that field (set `replace` to overwrite its value, which works in the background); without, into whatever has focus in the window.",
+        parameters: z.object({
+            ...target,
+            text: z.string(),
+            ref: z.string().optional(),
+            replace: z.boolean().optional().describe("Replace the field's value instead of typing at the caret."),
+        }),
+        principals,
+        renderTitle: (input) => `computer_type(${input.ref ?? input.app ?? ""})`,
+        async execute(input, ctx) {
+            return run(session, ctx, () => doType(session, input, ctx));
+        },
+    });
+    const key = defineTool({
+        name: "computer_key",
+        description: 'Press a key or chord, e.g. "Enter", "Escape", "Tab", "mod+s", "cmd+shift+t", "ctrl+c".',
+        parameters: z.object({ ...target, key: z.string() }),
+        principals,
+        renderTitle: (input) => `computer_key(${input.key})`,
+        async execute(input, ctx) {
+            return run(session, ctx, () => doKey(session, input, ctx));
+        },
+    });
+    const scroll = defineTool({
+        name: "computer_scroll",
+        description: "Scroll inside a window, at a ref or a screenshot point. dy > 0 scrolls down.",
+        parameters: z.object({
+            ...target,
+            ref: z.string().optional(),
+            x: z.number().optional(),
+            y: z.number().optional(),
+            dx: z.number().optional(),
+            dy: z.number().optional(),
+        }),
+        principals,
+        renderTitle: (input) => `computer_scroll(${input.dy ?? 0})`,
+        async execute(input, ctx) {
+            return run(session, ctx, () => doScroll(session, input, ctx));
+        },
+    });
+    const drag = defineTool({
+        name: "computer_drag",
+        description: "Drag from one screenshot point to another. Usually needs computer_take_control first.",
+        parameters: z.object({
+            ...target,
+            from: z.object({ x: z.number(), y: z.number() }),
+            to: z.object({ x: z.number(), y: z.number() }),
+            modifiers: modifiersParam,
+        }),
+        principals,
+        renderTitle: () => "computer_drag",
+        async execute(input, ctx) {
+            return run(session, ctx, () => doDrag(session, input, ctx));
+        },
+    });
+    const menu = defineTool({
+        name: "computer_menu",
+        description: 'Choose an item from the app\'s menu bar by its path, e.g. ["File", "New Note"] or ["Format", "Font", "Bold"]. Runs in the background through accessibility — prefer it to the item\'s keyboard shortcut, which needs the real keyboard when the app has more than one window.',
+        parameters: z.object({
+            ...target,
+            path: z.array(z.string().min(1)).min(1).max(16).describe('Menu titles from the menu bar down, as the app shows them, e.g. ["File", "New Note"].'),
+        }),
+        principals,
+        renderTitle: (input) => `computer_menu(${input.path.join(" › ")})`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                const t = await session.runtime.target();
+                if (!t.invokeMenu) {
+                    return toolError("Menus cannot be chosen directly on this machine; use the item's keyboard shortcut with computer_key.");
+                }
+                const window = await session.resolveWindow(input);
+                session.guard(window, "click");
+                await session.respectUser(window, t);
+                await session.announce({ kind: "click", window: window.id, app: window.app }, t);
+                await t.invokeMenu(window.id, input.path);
+                return toolText(`Chose ${input.path.join(" › ")} in ${window.app}.`);
+            });
+        },
+    });
+    const openApp = defineTool({
+        name: "computer_open_app",
+        description: "Launch an app, or bring it up if it is running, by name or id.",
+        parameters: z.object({ app: z.string() }),
+        principals,
+        renderTitle: (input) => `computer_open_app(${input.app})`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                const t = await session.runtime.target();
+                await t.open(input.app);
+                // Give it a moment to put a window up, then report what came up.
+                for (let i = 0; i < 20; i++) {
+                    try {
+                        const window = await session.resolveWindow({ app: input.app });
+                        session.guard(window, "look");
+                        await session.announce({ kind: "look", window: window.id, app: window.app }, t);
+                        return toolText(`${window.app} is open — window ${window.id}: ${window.title || "(untitled)"}.`);
+                    }
+                    catch (err) {
+                        if (err instanceof ComputerError && err.code === "refused")
+                            throw err;
+                        await sleep(250, ctx.signal);
+                    }
+                }
+                return toolText(`Asked ${input.app} to open; no window has appeared yet. Call computer_list_apps shortly.`);
+            });
+        },
+    });
+    const takeControl = defineTool({
+        name: "computer_take_control",
+        description: "Take the real mouse and keyboard, for actions that cannot run in the background. The user is shown that you are in control and can take it back at any time. Keep it short: do the whole sequence in one computer_batch, then call computer_release_control.",
+        parameters: z.object({ reason: z.string().describe("One line the user sees, e.g. 'Drag the file into the upload area'.") }),
+        principals: () => ["ComputerTakeover"],
+        renderTitle: (input) => `computer_take_control(${input.reason})`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                if (session.runtime.takeover() === "never") {
+                    return toolError("Taking over the mouse and keyboard is turned off in Settings › Computer use. Stay with background actions, or ask the user to do this step.");
+                }
+                const t = await session.runtime.target();
+                await session.runtime.arbiter.takeControl(session.holder(), ctx.signal, (holder) => ctx.onProgress({ toolCallId: ctx.toolCallId, toolName: "computer_take_control", message: `Waiting for "${holder.label}" to release the computer…` }));
+                const front = await t.frontmost().catch(() => undefined);
+                if (front)
+                    ctx.host.state.set(PREVIOUS_FRONT, front.pid);
+                await session.announce({ kind: "control", app: input.reason });
+                return toolText("You have control of the mouse and keyboard. Act now, then call computer_release_control. If the user moves the mouse or presses a key, control returns to them and your action stops.");
+            });
+        },
+    });
+    const releaseControl = defineTool({
+        name: "computer_release_control",
+        description: "Give the mouse and keyboard back to the user and return focus to where they were.",
+        parameters: z.object({}),
+        sessionLocal: true,
+        renderTitle: () => "computer_release_control",
+        async execute(_input, ctx) {
+            return run(session, ctx, async () => {
+                const released = session.runtime.arbiter.releaseControl(session.id);
+                const previous = ctx.host.state.get(PREVIOUS_FRONT);
+                ctx.host.state.delete(PREVIOUS_FRONT);
+                if (released && typeof previous === "number") {
+                    await (await session.runtime.target()).activate(previous).catch(() => { });
+                }
+                await session.announce({ kind: "release" });
+                return toolText(released ? "Control returned to the user." : "You did not have control.");
+            });
+        },
+    });
+    const single = {
+        click: (input, ctx) => doClick(session, input, ctx).then(textOf),
+        type: (input, ctx) => doType(session, input, ctx).then(textOf),
+        key: (input, ctx) => doKey(session, input, ctx).then(textOf),
+        scroll: (input, ctx) => doScroll(session, input, ctx).then(textOf),
+        drag: (input, ctx) => doDrag(session, input, ctx).then(textOf),
+        wait: async (input, ctx) => {
+            await sleep(Math.min(5_000, input.ms ?? 500), ctx.signal);
+            return "waited";
+        },
+    };
+    /**
+     * Each action in a batch, with the fields its single tool takes — spelled
+     * out, so the model is shown that a key action needs its `key`. It used to
+     * be {action, ...anything}: `{"action":"key"}` went through, and failed
+     * deep inside as "Cannot read properties of undefined (reading 'split')",
+     * an error nobody could act on.
+     */
+    const point = { x: z.number(), y: z.number() };
+    const batchAction = z.discriminatedUnion("action", [
+        z.object({
+            action: z.literal("click"),
+            ref: z.string().optional().describe("Element ref from computer_read_app."),
+            x: z.number().optional().describe("Pixel x in the latest screenshot of the window."),
+            y: z.number().optional(),
+            button: z.enum(["left", "right", "middle"]).optional(),
+            double: z.boolean().optional(),
+            modifiers: modifiersParam,
+        }),
+        z.object({
+            action: z.literal("type"),
+            text: z.string(),
+            ref: z.string().optional(),
+            replace: z.boolean().optional(),
+        }),
+        z.object({
+            action: z.literal("key"),
+            key: z.string().min(1).describe('The key or chord, e.g. "Enter", "Tab", "mod+n", "cmd+shift+t".'),
+        }),
+        z.object({
+            action: z.literal("scroll"),
+            ref: z.string().optional(),
+            x: z.number().optional(),
+            y: z.number().optional(),
+            dx: z.number().optional(),
+            dy: z.number().optional(),
+        }),
+        z.object({ action: z.literal("drag"), from: z.object(point), to: z.object(point), modifiers: modifiersParam }),
+        z.object({ action: z.literal("wait"), ms: z.number().int().min(0).max(5_000).optional() }),
+    ]);
+    const batch = defineTool({
+        name: "computer_batch",
+        description: 'Run several actions in one window, in order, stopping at the first failure. Each action is one of: {action:"click", ref | x,y, button?, double?, modifiers?}, {action:"type", text, ref?, replace?}, {action:"key", key}, {action:"scroll", ref | x,y, dx?, dy?}, {action:"drag", from:{x,y}, to:{x,y}}, {action:"wait", ms?}. All actions use the batch\'s app/window.',
+        parameters: z.object({
+            ...target,
+            actions: z.array(batchAction).min(1).max(30),
+        }),
+        // One app per batch: a batch naming one approved app must not smuggle in
+        // an action on another, since any matching principal allows the call.
+        principals: (input) => principals(input),
+        renderTitle: (input) => `computer_batch(${input.actions.length} actions)`,
+        async execute(input, ctx) {
+            return run(session, ctx, async () => {
+                const window = await session.resolveWindow(input);
+                const out = [];
+                for (const [i, raw] of input.actions.entries()) {
+                    const { action, ...rest } = raw;
+                    if (typeof rest["ref"] === "string" && rest["ref"] !== "" && !String(rest["ref"]).startsWith(`w${window.id}.`)) {
+                        out.push(`${i + 1}. ${action}: refused — ref ${String(rest["ref"])} is not in this batch's window`);
+                        break;
+                    }
+                    try {
+                        out.push(`${i + 1}. ${action}: ${await single[action]({ ...rest, window: window.id }, ctx)}`);
+                    }
+                    catch (err) {
+                        out.push(`${i + 1}. ${action}: failed — ${errorMessage(err)}`);
+                        return toolError(out.join("\n"));
+                    }
+                }
+                return toolText(out.join("\n"));
+            });
+        },
+    });
+    return [enableTool(session), listApps, readApp, screenshot, zoom, click, typeText, key, menu, scroll, drag, openApp, takeControl, releaseControl, batch];
+}
+const PREVIOUS_FRONT = "computer:previousFront";
+async function doClick(session, input, _ctx) {
+    const t = await session.runtime.target();
+    const window = await session.resolveWindow(input);
+    session.guard(window, "click");
+    await session.respectUser(window, t);
+    const foreground = session.runtime.arbiter.holdsControl(session.id);
+    if (input.ref && !input.modifiers?.length && !input.double && (input.button ?? "left") !== "middle") {
+        const frame = session.frameOf(input.ref);
+        await session.announce({ kind: "click", window: window.id, app: window.app, ...(frame ? { frame, point: centre(frame) } : {}) }, t);
+        try {
+            await t.axAction(input.ref, input.button === "right" ? "showMenu" : "press");
+            return toolText(`Clicked ${input.ref} in ${window.app}.`);
+        }
+        catch (err) {
+            // An element with no press action still has a place on screen.
+            if (!(err instanceof ComputerError) || err.code !== "unsupported" || !frame)
+                throw err;
+            const point = centre(frame);
+            await t.input({ type: "click", ...point, button: input.button ?? "left" }, { window: window.id, foreground });
+            return toolText(`Clicked ${input.ref} in ${window.app} (by position).`);
+        }
+    }
+    const point = input.ref ? refPoint(session, input.ref) : pointFrom(session, window, input);
+    await session.announce({ kind: "click", window: window.id, app: window.app, point }, t);
+    await t.input({ type: "click", ...point, button: input.button ?? "left", count: input.double ? 2 : 1, ...(input.modifiers ? { modifiers: input.modifiers } : {}) }, { window: window.id, foreground });
+    return toolText(`Clicked in ${window.app}.`);
+}
+async function doType(session, input, ctx) {
+    const t = await session.runtime.target();
+    const window = await session.resolveWindow(input);
+    session.guard(window, "type");
+    if (input.ref && session.isSecure(input.ref)) {
+        throw new ComputerError("refused", "That is a password field. Ask the user to type it themselves.");
+    }
+    await session.respectUser(window, t);
+    const frame = input.ref ? session.frameOf(input.ref) : undefined;
+    await session.announce({ kind: "type", window: window.id, app: window.app, ...(frame ? { frame, point: centre(frame) } : {}) }, t);
+    if (input.ref && input.replace) {
+        await t.axAction(input.ref, "setValue", input.text);
+        return toolText(`Set the value of ${input.ref}.`);
+    }
+    if (input.ref)
+        await t.axAction(input.ref, "focus").catch(() => { });
+    await session.waitForQuiet(t, ctx.signal);
+    await t.input({ type: "text", text: input.text }, { window: window.id, foreground: session.runtime.arbiter.holdsControl(session.id) });
+    return toolText(`Typed ${input.text.length} characters into ${window.app}.`);
+}
+async function doKey(session, input, ctx) {
+    const t = await session.runtime.target();
+    const window = await session.resolveWindow(input);
+    session.guard(window, "type");
+    await session.respectUser(window, t);
+    if (typeof input.key !== "string" || !input.key.trim()) {
+        throw new ComputerError("refused", 'Say which key: {key: "Enter"}, {key: "mod+n"}.');
+    }
+    const { key, modifiers } = parseChord(input.key, (await session.runtime.info()).platform);
+    await session.announce({ kind: "key", window: window.id, app: window.app }, t);
+    await session.waitForQuiet(t, ctx.signal);
+    await t.input({ type: "key", key, modifiers }, { window: window.id, foreground: session.runtime.arbiter.holdsControl(session.id) });
+    return toolText(`Pressed ${input.key} in ${window.app}.`);
+}
+async function doScroll(session, input, _ctx) {
+    const t = await session.runtime.target();
+    const window = await session.resolveWindow(input);
+    session.guard(window, "click");
+    await session.respectUser(window, t);
+    const point = input.ref
+        ? refPoint(session, input.ref)
+        : input.x !== undefined && input.y !== undefined
+            ? session.toScreen(window, input.x, input.y)
+            : centre(window.frame);
+    await session.announce({ kind: "scroll", window: window.id, app: window.app, point }, t);
+    await t.input({ type: "scroll", ...point, dx: input.dx ?? 0, dy: input.dy ?? 0 }, { window: window.id, foreground: session.runtime.arbiter.holdsControl(session.id) });
+    return toolText(`Scrolled in ${window.app}.`);
+}
+async function doDrag(session, input, _ctx) {
+    const t = await session.runtime.target();
+    const window = await session.resolveWindow(input);
+    session.guard(window, "click");
+    await session.respectUser(window, t);
+    const from = session.toScreen(window, input.from.x, input.from.y);
+    const to = session.toScreen(window, input.to.x, input.to.y);
+    await session.announce({ kind: "drag", window: window.id, app: window.app, point: to }, t);
+    await t.input({ type: "drag", from, to, ...(input.modifiers ? { modifiers: input.modifiers } : {}) }, { window: window.id, foreground: session.runtime.arbiter.holdsControl(session.id) });
+    return toolText(`Dragged in ${window.app}.`);
+}
+// ---- helpers ---------------------------------------------------------------
+function pointFrom(session, window, input) {
+    if (input.x === undefined || input.y === undefined) {
+        throw new ComputerError("refused", "Pass a ref from computer_read_app, or x and y from a screenshot.");
+    }
+    return session.toScreen(window, input.x, input.y);
+}
+function refPoint(session, ref) {
+    const frame = session.frameOf(ref);
+    if (!frame)
+        throw new ComputerError("stale_ref", `Unknown ref ${ref}. Read the window again with computer_read_app.`);
+    return centre(frame);
+}
+/** The display a window is mostly on: the one holding its centre, else the first it touches. */
+function displayOf(frame, displays) {
+    const c = centre(frame);
+    const holds = (d, x, y) => x >= d.frame.x && x < d.frame.x + d.frame.w && y >= d.frame.y && y < d.frame.y + d.frame.h;
+    return displays.find((d) => holds(d, c.x, c.y)) ?? displays.find((d) => intersects(d.frame, frame));
+}
+function intersects(a, b) {
+    return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h;
+}
+/** `rect` clipped to `bounds`. */
+function within(rect, bounds) {
+    const x = Math.max(rect.x, bounds.x);
+    const y = Math.max(rect.y, bounds.y);
+    const right = Math.min(rect.x + rect.w, bounds.x + bounds.w);
+    const bottom = Math.min(rect.y + rect.h, bounds.y + bounds.h);
+    if (right <= x || bottom <= y)
+        throw new ComputerError("refused", "That element is outside its window; scroll it into view first.");
+    return { x, y, w: right - x, h: bottom - y };
+}
+function zoomCaption(what, region, shot) {
+    const where = region ? `region [${region.join(", ")}] of the last screenshot` : "the region";
+    return `${what} — ${where}, at ${shot.width}x${shot.height} px. A reading aid: x/y for other tools still refer to the screenshot, not this image.`;
+}
+function centre(frame) {
+    return { x: frame.x + frame.w / 2, y: frame.y + frame.h / 2 };
+}
+/** `mod+shift+s` → key "s" with the platform's command key and shift. */
+export function parseChord(chord, platform) {
+    const parts = chord.split("+").map((p) => p.trim()).filter(Boolean);
+    const key = parts.pop() ?? chord;
+    const modifiers = parts.map((m) => {
+        const lower = m.toLowerCase();
+        if (lower === "mod")
+            return platform === "darwin" ? "cmd" : "ctrl";
+        if (lower === "option")
+            return "alt";
+        if (lower === "control")
+            return "ctrl";
+        if (lower === "command" || lower === "meta")
+            return platform === "darwin" ? "cmd" : "win";
+        return lower;
+    });
+    return { key, modifiers };
+}
+function formatNode(node) {
+    const indent = "  ".repeat(Math.max(0, node.depth));
+    const name = node.name ? ` "${node.name.slice(0, 120)}"` : "";
+    const value = node.secure ? " = •••" : node.value ? ` = "${node.value.slice(0, 200)}"` : "";
+    const flags = node.disabled ? " (disabled)" : "";
+    return `${indent}[${node.ref}] ${node.role}${name}${value}${flags}`;
+}
+function textOf(result) {
+    return result.content.map((b) => (b.type === "text" ? b.text : "")).join(" ");
+}
+function sleep(ms, signal) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener("abort", () => {
+            clearTimeout(timer);
+            resolve();
+        }, { once: true });
+    });
+}
+/**
+ * Turn failures into answers the model can act on.
+ *
+ * A refusal from the policy, the arbiter or the helper becomes a plain
+ * sentence; anything else is a crash, and the helper is restarted on the
+ * next call rather than left half-alive.
+ */
+async function run(session, ctx, fn) {
+    try {
+        return await fn();
+    }
+    catch (err) {
+        if (err instanceof ComputerError) {
+            if (err.code === "permission") {
+                const what = err.grant === "screen" ? "Screen Recording" : "Accessibility";
+                return toolError(`${what} permission is not granted to OHarness. Ask the user to allow it; the app shows them how. (${err.message})`);
+            }
+            if (err.code === "needs_foreground") {
+                return toolError(`${err.message} This action needs the real mouse and keyboard: call computer_take_control, act, then computer_release_control.`);
+            }
+            return toolError(err.message);
+        }
+        ctx.logger.warn("computer tool failed", { error: errorMessage(err) });
+        session.runtime.reset();
+        return toolError(`The computer helper failed: ${errorMessage(err)}`);
+    }
+}
+/**
+ * `computer_enable`: the agent switches Computer use on for its conversation.
+ *
+ * The setting is read on every question rather than when the tool is built,
+ * because a conversation's tools outlive a change in Settings: `off` hides the
+ * tool, and `auto` makes it session-local, which the permission engine lets
+ * through unasked — what it changes is a flag in this conversation, and each
+ * app the agent then reaches still asks on its own. `ask` leaves it an
+ * ordinary call, so the prompt goes wherever prompts go, a paired phone too.
+ */
+function enableTool(session) {
+    const tool = defineTool({
+        name: COMPUTER_ENABLE_TOOL,
+        description: "Switch on Computer use for this conversation, when the request needs you to see or operate apps on the user's computer (native apps such as Notes, Finder, Excel or Settings). " +
+            "Not for web pages (use the browser tools), files or commands (use the file tools and bash). The user may be asked to approve; once on, the computer_* tools appear.",
+        parameters: z.object({
+            reason: z.string().describe("What you need the computer for, in a few words the user will read, e.g. \"add the meeting to Notes\"."),
+        }),
+        // Switching it on looks at nothing; plan mode may still turn it on, to look.
+        observes: true,
+        renderTitle: (input) => `computer_enable(${input.reason})`,
+        async execute(_input, ctx) {
+            const state = ctx.host.state;
+            if (computerOn(state))
+                return toolText("Computer use is already on for this conversation.");
+            if (session.runtime.enableMode() === "off") {
+                return toolError("Computer use can only be switched on by the user here, from the + menu. Ask them to.");
+            }
+            setComputerOn(state, true);
+            session.bind(ctx.sessionId);
+            session.runtime.emitEnabled(ctx.sessionId);
+            return toolText("Computer use is on for this conversation. The computer_* tools are now available: start with computer_list_apps. Each app still asks the user the first time.");
+        },
+    });
+    Object.defineProperty(tool, "hidden", { get: () => session.runtime.enableMode() === "off", enumerable: true });
+    Object.defineProperty(tool, "sessionLocal", { get: () => session.runtime.enableMode() === "auto", enumerable: true });
+    return tool;
+}
+//# sourceMappingURL=tools.js.map