@forsion/tangu-computer-use 0.5.8

package/src/setup.ts

@@ -0,0 +1,97 @@
+/**
+ * `tangu computer-use <setup|doctor|stop>`。复用 vendor 的 ensureComputerUseSetup —— 给它一个 readline 版
+ * ExtensionContext(hasUI=true),vendor 的 ensurePermissions 交互式引导就跑在 CLI 上。doctor 用 hasUI=false
+ * 走非交互检查(权限缺 → vendor 抛指导文本)。
+ */
+import { createInterface } from 'node:readline/promises';
+import { ensureComputerUseSetup, shutdownComputerUseSession } from './vendor/bridge.ts';
+import { helperInstalled, helperExecutablePath } from './helperState.ts';
+import type { ExtensionContext } from './pi-compat.ts';
+
+function cliCtx(hasUI: boolean): ExtensionContext {
+  return {
+    cwd: process.cwd(),
+    hasUI,
+    ui: {
+      notify: (msg, level) => console.log(`[${level || 'info'}] ${msg}`),
+      select: async (prompt, options) => {
+        console.log(prompt);
+        options.forEach((o, i) => console.log(`  ${i + 1}. ${o}`));
+        const rl = createInterface({ input: process.stdin, output: process.stdout });
+        try {
+          const a = (await rl.question('> ')).trim();
+          const n = Number.parseInt(a, 10);
+          return Number.isFinite(n) && n >= 1 && n <= options.length ? options[n - 1] : a;
+        } finally {
+          rl.close();
+        }
+      },
+    },
+    sessionManager: { getBranch: () => [] },
+  };
+}
+
+export async function cliMain(argv: string[]): Promise<number> {
+  const sub = argv[0];
+  // setup / doctor 是一次性命令:返回前释放 helper 会话。Windows / Linux 的 helper 是 stdio 子进程,管道开着
+  // 事件循环就空不了,结果打印完进程也退不出(09-25 windows-2022 实测 doctor 挂满 90s)。宿主刻意不在插件命令后
+  // 强退(tangu worker 这类命令靠打开的句柄常驻),只能命令自己收。macOS 的常驻 daemon 不受影响(会话关闭本就不杀它)。
+  if (sub === 'setup' || sub === 'doctor') {
+    try {
+      return await (sub === 'setup' ? setup() : doctor());
+    } finally {
+      await shutdownComputerUseSession().catch(() => {}); // 收尾失败不能盖掉命令本身的结果
+    }
+  }
+  if (sub === 'stop') {
+    await shutdownComputerUseSession();
+    // mac 的 helper 是常驻 daemon,vendor 的 session shutdown 故意不杀它(TCC 身份稳定、下次秒连)。
+    // 但用户敲 stop 的预期就是「别再碰我屏幕」→ 真杀;杀进程不影响 TCC 授权,下次使用会自动重启。
+    if (process.platform === 'darwin') {
+      const { execFile } = await import('node:child_process');
+      await new Promise<void>((res) => execFile('pkill', ['-f', helperExecutablePath()], () => res()));
+    }
+    console.log('✓ helper stopped.');
+    return 0;
+  }
+  console.log('usage: tangu computer-use <setup|doctor|stop>');
+  return sub ? 1 : 0;
+}
+
+async function setup(): Promise<number> {
+  try {
+    await ensureComputerUseSetup(cliCtx(true));
+    console.log('✓ Computer Use is ready.');
+    return 0;
+  } catch (e: any) {
+    console.error(`✗ ${e?.message || e}`);
+    return 1;
+  }
+}
+
+async function doctor(): Promise<number> {
+  if (!helperInstalled()) {
+    console.log(`✗ native helper not installed (expected at ${helperExecutablePath()}).`);
+    console.log('  Fix: run `tangu computer-use setup`.');
+    return 1;
+  }
+  try {
+    await ensureComputerUseSetup(cliCtx(false)); // helper 已装,这里只查权限;缺 → 抛指导文本
+    console.log('✓ helper installed, permissions granted, backend ready.');
+    return 0;
+  } catch (e: any) {
+    const msg = String(e?.message || e);
+    console.log(`✗ not ready: ${msg}`);
+    // 协议不匹配 = 装着的 helper 二进制比宿主老(新命令/新原生功能它都没有)。
+    // 重装会换二进制,macOS 因此可能重置辅助功能/录屏授权,setup 默认会拒绝 ad-hoc 覆盖 —— 明说怎么绕。
+    if (/protocol/i.test(msg)) {
+      console.log('  The installed helper binary is older than this plugin.');
+      console.log('  Fix: PI_COMPUTER_USE_ALLOW_ADHOC_UPDATE=1 tangu computer-use setup');
+      console.log('       then `tangu computer-use stop` so the new binary takes over.');
+      console.log('       macOS will ask you to re-grant Accessibility + Screen Recording.');
+    } else {
+      console.log('  Fix: run `tangu computer-use setup`.');
+    }
+    return 1;
+  }
+}

package/src/tools.ts

@@ -0,0 +1,289 @@
+/**
+ * 上游 pi 的 11 个 defineTool + 自研 ensure_app → Tangu 的 12 个 ToolDef。翻译要点:
+ *   - parameters:上游 typebox schema 手翻成纯 JSON Schema 字面量(零依赖,不引 typebox)。
+ *   - execute 签名:pi 的 (toolCallId, params, signal, onUpdate, ctx) → Tangu 的 (args, ctx)。
+ *   - 结果:AgentToolResult.content 的 text block 拼成返回 string;image block(observe 截图)→ ctx.collectImage(回灌模型)。
+ *   - 门禁:host + hostExec + 插件启用 + macOS(stickers 同款);浏览器三工具再叠 browser_use 设置。
+ *   - 审批:动作类(act_ui/launch/navigate/evaluate_browser)声明 approval:'command'(与 run_bash 同档);观察类免审。
+ *   - setup:每次工具执行前惰性 ensureComputerUseSetup(hasUI=false)→ 权限缺时 vendor 抛 nonInteractiveError,
+ *     被 catch 成指导文本返回给模型(提示跑 `tangu computer-use setup`)。
+ */
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import type { ToolDef, ToolProvider, ToolContext, AppProfile, PluginStore } from '@forsion/tangu-agent';
+import type { ExtensionContext, AgentToolResult } from './pi-compat.ts';
+import {
+  executeFind, executeObserve, executeSearchUi, executeExpandUi, executeInspectUi,
+  executeAct, executeReadText, executeWaitFor, executeLaunchBrowser, executeNavigateBrowser,
+  executeEvaluateBrowser, ensureComputerUseSetup,
+} from './vendor/bridge.ts';
+import { getComputerUseConfig } from './vendor/config.ts';
+import { isSupportedPlatform } from './helperState.ts';
+import { autoInstallFailed, ensureHelperCurrent, guideMissingPermissions } from './onboarding.ts';
+import { foregroundNote } from './foregroundNote.ts';
+
+const execFileP = promisify(execFile);
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+export const PLUGIN_ID = 'computer-use';
+
+type PiExecute = (
+  toolCallId: string,
+  params: any,
+  signal: AbortSignal | undefined,
+  onUpdate: undefined,
+  ctx: ExtensionContext,
+) => Promise<AgentToolResult<any>>;
+
+// ── JSON Schema 片段(翻译自上游 extensions/computer-use.ts 的 typebox;v0.5.0 契约)──
+// v0.5.0 收紧了整个工具面:动作里删掉 doubleClick(用 clickCount)与 wait(用 wait_for/expect);
+// click 拆成「按 ref」与「按坐标」两个互斥变体(不再混着给);observe 只认 @r root(app/windowTitle
+// 猜名字那套没了);search 用 capability 且不分页(refine 而非 paging);read_text 兼收 @e 与 @o
+// (@o = 输出被截断时给的续读句柄,不需要 stateId);wait_for 与 act.expect 共用同一套条件字段。
+const S = (description?: string, maxLength?: number) => ({ type: 'string', ...(description ? { description } : {}), ...(maxLength ? { maxLength } : {}) });
+const N = (description?: string, extra: Record<string, number> = {}) => ({ type: 'number', ...(description ? { description } : {}), ...extra });
+const stateId = { type: 'string', description: 'Required state id owning every @e ref used by this operation' };
+const mouseButton = { type: 'string', enum: ['left', 'right', 'middle'] };
+const obj = (properties: Record<string, unknown>, required: string[] = []) => ({ type: 'object', properties, required });
+
+// wait_for 与 act_ui.expect 共用的条件字段(上游 conditionProperties)。
+const conditionProps = {
+  ref: S('Specific @e ref to test', 128),
+  scopeRef: S('Restrict matching to this @e subtree', 128),
+  text: S('Text that must match', 512),
+  role: S('Exact normalized role', 128),
+  value: S('Exact normalized value; normally pair with ref'), // 上游此项刻意无 maxLength(要比对的可能是整段文本值)
+  until: { type: 'string', enum: ['present', 'absent'], description: 'Desired condition, default present' },
+  timeoutMs: N('Maximum wait, default 10000ms', { minimum: 100, maximum: 60_000 }),
+};
+
+const clickCount = N('1-3', { minimum: 1, maximum: 3 });
+const point = { x: { type: 'number' }, y: { type: 'number' } };
+
+// act_ui 的 actions:上游 uiAction union(8 变体)→ anyOf
+const uiAction = {
+  anyOf: [
+    obj({ action: { const: 'press' }, ref: S('Actionable outline ref') }, ['action', 'ref']),
+    obj({ action: { const: 'click' }, ref: S('Actionable outline ref'), button: mouseButton, clickCount }, ['action', 'ref']),
+    obj({ action: { const: 'click' }, ...point, button: mouseButton, clickCount }, ['action', 'x', 'y']),
+    obj({ action: { const: 'setText' }, ref: S('Editable outline ref'), text: { type: 'string' } }, ['action', 'ref', 'text']),
+    obj({ action: { const: 'typeText' }, ref: S('Omit after a click to type into the focus established by that click'), text: { type: 'string' } }, ['action', 'text']),
+    obj({ action: { const: 'keypress' }, ref: S('Omit to send keys to the focused control'), keys: { type: 'array', items: { type: 'string' }, minItems: 1 } }, ['action', 'keys']),
+    obj({ action: { const: 'scroll' }, ref: S('Outline ref to scroll'), scrollX: { type: 'number' }, scrollY: { type: 'number' } }, ['action']),
+    obj({ action: { const: 'drag' }, path: { type: 'array', items: obj(point, ['x', 'y']), minItems: 2 } }, ['action', 'path']),
+    obj({ action: { const: 'moveMouse' }, ...point }, ['action', 'x', 'y']),
+  ],
+};
+
+interface ToolSpec {
+  name: string;
+  description: string;
+  parameters: ReturnType<typeof obj>;
+  exec?: PiExecute;
+  // 适配层自有工具(非 vendor 委托),给定则走它,拿到 args/tcx/pctx 直接返回 string。
+  run?: (args: Record<string, any>, tcx: ToolContext, pctx: ExtensionContext) => Promise<string>;
+  sideEffect: 'read' | 'system' | 'browser';
+  approval?: 'command';
+  browserOnly?: boolean;
+  darwinOnly?: boolean;
+  timeoutMs: number;
+}
+
+const SPECS: ToolSpec[] = [
+  {
+    name: 'find_roots', exec: executeFind as PiExecute, sideEffect: 'read', timeoutMs: 30_000,
+    description: 'Find a bounded, ranked set of controllable UI roots with refs, geometry, and focus state. Find a target root before observe_ui when needed.',
+    parameters: obj({
+      text: S('Ranked app or title text', 256),
+      app: S('Exact normalized app name', 256),
+      bundleId: S('Exact bundle id'),
+      pid: N('Exact process id'),
+      kind: { type: 'string', enum: ['window', 'menu', 'sheet', 'popover', 'dialog', 'browser_page'], description: 'Exact root kind' },
+    }),
+  },
+  {
+    name: 'observe_ui', exec: executeObserve as PiExecute, sideEffect: 'read', timeoutMs: 60_000,
+    description: 'Capture the current/frontmost root or one exact @r root and return a bounded UI outline. Use mode=semantic to skip OCR and images, visual to force them, fused for automatic selection. Primary UI observation tool; follow with search_ui/expand_ui/inspect_ui/act_ui. Use @e outline refs from observe_ui/search_ui for act_ui; pictureOnly refs are coordinate-only.',
+    parameters: obj({
+      root: S('Exact @r ref issued by find_roots'),
+      mode: { type: 'string', enum: ['semantic', 'visual', 'fused'], description: 'Observation mode, default fused' },
+    }),
+  },
+  {
+    name: 'search_ui', exec: executeSearchUi as PiExecute, sideEffect: 'read', timeoutMs: 15_000,
+    description: 'Return a bounded, deterministically ranked search of the cached outline. At least one predicate is required. Find targets not shown in the compact observe_ui output; refine broad searches instead of paging matches.',
+    parameters: obj({ text: S('Human-readable text or label', 256), role: S('Exact normalized role, e.g. button', 128), capability: S('Exact capability, e.g. press', 128), stateId }, ['stateId']),
+  },
+  {
+    name: 'expand_ui', exec: executeExpandUi as PiExecute, sideEffect: 'read', timeoutMs: 15_000,
+    description: 'Unfold bounded local outline context for one @e ref. Expand a specific ref instead of dumping unrelated UI.',
+    parameters: obj({ ref: S('Outline ref from observe_ui/search_ui, e.g. @e12'), depth: N('Subtree depth, default 3', { minimum: 1, maximum: 8 }), stateId }, ['ref', 'stateId']),
+  },
+  {
+    name: 'inspect_ui', exec: executeInspectUi as PiExecute, sideEffect: 'read', timeoutMs: 15_000,
+    description: 'Inspect one exact outline ref with fields, geometry, capabilities, and annotations. Use when a target\'s evidence or provenance matters.',
+    parameters: obj({ ref: S('Outline ref from observe_ui/search_ui, e.g. @e12'), stateId }, ['ref', 'stateId']),
+  },
+  {
+    name: 'act_ui', exec: executeAct as PiExecute, sideEffect: 'system', approval: 'command', timeoutMs: 120_000,
+    description: 'Perform one or more precisely targeted checked actions and return the successor state. Pass dependent click/type steps together and use expect for observable completion instead of a separate observe_ui call. After clicking an editable region, omit ref from typeText/keypress so input follows the established focus. To fill a text field prefer a single setText action — it writes the value in the background without taking focus; only click the field first (which takes the foreground) if setText reports it did not take, as happens on some web/Electron inputs. When any action takes the foreground the result says so on a [foreground] line. For strict background-only runs, turn on the plugin\'s "strict background" setting.',
+    parameters: obj({
+      stateId,
+      expect: obj(conditionProps),
+      actions: { type: 'array', items: uiAction, minItems: 1, maxItems: 20 },
+    }, ['stateId', 'actions']),
+  },
+  {
+    name: 'ensure_app', run: runEnsureApp, sideEffect: 'system', darwinOnly: true, timeoutMs: 25_000,
+    description: 'Make sure a desktop app is running and observable WITHOUT bringing it to the foreground, then return its UI roots. Use this before observe_ui when the target app may not be running yet (observe_ui only sees already-running apps). Background-launches by bundleId (preferred) or app name and waits until a window is observable. macOS only.',
+    parameters: obj({
+      app: S('App name, e.g. "TextEdit"'),
+      bundleId: S('Bundle id, e.g. "com.apple.TextEdit" (preferred — unambiguous)'),
+    }),
+  },
+  {
+    name: 'read_text', exec: executeReadText as PiExecute, sideEffect: 'read', timeoutMs: 30_000,
+    description: 'Read a fixed-size page from an @e UI ref or immutable @o truncated-output ref. Use @e with its stateId; @o continuation refs (handed out when a tool result was truncated) do not need stateId.',
+    parameters: obj({ ref: S('@e UI ref or @o output ref'), offset: N('Byte offset, default 0', { minimum: 0 }), stateId }, ['ref']),
+  },
+  {
+    name: 'wait_for', exec: executeWaitFor as PiExecute, sideEffect: 'read', timeoutMs: 30_000,
+    description: 'Wait for one scoped UI condition and return the successor state. Use after asynchronous UI changes instead of polling observe_ui.',
+    parameters: obj({ ...conditionProps, stateId }, ['stateId']),
+  },
+  {
+    name: 'launch_browser', exec: executeLaunchBrowser as PiExecute, sideEffect: 'browser', approval: 'command', browserOnly: true, timeoutMs: 60_000,
+    description: 'Launch the configured Tangu-managed CDP browser and return an observed browser-page state. Use for browser work that needs a managed CDP context. Prefer curl through bash when the page is directly fetchable. Which browser starts is a plugin setting, not a parameter.',
+    parameters: obj({ url: S('Initial URL', 8192) }),
+  },
+  {
+    name: 'navigate_browser', exec: executeNavigateBrowser as PiExecute, sideEffect: 'browser', approval: 'command', browserOnly: true, timeoutMs: 60_000,
+    description: 'Navigate an observed CDP browser-page state to an HTTP(S) URL. Native browser windows use act_ui; this tool is CDP-only.',
+    parameters: obj({ url: S('HTTP(S) URL', 8192), stateId }, ['url', 'stateId']),
+  },
+  {
+    name: 'evaluate_browser', exec: executeEvaluateBrowser as PiExecute, sideEffect: 'browser', approval: 'command', browserOnly: true, timeoutMs: 30_000,
+    description: 'Evaluate targeted JavaScript in a CDP browser-page state; returned output is strictly bounded. Prefer observe/search/read; return selected fields, aggregates, or bounded slices.',
+    parameters: obj({ stateId, expression: S('JavaScript expression', 65_536) }, ['stateId', 'expression']),
+  },
+];
+
+/** 把 pluginStore 的设置就地注入 vendor 的 activeConfig(getComputerUseConfig 返回其引用)。 */
+export function syncSettings(store: PluginStore): void {
+  const s = store.getScopeSettings(PLUGIN_ID, 'global');
+  const cfg = getComputerUseConfig();
+  if (typeof s.browser_use === 'boolean') cfg.browser_use = s.browser_use;
+  if (typeof s.headless === 'boolean') cfg.headless = s.headless;
+  if (typeof s.cursor_overlay === 'boolean') cfg.cursor_overlay = s.cursor_overlay;
+  if (s.managed_browser === 'chrome' || s.managed_browser === 'helium') cfg.managed_browser = s.managed_browser;
+}
+
+/** 构造 vendor 需要的 ExtensionContext。工具执行 hasUI=false(权限缺→vendor 抛指导文本);sessionManager 给空 stub。 */
+export function makePiCtx(tcx: ToolContext, hasUI: boolean): ExtensionContext {
+  return {
+    cwd: tcx.cwd || process.cwd(),
+    hasUI,
+    ui: { notify: () => {}, select: async () => undefined },
+    sessionManager: { getBranch: () => [] },
+    signal: tcx.signal,
+  };
+}
+
+/** AgentToolResult → 返回给模型的 string;image block(截图)经 collectImage 回灌;末尾附前台透明化提示。 */
+function render(result: AgentToolResult, tcx: ToolContext): string {
+  const parts: string[] = [];
+  for (const block of result.content) {
+    if (block.type === 'text') parts.push(block.text);
+    else if (block.type === 'image' && tcx.collectImage) {
+      tcx.collectImage({ url: `data:${block.mimeType || 'image/jpeg'};base64,${block.data}`, name: 'screen.jpg' });
+    }
+  }
+  const body = parts.join('\n') || '(no output)';
+  return body + foregroundNote((result as { details?: { execution?: unknown } }).details?.execution);
+}
+
+/** ensure_app:后台启动目标 app(open -g,不抢前台)+ 轮询 find_roots 直到有可观察窗口,回其 roots。 */
+async function runEnsureApp(args: Record<string, any>, tcx: ToolContext, pctx: ExtensionContext): Promise<string> {
+  if (process.platform !== 'darwin') {
+    return 'ensure_app (background launch) is macOS-only. On other platforms, launch the app yourself, then use observe_ui.';
+  }
+  const bundleId = typeof args.bundleId === 'string' ? args.bundleId.trim() : '';
+  const app = typeof args.app === 'string' ? args.app.trim() : '';
+  if (!bundleId && !app) return 'ensure_app requires "app" or "bundleId".';
+  // -g:启动但不置前台。execFile 传 argv(非 shell),bundleId/app 作单参无注入。
+  const openArgs = ['-g', ...(bundleId ? ['-b', bundleId] : ['-a', app])];
+  try {
+    await execFileP('open', openArgs, { timeout: 8_000 });
+  } catch (e: any) {
+    return `Could not launch ${bundleId || app}: ${e?.stderr?.toString().trim() || e?.message || e}. Check the app/bundleId is installed.`;
+  }
+  const deadline = Date.now() + 12_000;
+  let last = '';
+  while (Date.now() < deadline) {
+    if (tcx.signal?.aborted) return 'ensure_app aborted.';
+    const found = await spec_executeFind(pctx, tcx, app, bundleId);
+    last = found;
+    if (/@r\d/.test(found)) return `Launched ${bundleId || app} in the background (no foreground focus).\n${found}`;
+    await sleep(400);
+  }
+  return `Launched ${bundleId || app} in the background, but no window is observable yet. Give it a moment and call observe_ui.\n${last}`;
+}
+
+/** 内部:调 vendor executeFind 并渲染(供 ensure_app 轮询)。
+ *  bundleId 是精确匹配,有就用它;只有 app 名时走 **text**(排序匹配)而不是 `app` ——
+ *  v0.5.0 起 `app` 是「精确的规范化 app 名」,用户随口给的 "TextEdit" 未必对得上,
+ *  精确匹配落空会让 ensure_app 白等满 12 秒再报「没有可观察窗口」。 */
+async function spec_executeFind(pctx: ExtensionContext, tcx: ToolContext, app: string, bundleId: string): Promise<string> {
+  try {
+    const params = bundleId ? { bundleId } : { text: app };
+    const r = await executeFind('', params, tcx.signal, undefined, pctx);
+    return render(r, tcx);
+  } catch {
+    return '';
+  }
+}
+
+export function buildToolProvider(store: PluginStore): ToolProvider {
+  const gate = (spec: ToolSpec) => (profile: AppProfile): boolean =>
+    !!profile.capabilities.hostExec
+    && store.isPluginEnabledSync(PLUGIN_ID)
+    && isSupportedPlatform()
+    && (!spec.darwinOnly || process.platform === 'darwin')
+    && (!spec.browserOnly || getComputerUseConfig().browser_use !== false);
+
+  const toTool = (spec: ToolSpec): ToolDef => ({
+    name: spec.name,
+    mode: 'host',
+    isEnabledFor: gate(spec),
+    capabilities: { sideEffect: spec.sideEffect, parallel: false, concurrencyKey: 'computer-use', defaultTimeoutMs: spec.timeoutMs, approval: spec.approval },
+    definition: { type: 'function', function: { name: spec.name, description: spec.description, parameters: spec.parameters } },
+    execute: async (args: Record<string, any>, tcx: ToolContext): Promise<string> => {
+      syncSettings(store);
+      // ⚠️这里**绝不能**再加「没装就 return 指导文本」的早退:vendor 的 ensureInstalled() 本来就会
+      // 自动跑 setup-helper.mjs,早退等于把上游的自动安装堵死(用户报的「首次安装还要敲命令」就是它)。
+      // 我们只补上游缺的那半:装着的二进制比 bundle 老时也要重装。
+      const installNote = await ensureHelperCurrent(tcx.signal);
+      const pctx = makePiCtx(tcx, false);
+      try {
+        await ensureComputerUseSetup(pctx, tcx.signal);
+      } catch (e: any) {
+        // 权限是系统安全设置,只能用户自己拨 —— 但可以替他把 App 登记好、把面板打开。
+        const guidance = await guideMissingPermissions(tcx.signal);
+        // ⚠️只有「自动安装本身失败了」才该提终端命令。就绪失败的原因多得很(Linux 缺 AT-SPI、
+        // 架构不符、daemon 起不来……),对这些 case 说「去跑 setup」是把模型引到错的方向上,
+        // 而真正的原因反而被这句话盖住了。
+        const fallback = autoInstallFailed()
+          ? `Computer Use is not ready: ${e?.message || e}\nAutomatic helper installation failed — run \`tangu computer-use setup\` in a terminal to see why.`
+          : `Computer Use is not ready: ${e?.message || e}`;
+        return [installNote, guidance ?? fallback].filter(Boolean).join('\n');
+      }
+      // 装/升级过就在结果里说一句 —— 用户装了什么、agent 替他做了什么,不该是暗箱。
+      const prefix = installNote ? `${installNote}\n` : '';
+      if (spec.run) return prefix + await spec.run(args, tcx, pctx);
+      const result = await spec.exec!('', args, tcx.signal, undefined, pctx);
+      return prefix + render(result, tcx);
+    },
+  });
+
+  return { id: 'plugin:computer-use', tools: () => SPECS.map(toTool) };
+}

package/src/vendor/actions.ts

@@ -0,0 +1,130 @@
+import type { MouseButtonName, UiAction } from "./contract.ts";
+import type { OutlineNode } from "./outline.ts";
+import { toFiniteNumber } from "./platform/coerce.ts";
+
+export type ActionTarget = { ref: string } | { x: number; y: number } | { focus: { x: number; y: number } };
+
+export type PreparedAction =
+	| { action: "press" | "click"; target: ActionTarget; params: { button?: MouseButtonName; clickCount?: number }; establishesFocus: boolean; usesCurrentFocus: false; needsForeground: boolean }
+	| { action: "setText"; target: ActionTarget; params: { text: string }; establishesFocus: false; usesCurrentFocus: false; needsForeground: false }
+	| { action: "typeText"; target: ActionTarget; params: { text: string }; establishesFocus: false; usesCurrentFocus: boolean; needsForeground: false }
+	| { action: "keypress"; target: ActionTarget; params: { keys: string[] }; establishesFocus: false; usesCurrentFocus: boolean; needsForeground: false }
+	| { action: "scroll"; target: ActionTarget; params: { scrollX: number; scrollY: number }; establishesFocus: false; usesCurrentFocus: false; needsForeground: false }
+	| { action: "drag"; target: ActionTarget; params: { path: Array<{ x: number; y: number }> }; establishesFocus: false; usesCurrentFocus: false; needsForeground: false }
+	| { action: "moveMouse"; target: ActionTarget; params: Record<string, never>; establishesFocus: false; usesCurrentFocus: false; needsForeground: false }
+	| { action: "wait"; params: { ms: number }; establishesFocus: false; usesCurrentFocus: false; needsForeground: false };
+
+export interface ActionState {
+	currentFocus: boolean;
+}
+
+export interface ActionEnvironment {
+	headless: boolean;
+	image?: { width: number; height: number };
+	node(ref: string): OutlineNode;
+	center(node: OutlineNode): { x: number; y: number };
+	validatePoint(x: number, y: number, label?: string): void;
+}
+
+function mouseButton(value: unknown): MouseButtonName {
+	return value === "right" || value === "middle" ? value : "left";
+}
+
+function clickCount(value: unknown, fallback = 1): number {
+	return Math.max(1, Math.min(3, Math.round(toFiniteNumber(value, fallback))));
+}
+
+function scrollDelta(value: unknown): number {
+	return Math.max(-10_000, Math.min(10_000, Math.round(toFiniteNumber(value, 0))));
+}
+
+function keys(value: unknown): string[] {
+	if (!Array.isArray(value) || value.length === 0) throw new Error("keypress.keys must contain at least one key.");
+	return value.map((key) => String(key));
+}
+
+function path(value: UiAction["path"], env: ActionEnvironment): Array<{ x: number; y: number }> {
+	if (!Array.isArray(value) || value.length < 2) throw new Error("drag.path must contain at least two points.");
+	return value.map((point, index) => {
+		const x = Array.isArray(point) ? toFiniteNumber(point[0], NaN) : toFiniteNumber(point?.x, NaN);
+		const y = Array.isArray(point) ? toFiniteNumber(point[1], NaN) : toFiniteNumber(point?.y, NaN);
+		env.validatePoint(x, y, `Drag point ${index + 1}`);
+		return { x, y };
+	});
+}
+
+function nativeTarget(action: UiAction, operation: PreparedAction["action"], env: ActionEnvironment): ActionTarget {
+	if (action.ref?.trim()) {
+		const node = env.node(action.ref.trim());
+		const semanticClick = operation === "click" || operation === "press";
+		if (semanticClick && node.isTextInput) {
+			const point = env.center(node);
+			env.validatePoint(point.x, point.y);
+			return point;
+		}
+		const onlyIncidentalActions = node.actions.every((candidate) => candidate === "AXShowMenu" || candidate === "AXScrollToVisible");
+		if (node.wireRef && !node.pictureOnly && (!semanticClick || node.canPress || node.canFocus || node.canSetValue || !onlyIncidentalActions)) {
+			return { ref: node.wireRef };
+		}
+		const point = env.center(node);
+		env.validatePoint(point.x, point.y);
+		return point;
+	}
+	const x = toFiniteNumber(action.x, NaN);
+	const y = toFiniteNumber(action.y, NaN);
+	if (Number.isFinite(x) && Number.isFinite(y)) {
+		env.validatePoint(x, y);
+		return { x, y };
+	}
+	if (operation === "drag" && action.path?.length) return path(action.path, env)[0];
+	throw new Error(`${operation} requires either ref or both x and y.`);
+}
+
+function focusedTarget(env: ActionEnvironment): ActionTarget {
+	if (!env.image) throw new Error("Focused keyboard input requires an image-bearing state.");
+	return { focus: { x: Math.floor(env.image.width / 2), y: Math.floor(env.image.height / 2) } };
+}
+
+function containsEditable(node: OutlineNode): boolean {
+	if (node.canSetValue || node.role.toLowerCase().includes("text")) return true;
+	return node.children.some(containsEditable);
+}
+
+export function prepareAction(action: UiAction, state: ActionState, env: ActionEnvironment): PreparedAction {
+	const operation = action.action;
+	const usesCurrentFocus = !env.headless && state.currentFocus && !action.ref && (operation === "typeText" || operation === "keypress");
+	const target = usesCurrentFocus ? focusedTarget(env) : nativeTarget(action, operation, env);
+	const establishesFocus = !env.headless && Boolean(action.ref) && (operation === "click" || operation === "press") && containsEditable(env.node(action.ref!));
+	const needsForeground = !env.headless && (operation === "click" || operation === "press") && "x" in target;
+
+	switch (operation) {
+		case "press":
+		case "click": return { action: operation, target, params: { button: mouseButton(action.button), clickCount: clickCount(action.clickCount) }, establishesFocus, usesCurrentFocus: false, needsForeground };
+		case "setText": return { action: operation, target, params: { text: action.text ?? "" }, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+		case "typeText": return { action: operation, target, params: { text: action.text ?? "" }, establishesFocus: false, usesCurrentFocus, needsForeground: false };
+		case "keypress": return { action: operation, target, params: { keys: keys(action.keys) }, establishesFocus: false, usesCurrentFocus, needsForeground: false };
+		case "scroll": return { action: operation, target, params: { scrollX: scrollDelta(action.scrollX), scrollY: scrollDelta(action.scrollY) }, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+		case "drag": return { action: operation, target, params: { path: path(action.path, env) }, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+		case "moveMouse": return { action: operation, target, params: {}, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+	}
+}
+
+export function canRetryInForeground(action: PreparedAction, outcome: "worked" | "didnt" | "unknown", headless: boolean): boolean {
+	return !headless && outcome === "didnt" && (action.action === "typeText" || action.action === "keypress");
+}
+
+export function outcomeAfterCheck(current: "worked" | "didnt" | "unknown", check: "verified" | "preexisting" | "failed"): "worked" | "didnt" | "unknown" {
+	if (check === "verified") return "worked";
+	if (check === "failed") return "didnt";
+	return current;
+}
+
+export function outcomeAfterObservedValues(
+	current: "worked" | "didnt" | "unknown",
+	actions: UiAction[],
+	valueForRef: (ref: string) => string | undefined,
+): "worked" | "didnt" | "unknown" {
+	if (actions.length === 0 || actions.some((action) => action.action !== "setText" || !action.ref)) return current;
+	const matches = actions.every((action) => valueForRef(action.ref!) === (action.text ?? ""));
+	return matches ? "worked" : current;
+}