@mirasim/computer-use 0.0.0-stage → 0.1.0-rc.3

package/dist/platform/windows/launch.d.ts

@@ -0,0 +1,23 @@
+import type { ActResult, TargetInfo } from "../types.ts";
+export declare const LAUNCH_WAIT_MS = 10000;
+export declare const LAUNCH_NEEDS_DESKTOP: string;
+export interface StartApp {
+    name: string;
+    id: string;
+}
+export interface Started {
+    name: string;
+    via: string;
+    pid: number;
+}
+interface Row {
+    hwnd: number;
+    pid: number;
+    process: string;
+    title: string;
+}
+export declare function filterApps(list: StartApp[], query: string): ActResult;
+export declare function startAppTarget(list: StartApp[], query: string): TargetInfo | null;
+export declare function launchedBy(app: string, started: Started): (r: Row) => boolean;
+export declare function pickAppeared<R extends Row>(appeared: R[], named: (r: R) => boolean): R | undefined;
+export {};

package/dist/platform/windows/launch.js

@@ -0,0 +1 @@
+const s=1e4,r="starting an app opens its window in front of the user and takes their keyboard focus, so it needs interaction:'desktop': explain that and obtain permission first. Nothing was started";function a(t,e){const n=e.trim().toLowerCase(),i=n?t.filter(o=>o.name.toLowerCase().includes(n)||o.id.toLowerCase().includes(n)):t;return{ok:!0,value:i,note:`${i.length} of ${t.length} Start-menu apps${n?` match "${e}"`:""}`}}function p(t,e){const n=e.trim().toLowerCase(),i=n?t.filter(o=>o.id.toLowerCase()===n||o.name.toLowerCase()===n):[];return i.length!==1?null:{id:`app:${i[0].id}`,kind:"app",title:`${i[0].name} \u2014 no window open; binding it starts the app (its window opens in front of the user)`,active:!1}}function c(t,e){const n=t.trim().toLowerCase().split(/[\\/]/).pop().replace(/\.exe$/,""),i=e.name.toLowerCase();return o=>e.pid!==0&&o.pid===e.pid||o.process.toLowerCase()===n||o.title.toLowerCase().includes(i)}function d(t,e){return t.find(e)??(t.length===1?t[0]:void 0)}export{r as LAUNCH_NEEDS_DESKTOP,s as LAUNCH_WAIT_MS,a as filterApps,c as launchedBy,d as pickAppeared,p as startAppTarget};

package/dist/platform/windows/menu-verb.d.ts

@@ -0,0 +1,7 @@
+import type { ActResult } from "../types.ts";
+export interface MenuHost {
+    call<T>(op: string, params?: Record<string, unknown>): Promise<T>;
+    hwnd(): number;
+    canOpen: boolean;
+}
+export declare function menuVerb(host: MenuHost, rawPath: unknown): Promise<ActResult>;

package/dist/platform/windows/menu-verb.js

@@ -0,0 +1,6 @@
+import{UiaRequestError as $}from"./uia-client.js";function d(e){return e.split("	")[0].replace(/\s*\(&?[^()\s]\)/g,"").replace(/&(.)/g,"$1").replace(/(…|\.\.\.)\s*$/,"").trim()}const f=(e,n)=>d(e).toLowerCase()===d(n).toLowerCase(),m=e=>e.map(d).join(" \u203A "),E=new Set(["\u7CFB\u7EDF","system","\u7CFB\u7EDF\u83DC\u5355\u680F","system menu bar"]);function C(e){return`${m(e.path)}${e.submenu?" \u203A":""}${e.checked?" (checked)":""}${e.enabled?"":" (disabled)"}`}function g(e,n,o){const t=e.filter(s=>s.enabled),a=e.length-t.length,r=(a?`
+(${a} marked disabled do not apply to what is open or selected right now)`:"")+(o.length?`
+(${o.map(s=>`${s} \u203A`).join(", ")} list their items only once opened \u2014 that needs interaction:'desktop')`:"");return{ok:!0,note:`${t.length} menu commands runnable (${n}; listed below \u2014 this verb returns them):
+${e.map(C).join(`
+`)}${r}`}}function L(e){return e.split(`
+`).slice(1).filter(n=>n&&!n.startsWith("(")&&!n.endsWith("(disabled)"))}function I(e,n){const o=e.find(t=>t.path.length===n.length&&t.path.every((a,r)=>f(a,n[r])));if(o)return o;if(n.length===1){const t=e.filter(a=>f(a.path[a.path.length-1],n[0]));if(t.length===1)return t[0];if(t.length>1)return`"${n[0]}" names ${t.length} menu commands (${t.map(a=>m(a.path)).join(", ")}) \u2014 give the whole path`}return`no menu command "${n.join(" \u203A ")}" \u2014 menu({}) lists them`}async function O(e){try{return(await e.call("hmenu",{hwnd:e.hwnd()})).items}catch(n){if(n instanceof $&&n.code==="unknown_op")return[];throw n}}async function _(e,n,o){const t=I(n,o);return typeof t=="string"?{ok:!1,note:t}:t.submenu?g(n.filter(a=>a.path.length===t.path.length+1&&t.path.every((r,s)=>r===a.path[s])),`items of ${m(t.path)}`,[]):t.enabled?(await e.call("hmenucmd",{hwnd:e.hwnd(),item:t.id}),{ok:!0,note:`ran ${m(t.path)} through the menu's own command (no menu was opened) \u2014 observe for its effect; a dialog it opens is listed with the window`}):{ok:!1,note:`${m(t.path)} is disabled right now \u2014 the app does not offer it for what is open or selected. Nothing was run`}}async function w(e){const n=(await e.call("tree",{hwnd:e.hwnd(),max:3e3})).elements,o=[],t=[];return n.forEach((a,r)=>{for(;t.length&&n[t[t.length-1]].depth>=a.depth;)t.pop();o.push(t.length?t[t.length-1]:-1),t.push(r)}),{nodes:n,parent:o}}function T(e){const n=(t,a)=>{if(t.id==="SystemMenuBar"||E.has(d(t.name).toLowerCase()))return!1;for(let r=e.parent[a];r!==-1;r=e.parent[r])if(["TitleBar","MenuItem"].includes(e.nodes[r].type))return!1;return p(e,a).length>0},o=e.nodes.findIndex((t,a)=>t.type==="MenuBar"&&n(t,a));return o!==-1?o:e.nodes.findIndex((t,a)=>t.type==="Menu"&&n(t,a))}function p(e,n){const o=[];return e.nodes.forEach((t,a)=>{if(t.type!=="MenuItem")return;let r=e.parent[a];for(;r!==-1&&r!==n&&e.nodes[r].type!=="MenuItem";)r=e.parent[r];r===n&&o.push(a)}),o}const k=e=>!e.offscreen&&e.rect.w>0&&e.rect.h>0,b=e=>`${e.name}|${e.rect.x},${e.rect.y}`,v=e=>e.expanded==="collapsed"||e.expanded==="expanded",A=e=>new Promise(n=>setTimeout(n,e));async function j(e,n){try{await e.call("pattern",{key:n.key,action:"expand"})}catch(o){if(!(o instanceof $))throw o;await S(e,n)}await A(250)}async function S(e,n){try{return await e.call("press",{key:n.key})}catch(o){if(!(o instanceof $)||o.code!=="no_action"&&o.code!=="prefer_mouse")throw o;return await e.call("click",{x:Math.round(n.rect.x+n.rect.w/2),y:Math.round(n.rect.y+n.rect.h/2),button:"left",count:1}),{via:"mouse"}}}function B(e,n,o){const t=new Set(e.nodes.filter(i=>i.type==="MenuItem"&&k(i)).map(b)),a=n.nodes.findIndex(i=>i.type==="MenuItem"&&b(i)===b(o)),r=a===-1?[]:p(n,a),s=n.nodes.map((i,u)=>i.type==="MenuItem"&&k(i)&&!t.has(b(i))?u:-1).filter(i=>i!==-1),c=new Set;return[...r,...s].filter(i=>{const u=b(n.nodes[i]);return!c.has(u)&&!!c.add(u)})}async function y(e,n){n>0&&await e.call("key",{chord:"escape",count:n+1}).catch(()=>{})}async function N(e,n,o){const t=[],a=[];for(const r of p(n,o)){const s=n.nodes[r];t.push({path:[s.name],enabled:!s.disabled,submenu:!0});let c=p(n,r).map(i=>n.nodes[i]);if(!c.length&&e.canOpen&&!s.disabled){await j(e,s);const i=await w(e);c=B(n,i,s).map(u=>i.nodes[u]),await y(e,1)}else c.length||a.push(d(s.name));for(const i of c)t.push({path:[s.name,i.name],enabled:!i.disabled,submenu:v(i),...i.toggle==="on"?{checked:!0}:{}})}return g(t,"the window's menu bar",a)}async function W(e,n,o,t){if(t.length===1&&!p(n,o).some(s=>f(n.nodes[s].name,t[0]))){const s=await N(e,n,o),c=L(s.note??"").map(u=>({path:u.replace(/ ›$| \((checked|disabled)\)/g,"").split(" \u203A "),enabled:!0,submenu:!1})),i=I(c,t);if(typeof i=="string")return{ok:!1,note:i};t=i.path,n=await w(e)}let a=p(n,o).map(s=>n.nodes[s]),r=0;for(const[s,c]of t.entries()){const i=a.find(l=>f(l.name,c)&&k(l))??a.find(l=>f(l.name,c)),u=t.slice(0,s+1).join(" \u203A ");if(!i){await y(e,r);const l=a.map(h=>d(h.name)).filter(Boolean);return{ok:!1,note:`no "${c}" in ${s?t.slice(0,s).join(" \u203A "):"the menu bar"} (it has: ${l.join(", ")||"nothing readable"}). Nothing was run`}}if(i.disabled)return await y(e,r),{ok:!1,note:`${u} is disabled right now \u2014 the app does not offer it for what is open or selected. Nothing was run`};const M=s===t.length-1;if(M&&!v(i)&&s>0){const l=await S(e,i);return{ok:!0,journal:{via:l.via},note:l.pending?`ran ${u}; the app is still handling it \u2014 typically the dialog it opened. Observe to see it; do not run it again`:`ran ${u} (${l.via==="mouse"?"clicked":`pressed through UI Automation (${l.via})`}) \u2014 observe for its effect`}}if(!e.canOpen)return{ok:!1,note:"running a menu command opens the menu and moves the window's focus, so it needs interaction:'desktop'; explain that and obtain permission first. Nothing was run"};const U=await w(e);await j(e,i),r++;const x=await w(e);if(a=B(U,x,i).map(l=>x.nodes[l]),M){await y(e,r);const l=a.map(h=>({path:[...t,h.name],enabled:!h.disabled,submenu:v(h)}));return g(l,`items of ${t.map(d).join(" \u203A ")}`,[])}}return{ok:!1,note:"menu needs a path"}}async function R(e,n){const o=typeof n=="string"?n.trim():"",t=o?o.split(/\s*(?:›|>|\/)\s*/).filter(Boolean):[],a=await O(e);if(a.length)return t.length?e.canOpen?_(e,a,t):{ok:!1,note:"running a menu command acts on the app, so it needs interaction:'desktop'; explain that and obtain permission first. Nothing was run"}:g(a.filter(c=>!c.submenu||c.path.length>1),"the window's classic menu bar, state refreshed the way opening it would",[]);const r=await w(e),s=T(r);return s===-1?{ok:!1,note:"this window has no menu bar UI Automation can see \u2014 its commands are buttons, a ribbon or a toolbar: observe and click them"}:t.length?W(e,r,s,t):N(e,r,s)}export{R as menuVerb};

package/dist/platform/windows/prompt-fragment.d.ts

@@ -0,0 +1,2 @@
+export declare const WINDOWS_INTERACTION_NOTES: readonly string[];
+export declare function promptFragmentFor(): string;

package/dist/platform/windows/prompt-fragment.js

@@ -0,0 +1 @@
+const e='Every app \u2014 reading it included \u2014 must be cleared for this session. One not cleared yet is listed by its program name only ("WINWORD \u2014 not cleared in this session yet\u2026"); bind it by that name, and the bind clears it: at once when the user always allows it, otherwise by asking them (pass `purpose` to say why). If they decline, do not retry or ask again in this task.',t="Windows defaults to limited background UIA: supported setText, click, uiaAction and scroll by ref do not activate the window or inject mouse/keyboard input, and never fall back to those routes. Apps may still react by opening windows or changing focus; detected interference stops background actions on that window. Do not automatically replay uncertain actions. They are refused while the target window is in front, where the user may be typing: they would move focus inside it. Coordinates, type, keys, hover, activation and unsupported controls need interaction:'desktop', which brings the window to the FRONT and can use the real pointer/keyboard. Explain those effects and obtain permission before choosing desktop. Reading never activates the window; app-access approval is required in both modes.",a=[t,e,"Background setText requires a readable, writable ValuePattern on an editor in a standard Win32/WinForms/WPF window (including Notepad), a Chromium page (Chrome, Edge), an Electron app or a XAML app; password fields and unsupported editors are refused before writing. It never focuses, selects all or types as a fallback.","In a Chromium page, an Electron app (VS Code\u2026) or a XAML/DirectUI app (Settings, Calculator, Explorer), background click does what a desktop click does through UIA: it selects an item, presses a button or link, toggles a checkbox, or expands/collapses; uiaAction({ref, action}) runs a named one. Elsewhere it only selects standard list items and toggles WinForms/WPF checkboxes; other Win32/WinForms/WPF buttons, menus and expansion are excluded. Those apps must be unminimized (they may stay covered). A press is dispatched, not verified: observe its result. Missing patterns are refusals, not a reason to switch modes automatically.","Background scroll needs a specific ScrollPattern container ref from observe/find; window-wide scrolling has no background fallback. count is 1\u201320 UIA large increments; to:'top'/'bottom' uses the scroll percentage.","In desktop mode, click by ref does what a single click does: it SELECTS a list/tree/table item (open one with count:2), picks an entry of an open combo box list, and presses a control through its UIA action \u2014 WinForms buttons and menu items with the real mouse, since their handlers run inside UIA and a dialog they open would freeze it. A press that opens a dialog reports the app as still busy; observe and answer the dialog, never press again. Right/middle/double clicks and point clicks use the real mouse. A tree node [collapsed] is expanded with uiaAction({ref, action:'expand'}), not by clicking it. interaction:desktop is not proof of consent; obtain it first.","Open menus, dropdown lists and context menus are read with the window: their items are listed last.","An app with no window open is started by binding its Start-menu name (computer.surface('Calculator')) or, from a bound window, launchApp({app}); listApps({query}) gives the names, which are in the system language. Starting one opens its window in front of the user, so both need interaction:'desktop' and the user's permission; the app itself is cleared like any bind.","A menu-bar command runs by its path: menu({path:'File \u203A Save As'}) \u2014 names in the system language, so list them first with menu({}); the last segment alone works when unique, and a path ending at a submenu lists it. A classic Win32 menu bar runs the command directly; others are opened level by level. Prefer it to clicking through menus yourself.","A datetime element (a Win32/WinForms date or time picker) shows its text and exact date; setText(ref, '2026-03-09'), '14:30' or '2026-03-09 14:30' types it field by field as a person would (the app sees each step) and reads it back. A calendar element is read-only here: pick a day by clicking it in a screenshot.","To change part of a field, selectText({ref, text, near}) selects that text (read back from the field) and focuses it; type, keys or paste then replace just that. caret:'before'/'after' puts the caret there instead; caret alone goes to the start/end. paste({text, html, ref}) uses Ctrl+V through the user's clipboard, which is put back afterwards and kept out of clipboard history: tell the user, and pass ref so a paste into the wrong field is refused before the clipboard is touched.","drag({ref or x/y, toRef or toX/toY}) holds the real left button along a path and releases it: files onto folders, slider thumbs, strokes on a drawing canvas (which often has no ref \u2014 take a screenshot and drag by coordinates inside it).","If the user is typing in another window, Windows can refuse to bring the target forward; the action then reports that and sends nothing. While a Windows security prompt is in front, nothing can come forward until the user answers it: tell them, do not retry.","The user comes first. A desktop action waits up to 1.5 s for them to pause the keyboard and pointer for 2 s, and otherwise is refused with nothing sent \u2014 wait and retry. If they switched to another app since your last action, they have taken over: actions are refused until you tell them what is left and, once they agree, take the window back with kind:'activate'. A type or keys call stops part-way when they touch the keyboard or mouse; observe what landed instead of resending.","setText waits for the requested value. type captures the focused editor and its selection before input, then waits for that same editor to match the expected edit when readable. A mismatch stops subsequent actions; unavailable read-back is explicitly unverified. Do not replay uncertain input. Field comparisons normalize CRLF/LF; real trailing newlines are retained.","keys accepts Multiply, Add, Subtract and Divide (also NumpadMultiply, NumpadAdd, NumpadSubtract and NumpadDivide), case-insensitively. These send the native keypad key, without depending on Shift+8 or a keyboard layout.","Decimal and NumpadDecimal enter the system's decimal separator as text, without changing NumLock. They do not accept modifiers. NumLock itself is available only as an explicit keypress; Decimal never toggles it."];function s(){return t+" Grounding: observations are text lists of one Windows application window's controls with refs, read from its UI Automation tree (the same tree screen readers use). Act by ref from the latest observation. set_text uses a field's UI Automation value; focusing/selecting/typing fallback exists only in desktop mode. A control you can SEE and cannot name (self-drawn, no node in the tree) is reachable by position: take w.observe({screenshot: true}) and w.click({x, y}) with coordinates read off that picture. Chromium/Electron apps publish their tree a moment after the first read, so a near-empty first observation of one is worth repeating once. Windows are surfaces: computer.surfaces() lists every visible application window by title and program name \u2014 "+e+" Use text() to read all text in the window."}export{a as WINDOWS_INTERACTION_NOTES,s as promptFragmentFor};

package/dist/platform/windows/text-verbs.d.ts

@@ -0,0 +1,10 @@
+import type { ActResult } from "../types.ts";
+import type { RefTarget } from "./elements.ts";
+export interface TextHost {
+    target(ref: unknown): RefTarget;
+    call<T>(op: string, params?: Record<string, unknown>): Promise<T>;
+    hwnd(): number;
+}
+export declare function selectText(host: TextHost, a: Record<string, any>): Promise<ActResult>;
+export declare function setDate(host: TextHost, t: RefTarget, text: string): Promise<ActResult>;
+export declare function paste(host: TextHost, a: Record<string, any>): Promise<ActResult>;

package/dist/platform/windows/text-verbs.js

@@ -0,0 +1,5 @@
+import{parseDateTime as _}from"./dates.js";import{UiaRequestError as d}from"./uia-client.js";const S=2500;function M(n){let o="";const e=[];for(let t=0;t<n.length;t++)e.push(t),n[t]==="\r"?(o+=`
+`,n[t+1]===`
+`&&t++):o+=n[t];return e.push(n.length),{lf:o,at:e}}function T(n,o,e,t){return Math.max(0,e-(n+o),n-(e+t))}const l=n=>`${n.type.toLowerCase()} "${n.name}"`;async function N(n,o){const e=n.target(o.ref),t=String(o.caret??"select");if(!["select","before","after"].includes(t))return{ok:!1,note:"caret is select, before or after"};if(!(e.type==="Edit"||e.type==="Document"||e.actions.includes("text")))return{ok:!1,note:`${l(e)} is not a text field \u2014 select_text works on text fields and documents; click a control instead`};let a;try{a=String((await n.call("value",{key:e.key})).value??"")}catch(r){if(r instanceof d)return{ok:!1,note:`${l(e)} cannot be read (${r.message}), so nothing in it can be selected`};throw r}const{lf:s,at:w}=M(a),i=String(o.text??"").replace(/\r\n?/g,`
+`);let h,k,y="";if(i){const r=[];for(let c=s.indexOf(i);c!==-1;c=s.indexOf(i,c+1))r.push(c);if(!r.length)return{ok:!1,note:`"${i}" is not in ${l(e)} \u2014 read it with text() and copy it exactly, whitespace included`};let u=r[0],$=0;const f=String(o.near??"").replace(/\r\n?/g,`
+`);if(f){const c=s.indexOf(f);if(c===-1)return{ok:!1,note:`"${f}" is not in ${l(e)}, so it cannot pick between the ${r.length} matches for "${i}" \u2014 drop near, or give text that really sits beside the one you mean`};let m=1/0;for(const[v,x]of r.entries()){const b=T(x,i.length,c,f.length);b<m&&(m=b,u=x,$=v)}}r.length>1&&(y=` (match ${$+1} of ${r.length}${f?`, the one nearest "${f}"`:", the first"})`),h=t==="after"?u+i.length:u,k=t==="select"?u+i.length:h}else{if(t==="select")return{ok:!1,note:"select_text needs the text to select, or caret:'before' / caret:'after' alone to put the caret at the start / end of the field"};h=k=t==="after"?s.length:0}const p=w[h],g=w[k];try{await n.call("selectrange",{key:e.key,start:p,length:g-p,want:a.slice(p,g),prefix:a.slice(0,p)})}catch(r){if(r instanceof d&&(r.code==="no_selection"||r.code==="stale_key"))return{ok:!1,note:`${r.message}${r.code==="no_selection"?" \u2014 setText replaces the whole field instead":""}`};if(r instanceof d&&r.code==="unknown_op")return{ok:!1,note:"this Windows native helper predates select_text; update @mirasim/computer-use, or the app that bundles it"};throw r}return i?t==="select"?{ok:!0,note:`selected "${i}" in ${l(e)}${y}, read back from the field \u2014 keys, type or paste now replace exactly this (setText still replaces the whole field)`}:{ok:!0,note:`caret placed ${t} "${i}" in ${l(e)}${y} \u2014 keys, type or paste now insert there`}:{ok:!0,note:`caret at the ${t==="after"?"end":"start"} of ${l(e)}, which has the keyboard focus \u2014 keys and type go there now`}}async function O(n,o,e){const t=_(e);if(!t)return{ok:!1,note:`"${e}" is not a date or time this picker can take \u2014 give the date as 2026-03-09, the time as 14:30, or both ("2026-03-09 14:30"). Nothing was typed`};let a;try{a=await n.call("setdate",{key:o.key,want:t})}catch(s){if(!(s instanceof d))throw s;return s.code==="unknown_op"?{ok:!1,note:"this Windows native helper predates date pickers; update the full desktop app. Nothing was typed"}:{ok:!1,mayHaveActed:s.code!=="date_unreadable",note:s.message}}return a.verified?{ok:!0,verified:!0,note:`${l(o)} now reads ${a.shown} (${a.iso}) \u2014 typed field by field the way a person would, so the app saw each step`}:{ok:!0,verified:!1,note:`${l(o)} reads ${a.shown} (${a.iso}) after typing, not what was asked \u2014 do not retype blindly; observe it`}}async function W(n,o){const e=String(o.text??"");if(!e)return{ok:!1,note:"paste needs text (and optionally html for apps that take rich text)"};const t=typeof o.ref=="string"?n.target(o.ref).key:void 0;let a;try{a=await n.call("paste",{hwnd:n.hwnd(),text:e,...typeof o.html=="string"&&o.html?{html:o.html}:{},...t?{key:t}:{},verifyMs:S})}catch(s){if(!(s instanceof d))throw s;if(["wrong_focus","clipboard_busy","clipboard_large","stale_key"].includes(s.code))return{ok:!1,note:s.message};if(s.code==="unknown_op")return{ok:!1,note:"this Windows native helper predates paste; update @mirasim/computer-use, or the app that bundles it. Nothing was put on the clipboard"};throw s}return{ok:!0,note:a.note,...typeof a.verified=="boolean"?{verified:a.verified}:{},journal:{restored:a.restored}}}export{W as paste,N as selectText,O as setDate};

package/dist/platform/windows/uia-client.d.ts

@@ -0,0 +1,32 @@
+export interface UiaTransport {
+    onReset?: () => void;
+    request(op: string, params?: Record<string, unknown>): Promise<unknown>;
+    close(): Promise<void>;
+}
+export declare class UiaRequestError extends Error {
+    readonly code: string;
+    constructor(code: string, message: string);
+}
+export interface UiaHelperOptions {
+    env?: NodeJS.ProcessEnv;
+    onStart?: (pid: number) => Promise<void>;
+    onLog?: (line: string) => void;
+    timeoutMs?: number;
+}
+export declare class UiaHelperProcess implements UiaTransport {
+    private readonly opts;
+    onReset?: () => void;
+    private child;
+    private starting;
+    private readonly pending;
+    private nextId;
+    private stderrTail;
+    private readonly env;
+    private readonly timeoutMs;
+    constructor(opts?: UiaHelperOptions);
+    request(op: string, params?: Record<string, unknown>): Promise<unknown>;
+    close(): Promise<void>;
+    private start;
+    private onLine;
+    private fail;
+}

package/dist/platform/windows/uia-client.js

@@ -0,0 +1,3 @@
+import{spawn as _,execFile as y}from"node:child_process";import{createHash as E}from"node:crypto";import{existsSync as c,mkdirSync as S,readdirSync as m,readFileSync as f,rmSync as b,statSync as h}from"node:fs";import{homedir as R}from"node:os";import{join as o}from"node:path";import{fileURLToPath as w}from"node:url";import{createInterface as x}from"node:readline";import{computerUseHome as T}from"../home.js";import{platformPackageHelper as $}from"../platform-package.js";class l extends Error{code;constructor(i,t){super(t),this.code=i,this.name="UiaRequestError"}}class j{opts;onReset;child=null;starting=null;pending=new Map;nextId=1;stderrTail="";env;timeoutMs;constructor(i={}){this.opts=i,this.env=i.env??process.env,this.timeoutMs=i.timeoutMs??2e4}async request(i,t={}){const e=await this.start(),s=this.nextId++;return new Promise((a,n)=>{const d=setTimeout(()=>{this.pending.delete(s),this.close().then(()=>n(new l("timeout",`uia helper: '${i}' did not answer within ${this.timeoutMs} ms; helper stopped`)))},this.timeoutMs);this.pending.set(s,{resolve:a,reject:n,timer:d}),e.stdin.write(`${JSON.stringify({id:s,op:i,...t})}
+`)})}async close(){const i=this.child;this.fail("uia helper closed; any in-flight action's outcome is unknown \u2014 observe before acting again"),!(!i||i.exitCode!==null)&&await new Promise(t=>{const e=setTimeout(()=>i.kill(),2e3);i.once("exit",()=>{clearTimeout(e),t()}),i.stdin.end()})}start(){return this.child?Promise.resolve(this.child):(this.starting??=(async()=>{const i=await P(this.env),t=_(i,[],{stdio:["pipe","pipe","pipe"],env:this.env,windowsHide:!0});return this.stderrTail="",t.stdin.on("error",()=>{}),t.stderr.on("data",e=>{const s=e.toString();this.stderrTail=`${this.stderrTail}${s}`.slice(-2e3);for(const a of s.split(`
+`)){const n=a.replace(/^computer-use-uia-helper:\s*/,"").trim();n&&this.opts.onLog?.(n)}}),x({input:t.stdout}).on("line",e=>{this.child===t&&this.onLine(e)}),t.on("exit",e=>{this.child===t&&this.fail(`uia helper exited (${e??"?"})${this.stderrTail?`: ${this.stderrTail.trim()}`:""}`)}),t.on("error",e=>{this.child===t&&this.fail(`uia helper could not start: ${e.message}`)}),this.child=t,t.pid&&await this.opts.onStart?.(t.pid),t})().finally(()=>{this.starting=null}),this.starting)}onLine(i){let t;try{t=JSON.parse(i)}catch{return}const e=typeof t.id=="number"?this.pending.get(t.id):void 0;e&&(this.pending.delete(t.id),clearTimeout(e.timer),t.ok?e.resolve(t.result??{}):e.reject(new l(t.error?.code??"uia_error",t.error?.message??"uia helper error")))}fail(i){this.child=null;for(const[t,e]of this.pending)clearTimeout(e.timer),e.reject(new l("helper_gone",i)),this.pending.delete(t);this.onReset?.()}}const u="computer-use-uia-helper.exe";function v(){const r=w(new URL("./uia/",import.meta.url));return c(o(r,"Cargo.toml"))?r:null}function I(){const r=w(new URL(`./${u}`,import.meta.url));return c(r)?r:null}function M(r){const i=E("sha256");for(const e of["Cargo.toml","Cargo.lock"])c(o(r,e))&&i.update(f(o(r,e)));const t=o(r,"src");for(const e of m(t).filter(s=>s.endsWith(".rs")).sort())i.update(e).update(f(o(t,e)));return i.digest("hex").slice(0,16)}async function P(r){const i=r.COMPUTER_USE_WINDOWS_UIA_BIN?.trim();if(i){if(!c(i))throw new Error(`COMPUTER_USE_WINDOWS_UIA_BIN points at ${i}, which does not exist`);return i}if(process.platform!=="win32")throw new l("unsupported","the windows driver needs Windows");const t=I()??$(u);if(t)return t;const e=v();if(!e)throw new Error(`windows uia helper: no prebuilt binary (COMPUTER_USE_WINDOWS_UIA_BIN, \`${u}\` beside the bundle, or the @mirasim/computer-use-${process.platform}-${process.arch} package) and no Rust sources beside the bundle \u2014 a packaged build must ship the helper`);const s=o(T(),"windows-uia"),a=o(s,M(e)),n=o(a,"release",u);return c(n)||(S(a,{recursive:!0}),await new Promise((d,g)=>{y(H(r),["build","--release","--manifest-path",o(e,"Cargo.toml"),"--target-dir",a],{timeout:6e5,env:r,windowsHide:!0},(p,O,U)=>{p?g(new Error(`windows uia helper: cargo build failed (is the Rust toolchain installed? winget install Rustlang.Rustup): ${String(U||p.message).slice(-800)}`)):d()})}),N(s,a)),n}function H(r){const i=o(r.USERPROFILE??R(),".cargo","bin","cargo.exe");return c(i)?i:"cargo"}function N(r,i){try{const t=m(r).map(e=>o(r,e)).filter(e=>e!==i&&h(e).isDirectory()).sort((e,s)=>h(s).mtimeMs-h(e).mtimeMs);for(const e of t.slice(3))b(e,{recursive:!0,force:!0})}catch{}}export{j as UiaHelperProcess,l as UiaRequestError};

package/dist/platform/windows/verbs.d.ts

@@ -0,0 +1,2 @@
+import type { VerbSpec } from "../types.ts";
+export declare const WINDOWS_VERBS: VerbSpec[];

package/dist/platform/windows/verbs.js

@@ -0,0 +1 @@
+import{Type as e}from"typebox";const n=[{kind:"activate",schema:e.Object({kind:e.Literal("activate")},{description:"Bring the bound window to the front (restoring it if minimized) and give it the user's keyboard; explain this and obtain permission first. Also how to take the window back after the user has taken over."})},{kind:"type",schema:e.Object({kind:e.Literal("type"),text:e.String({description:"Text to type with real keystrokes into whatever holds the window's keyboard focus"})},{description:"Type text as real key events into the focused control. When its original text and selection are readable, wait for the expected result in that same field; a mismatch stops the call without retyping. Brings the window to the front."})},{kind:"uia_action",schema:e.Object({kind:e.Literal("uia_action"),ref:e.String({description:"Element ref from the latest observe() (e.g. e12)"}),action:e.Union(["expand","collapse","select","toggle","invoke"].map(t=>e.Literal(t)),{description:"expand/collapse a tree node, combo box or menu; select an item; toggle a checkbox; invoke (an item's invoke OPENS it, like a double-click)"})},{description:"Run one UI Automation action the control itself offers, exactly as named \u2014 for what a click does not do: a click SELECTS a tree node ([collapsed]), expand shows its children. Refused when the control does not offer it. Desktop mode brings the window to the front; background mode runs it without doing so on Chromium, Electron and XAML controls."})},{kind:"list_apps",readOnly:!0,schema:e.Object({kind:e.Literal("list_apps"),query:e.Optional(e.String({description:"Only entries whose name or id contains this (any case), e.g. 'notepad' or 'paint'"}))},{description:"The apps launchApp can start: the Start menu's entries as {name, id} \u2014 names are in the system language (Calculatrice on a French Windows), so look them up here rather than guessing. A read; starts nothing."})},{kind:"launch_app",schema:e.Object({kind:e.Literal("launch_app"),app:e.String({description:"A Start-menu name or id from listApps() ('Notepad', 'Microsoft.WindowsCalculator_8wekyb3d8bbwe!App'), or a program name the Run box takes ('notepad', 'mspaint')"})},{description:"Start an app and bind its new window as this session's target (an app not cleared yet asks the user, like binding it). Opens a window in front of the user: explain and obtain permission first (interaction:'desktop'). A single-instance app may answer by bringing its existing window forward \u2014 or, like Windows 11 Notepad, open a tab in the user's own window; that window is then the target."})},{kind:"menu",schema:e.Object({kind:e.Literal("menu"),path:e.Optional(e.String({description:"Menu command to run, as the listing shows it \u2014 names are in the system language (Fichier \u203A Enregistrer sous on a French Windows), so list first (menu({})) rather than guessing. \u203A > / all separate; the last segment alone also works when unique; a path ending at a submenu lists its items. Omit to list the window's menu commands."}))},{description:"List the window's menu-bar commands, or run one by its path. A classic Win32 menu bar runs the command directly, opening nothing; other menu bars are opened level by level and the item pressed, the way a person does. Running one brings the window to the front: explain and obtain permission first. Listing is a read; in background mode it opens no menu."})},{kind:"select_text",schema:e.Object({kind:e.Literal("select_text"),ref:e.String({description:"The text field or document to select inside (from the latest observe())"}),text:e.Optional(e.String({description:"The exact text to select (copy it verbatim from what you read); omit it to put the caret at the start / end of the field with caret alone"})),near:e.Optional(e.String({description:"Nearby text that tells one occurrence from another; the match closest to it wins"})),caret:e.Optional(e.Union([e.Literal("select"),e.Literal("before"),e.Literal("after")],{description:"select (default) leaves it selected \u2014 type, keys or paste then replace it; before / after put the caret there instead"}))},{description:"Select part of a field's text, or put the caret in it, and give the field the keyboard focus \u2014 so type, keys or paste edit exactly that; set_text always replaces the whole field. The selection is read back from the field. Brings the window to the front."})},{kind:"paste",schema:e.Object({kind:e.Literal("paste"),text:e.String({description:"The content as plain text"}),html:e.Optional(e.String({description:"The same content as an HTML fragment, for apps that take rich text (Word, Outlook, web editors)"})),ref:e.Optional(e.String({description:"The element you EXPECT to have the keyboard focus. Not a destination \u2014 the paste goes to the focus \u2014 but a mismatch is refused before the clipboard is touched. Pass it whenever you know where the text should go."}))},{description:"Paste with Ctrl+V at the window's keyboard focus: brings the window to the front, puts the content on the user's clipboard (kept out of clipboard history and cloud sync) and puts their clipboard back afterwards, unless they copied something meanwhile. Explain these effects and obtain permission first. For long text and rich text; it is read back from the focused field when it can be."})},{kind:"drag",schema:e.Object({kind:e.Literal("drag"),ref:e.Optional(e.String({description:"Element to drag \u2014 REQUIRED unless x/y are given"})),x:e.Optional(e.Number({description:"Drag start x in the latest screenshot (instead of ref)"})),y:e.Optional(e.Number({description:"Drag start y"})),toRef:e.Optional(e.String({description:"Element to drop onto \u2014 REQUIRED unless toX/toY are given"})),toX:e.Optional(e.Number({description:"Destination x in the latest screenshot (instead of toRef)"})),toY:e.Optional(e.Number({description:"Destination y"})),steps:e.Optional(e.Number({description:"Motion events along the path (default 20, 2\u201360); more is slower and smoother, for apps that sample the pointer per frame"}))},{description:"Drag with the user's real pointer: press the left button at the start, move to the destination, release \u2014 a file onto a folder, a slider thumb, a line in a drawing app. Brings the window to the front and moves the pointer; explain and obtain permission first. Needs a start (ref or x/y) AND a destination (toRef or toX/toY). Stops and releases if the user touches the mouse."})}];export{n as WINDOWS_VERBS};

package/extension/background.js

@@ -0,0 +1,280 @@
+/**
+ * Service worker: owns the single active bridge connection.
+ *
+ * ONE way in: **native messaging**. `chrome.runtime.connectNative` makes Chrome spawn mirasim's
+ * host, which pumps into the running bridge. No address is handed over, so nothing expires —
+ * after a browser restart, a mirasim update or a worker eviction this worker just dials again.
+ * An open port also keeps the worker alive.
+ *
+ * The **connect page** (`bridge.connect` below, over a one-shot `ws://127.0.0.1/…` relay) is NOT
+ * a product path: its relay uuid is minted per bridge process, so every restart would need a
+ * human click. Mirasim never opens it; it is the seam end-to-end tests use in a throwaway
+ * Chromium with no native-host manifest.
+ *
+ * One bridge at a time: a new connection replaces the old.
+ *
+ * Reconnect policy: probe with ping/pong, tell "host not installed" from "host went away" by the
+ * lastError text and stop fast retries on the former, back off 1s→30s on the latter, and arm a
+ * `chrome.alarms` alongside the timer because an evicted worker takes its timers with it.
+ */
+
+import { RelayConnection, debugLog } from "./relay.js";
+import { MirasimTabGroup } from "./group.js";
+import { nativeTransport, webSocketTransport } from "./transport.js";
+
+const CONNECT_TIMEOUT_MS = 10_000;
+/** How long the native probe waits for `pong` before calling the route dead. */
+const PROBE_TIMEOUT_MS = 10_000;
+const BACKOFF_FIRST_MS = 1_000;
+const BACKOFF_MAX_MS = 30_000;
+const RECONNECT_ALARM = "mirasim-bridge-reconnect";
+/** How often to look again once every candidate said "not installed". Never NEVER: a user who
+ *  installs mirasim while Chrome is open would wait for a browser restart. Not slow either: this
+ *  recheck IS the time-to-first-connection after loading the extension, and one wake a minute
+ *  is what the heartbeat already costs. */
+const ABSENT_RECHECK_MINUTES = 1;
+/**
+ * The heartbeat, armed even while CONNECTED. A closed native port lets the worker terminate
+ * (Chrome docs), so when the bridge's host exits the worker can die with it: no `onDisconnect`
+ * runs and no timer survives. Only this alarm wakes it to reconnect; the handler returns early
+ * while a connection exists.
+ */
+const HEARTBEAT_MINUTES = 1;
+
+/**
+ * Host names, in the order they are tried. The installed app owns the first; a dev instance
+ * writes the second, so the two can coexist on one machine without fighting over one manifest
+ * (the shape OpenAI's extension uses for its own `.dev` host).
+ */
+const HOST_CANDIDATES = ["ai.mirasim.browser_bridge", "ai.mirasim.browser_bridge.dev"];
+
+/** Chrome's way of saying the manifest is missing or policy forbids it — as opposed to a host
+ *  that started and then went away. The first means retrying forever is pointless. */
+const ABSENT_RE = /native messaging host not found|native messaging host is forbidden|not found|forbidden/i;
+
+class BridgeExtension {
+  constructor() {
+    this._activeGroup = null;
+    /** Transport of the active connection, for the status reply. */
+    this._transport = null;
+    /** Set once every candidate answered "not installed" — stops the retry loop until
+     *  something changes (a new install, or the user's explicit connect). */
+    this._hostAbsent = false;
+    this._connecting = false;
+    this._attempt = 0;
+    this._timer = null;
+    this._lastError = null;
+    this._updateVersion = null;
+    chrome.runtime.onUpdateAvailable.addListener(({ version }) => {
+      this._updateVersion = version;
+      if (this._activeGroup) this._activeGroup.updateAvailable(version);
+      else if (!this._connecting) chrome.runtime.reload();
+    });
+
+    chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
+      // Promise-based responses are not supported by Chrome; return true when async.
+      switch (message?.type) {
+        case "bridge.connect":
+          this._connectWs(message.relayUrl, sender).then(
+            () => sendResponse({ success: true }),
+            (e) => sendResponse({ success: false, error: e?.message ?? String(e) }),
+          );
+          return true;
+        case "bridge.status":
+          sendResponse({
+            connected: !!this._activeGroup,
+            transport: this._transport,
+            hostAbsent: this._hostAbsent,
+            reconnectAttempt: this._attempt,
+            error: this._lastError,
+            tabIds: this._activeGroup?.connectedTabIds() ?? [],
+          });
+          return false;
+        default:
+          return false;
+      }
+    });
+
+    // An alarm survives worker eviction; the timer does not. Both land here.
+    chrome.alarms.onAlarm.addListener((alarm) => {
+      if (alarm.name !== RECONNECT_ALARM) return;
+      if (this._activeGroup) return; // connected: the heartbeat has nothing to do
+      // Drop the absent flag first, or `_connectNative` returns at once and never rechecks.
+      this._hostAbsent = false;
+      void this._connectNative();
+    });
+    chrome.runtime.onStartup.addListener(() => void this._connectNative());
+    chrome.runtime.onInstalled.addListener(() => void this._connectNative());
+    // Worker start IS the reconnect trigger: eviction plus any event wakes us right back up.
+    void this._connectNative();
+  }
+
+  // ─── native messaging ────────────────────────────────────────────────────
+
+  /** Try each host in turn; keep the first that answers `pong`. */
+  async _connectNative() {
+    if (this._activeGroup || this._connecting || this._hostAbsent) return;
+    this._connecting = true;
+    let everyCandidateAbsent = true;
+    try {
+      for (const name of HOST_CANDIDATES) {
+        const port = await this._probe(name);
+        if (port === "absent") continue;
+        everyCandidateAbsent = false;
+        if (!port) continue;
+        await this._adopt(new RelayConnection(nativeTransport(port)), "native");
+        debugLog("bridge connected over native messaging:", name);
+        return;
+      }
+    } catch (e) {
+      this._lastError = e?.message ?? String(e);
+      debugLog("native connect failed:", e);
+    } finally {
+      this._connecting = false;
+    }
+    if (everyCandidateAbsent) {
+      // No manifest anywhere, or policy says no: stop the fast retries but keep a slow alarm,
+      // the only thing that survives worker eviction to notice an install.
+      this._hostAbsent = true;
+      if (this._timer) clearTimeout(this._timer);
+      this._timer = null;
+      this._armAlarm(ABSENT_RECHECK_MINUTES);
+      debugLog(`no native host installed; rechecking every ${ABSENT_RECHECK_MINUTES}min, fallback path meanwhile`);
+      return;
+    }
+    this._scheduleRetry();
+  }
+
+  /**
+   * Connect one candidate and ask it whether the whole route is alive.
+   *
+   * The host answers `pong` only once it has the bridge's socket, so a pong means mirasim is
+   * running and reachable — not merely that a binary exists. Returns the port on success,
+   * `"absent"` when Chrome says the host is not installed/forbidden, else null.
+   */
+  _probe(name) {
+    return new Promise((resolve) => {
+      let port;
+      try {
+        port = chrome.runtime.connectNative(name);
+      } catch (e) {
+        resolve(ABSENT_RE.test(e?.message ?? "") ? "absent" : null);
+        return;
+      }
+      let settled = false;
+      const finish = (value) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        port.onDisconnect.removeListener(onDisconnect);
+        port.onMessage.removeListener(onMessage);
+        if (value !== port) {
+          try {
+            port.disconnect();
+          } catch {
+            /* already gone */
+          }
+        }
+        resolve(value);
+      };
+      const onDisconnect = () => {
+        const message = chrome.runtime.lastError?.message ?? "";
+        this._lastError = message || "native host disconnected";
+        finish(ABSENT_RE.test(message) ? "absent" : null);
+      };
+      const onMessage = (msg) => {
+        if (msg?.type === "pong") finish(port);
+      };
+      port.onDisconnect.addListener(onDisconnect);
+      port.onMessage.addListener(onMessage);
+      const timer = setTimeout(() => finish(null), PROBE_TIMEOUT_MS);
+      try {
+        port.postMessage({ type: "ping" });
+      } catch {
+        finish(null);
+      }
+    });
+  }
+
+  _scheduleRetry() {
+    this._attempt += 1;
+    const delay = Math.min(BACKOFF_FIRST_MS * 2 ** (this._attempt - 1), BACKOFF_MAX_MS);
+    this._clearRetry();
+    this._timer = setTimeout(() => void this._connectNative(), delay);
+    // Alarms are minute-grained, so this is the floor, not the schedule: the timer wins while
+    // the worker lives and the alarm is what picks up after an eviction.
+    this._armAlarm(HEARTBEAT_MINUTES);
+    debugLog(`native reconnect in ${delay}ms (attempt ${this._attempt})`);
+  }
+
+  /** Idempotent enough for our purposes: creating an existing alarm replaces it. */
+  _armAlarm(minutes) {
+    chrome.alarms.create(RECONNECT_ALARM, { delayInMinutes: minutes, periodInMinutes: minutes });
+  }
+
+  /** Stop the FAST retries only. The alarm is never cleared here — see HEARTBEAT_MINUTES. */
+  _clearRetry() {
+    if (this._timer) clearTimeout(this._timer);
+    this._timer = null;
+  }
+
+  // ─── the connect page (TEST SEAM — see the file header) ──────────────────
+
+  async _connectWs(relayUrl, sender) {
+    if (typeof relayUrl !== "string" || !/^wss?:\/\/(127\.0\.0\.1|localhost|\[::1\])[:/]/.test(relayUrl))
+      throw new Error("Refusing non-local relay URL");
+    const ws = await this._openWebSocket(relayUrl);
+    await this._adopt(new RelayConnection(webSocketTransport(ws)), "ws");
+    debugLog("bridge connected:", relayUrl);
+    // A user who went through the page has told us the host is worth looking for again.
+    this._hostAbsent = false;
+
+    // The connect page did its one job; close it so nothing flashes longer than it must.
+    const senderTabId = sender?.tab?.id;
+    if (senderTabId !== undefined) await chrome.tabs.remove(senderTabId).catch(() => {});
+  }
+
+  _openWebSocket(relayUrl) {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(relayUrl);
+      const timer = setTimeout(() => {
+        ws.close();
+        reject(new Error("Timed out connecting to the mirasim bridge"));
+      }, CONNECT_TIMEOUT_MS);
+      ws.onopen = () => {
+        clearTimeout(timer);
+        resolve(ws);
+      };
+      ws.onerror = () => {
+        clearTimeout(timer);
+        reject(new Error("Could not reach the mirasim bridge (is mirasim running?)"));
+      };
+    });
+  }
+
+  // ─── whichever transport won ─────────────────────────────────────────────
+
+  async _adopt(connection, transport) {
+    this._activeGroup?.close("Replaced by a new bridge connection");
+    this._activeGroup = null;
+    const group = new MirasimTabGroup(connection);
+    group.onclose = () => {
+      if (this._activeGroup !== group) return;
+      this._activeGroup = null;
+      this._transport = null;
+      // Native reconnects on its own; the ws path needs a new hand-over.
+      if (transport === "native") this._scheduleRetry();
+    };
+    this._activeGroup = group;
+    this._transport = transport;
+    this._attempt = 0;
+    this._lastError = null;
+    this._clearRetry();
+    // Armed WHILE connected, deliberately — see HEARTBEAT_MINUTES.
+    this._armAlarm(HEARTBEAT_MINUTES);
+    await group.ready;
+    if (this._updateVersion) group.updateAvailable(this._updateVersion);
+  }
+}
+
+new BridgeExtension();

package/extension/connect.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Mirasim</title>
+    <style>
+      body {
+        font: 14px/1.5 system-ui, sans-serif;
+        color: #333;
+        display: grid;
+        place-items: center;
+        min-height: 90vh;
+        margin: 0;
+      }
+      main { text-align: center; max-width: 28rem; padding: 0 1rem; }
+      .err { color: #b91c1c; }
+      .quiet { color: #888; font-size: 12px; }
+    </style>
+  </head>
+  <body>
+    <main>
+      <p id="status">Connecting to Mirasim…</p>
+      <p class="quiet">This tab closes itself once the bridge is up.</p>
+    </main>
+    <script src="connect.js"></script>
+  </body>
+</html>

package/extension/connect.js

@@ -0,0 +1,28 @@
+/**
+ * Connect page: carries the relay URL from the local bridge to the service
+ * worker. The bridge opens this page through the browser binary (reusing the
+ * running Chrome); on success the service worker closes the tab, so the page
+ * is on screen for well under a second.
+ */
+
+const status = document.getElementById("status");
+const relayUrl = new URLSearchParams(location.search).get("relay");
+
+if (!relayUrl) {
+  status.textContent = "Missing relay URL — this page is meant to be opened by Mirasim.";
+  status.className = "err";
+} else {
+  chrome.runtime.sendMessage({ type: "bridge.connect", relayUrl }, (response) => {
+    if (chrome.runtime.lastError) {
+      status.textContent = `Bridge error: ${chrome.runtime.lastError.message}`;
+      status.className = "err";
+      return;
+    }
+    if (response?.success) {
+      status.textContent = "Connected — handing this tab back.";
+    } else {
+      status.textContent = `Could not connect: ${response?.error ?? "unknown error"}`;
+      status.className = "err";
+    }
+  });
+}