@mirasim/computer-use 0.0.0-stage → 0.1.0-rc.2

package/dist/platform/tools/repl.js

@@ -0,0 +1,19 @@
+import{createContext as Ie,runInContext as Ne}from"node:vm";import{Type as f}from"typebox";import{spec as pe,text as O}from"../types.js";import{MAX_ACTIONS as we,MAX_MS as ge,SYNC_MAX_MS as me,RETURN_CAP as ee,MAX_REF_OWNERS as je,trimObservation as Me}from"./repl-budget.js";import{createComputerUseApi as Pe,noteOf as ve,renderJournal as Fe,stringify as Ue,verbGate as Je}from"./api.js";import{prepareCode as Be}from"./repl-code.js";import{apiReference as be,platformReference as He,REPL_DESCRIPTION as We,RESET_DESCRIPTION as qe}from"./repl-docs.js";import{ComputerUseOwnership as Ge,OwnershipBusyError as ye}from"../ownership.js";const $e=new Set(["observe","find","text","value","page","screenshot","diagnostics","targets","log","verb","release"]),Ke=new Set(["dialog","close_tab"]);function nt(j,M={}){if(!j.length)throw new Error("createReplTools needs at least one platform");let D=null;const S=M.ownership??new Ge,v=new WeakMap,H=new WeakMap;let L=null,A=!1,te=Promise.resolve(),W=!1,P=null,I=null,b,q=!!M.docsDelivered,G=!!M.docsDelivered,x=null;const k=()=>{if(!x)throw new Error("no computer_repl call is in progress \u2014 every verb must be awaited inside the call that started it");return x},g=new Map,R=new Map,_=new WeakMap;for(const{driver:e}of j)e.onDisconnect(async()=>{for(const n of R.values()){if(_.get(n)?.slice.driver!==e)continue;const t=v.get(n);t&&await S.release(t),v.delete(n);for(const[i,s]of g)s===n.id&&g.delete(i)}});const K=j.length>1;let w=null,y=null;const Se=e=>{const n=w?.id;if(n){for(const t of e)g.delete(t),g.set(t,n);for(const t of g.keys()){if(g.size<=je)break;g.delete(t)}}},ke=e=>{if(!w)return;const n=g.get(e);if(n&&n!==w.id)throw new Error(`ref ${e} was read on surface ${n}, not on ${w.id} (${w.title}) \u2014 act on it through computer.surface('${n}'), or observe ${w.id} and use one of its refs`)},X=Je(e=>x?.push("verb","",!1,e)),u=j.map(e=>({...e,id:e.driver.platform,api:Pe(e.driver,k,{onRefs:Se,onTarget:ke,beforeAction:async(n,t)=>{if(!w)throw new Error("An action needs a bound surface.");const i=v.get(w);if(!i)throw new Error("The target was released; observe it before acting again.");try{await S.claim(i,await e.driver.actionResources(n,t))}catch(s){throw s instanceof ye&&(L=s.message),s}}}),docsDue:!M.docsDelivered,lastMiss:null}));if(b=u[0],K)for(const e of u)e.driver.surfaceIdPrefix=`${e.id}/`;const re=u[0].api,z=async()=>{const e=[],n=y?[y]:u;for(const t of n){let i;try{i=await t.driver.listTargets()}catch(s){if(n.length===1)throw s;const c=s instanceof Error?s.message:String(s);t.lastMiss!==c&&x?.push("targets",t.id,!1,`${t.id} is not reachable: ${c}`),t.lastMiss=c;continue}t.lastMiss=null;for(const s of i)e.push({...s,id:K?`${t.id}/${s.id}`:s.id,slice:t,inner:s.id})}return e};class Y extends Error{}const Q=async e=>{const n=_.get(e);if(b=n.slice,I=e,await n.slice.driver.isActiveTarget(n.inner)){await ne(e,!0);return}k().charge("focus");const t=await n.slice.driver.focusTarget(n.inner);if(k().push("focus",e.id,t.ok,ve(t)||`\u2192 ${e.title}`),!t.ok)throw R.delete(e.id),new Y(`surface ${e.id} (${e.title}) is gone from this session \u2014 closed, replaced or out of reach; when that is known, a NOTE in this result or the one before it names which (it is said once, when the loss is seen). Call computer.surfaces() and re-acquire what you need.`);await ne(e,!1)},ne=async(e,n)=>{const{slice:t,inner:i}=_.get(e),s=t.driver;let c=await s.ownershipResources();const l=v.get(e);if((!l||l!==c[0]?.key)&&!await s.isActiveTarget(i)){const o=await s.focusTarget(i);if(!o.ok)throw new Y(`Surface ${e.id} disappeared while binding; list surfaces and re-acquire it.`);if(k().push("focus",e.id,!0,ve(o)||`\u2192 ${e.title}`),c=await s.ownershipResources(),!await s.isActiveTarget(i))throw new Error("The target kept changing while binding; observe before continuing.")}const d=c[0]?.key;if(!d)throw new Error("The driver did not identify a resource to claim.");const E=!!l&&l!==d;if(E&&!c.some(o=>H.get(e)?.has(o.key)))throw R.delete(e.id),v.delete(e),H.delete(e),await S.release(l),new Error("This handle now resolves to a different target. List surfaces and bind the intended one again.");try{await S.claim(d,c)}catch(o){throw o instanceof ye&&(L=o.message),o}E&&await S.release(l),v.set(e,d),H.set(e,new Set(c.map(o=>o.key))),!l&&!n&&s.invalidateObservation(),await s.prepareTarget()},V=e=>{const n=R.get(e.id);if(n)return n.title=e.title,n.kind=e.kind,n.active=e.active,n.own=e.own,n.unloaded=e.unloaded,n;const t={id:e.id,kind:e.kind,title:e.title,active:e.active,...e.own===void 0?{}:{own:e.own},...e.unloaded?{unloaded:!0}:{}};_.set(t,{slice:e.slice,inner:e.inner});const i={};for(const[s,c]of Object.entries(e.slice.api))typeof c!="function"||s==="log"||s==="focus"||s==="targets"||(i[s]=async(...l)=>{w=t;try{try{await Q(t)}catch(d){if(s==="wait"&&d instanceof Y)return`${t.id} is gone \u2014 closed or replaced, so there is nothing left on it to wait for. computer.surfaces() lists what remains`;throw d}return await c(...l)}finally{w=null}});return i.focus=async()=>{w=t;try{await Q(t)}finally{w=null}},i.release=async()=>{k().charge("release");const s=v.get(t);await e.slice.driver.releaseInputResources(),s&&await S.release(s),v.delete(t),e.slice.driver.invalidateObservation();for(const[c,l]of g)l===t.id&&g.delete(c);return k().push("release",t.id,!0,"ownership released; observe again before continuing"),"Ownership released. Other sessions may now use this target; observe before continuing."},Object.assign(t,X(i)),R.set(e.id,t),t},Ee=(e,n)=>{const t=o=>[...o.filter(m=>m.own),...o.filter(m=>!m.own)].map(m=>`  ${m.id}  ${m.title}${m.own?"  (yours \u2014 opened by this session)":""}`).join(`
+`);if(!e.length)throw new Error("this driver has no surface right now");if(n===void 0){if(e.length===1)return e[0];throw new Error(`${e.length} surfaces are open \u2014 name one, computer.surface('<id or title>'):
+${t(e)}`)}const i=String(n).trim(),s=e.filter(o=>o.id===i);if(s.length===1)return s[0];const c=i.toLowerCase(),l=e.filter(o=>o.title.toLowerCase()===c),d=l.length?l:e.filter(o=>o.title.toLowerCase().includes(c));if(d.length===1)return d[0];const E=d.filter(o=>o.primary);if(E.length===1)return E[0];if(!d.length){const o=[...new Set(u.map(m=>m.driver.explainMiss(i)).filter(Boolean))].join(`
+`);throw new Error(`no surface matches ${JSON.stringify(i)} \u2014 the open ones are:
+${t(e)}${o?`
+${o}`:""}`)}throw new Error(`${d.length} surfaces match ${JSON.stringify(i)} \u2014 name one exactly:
+${t(d)}`)},Re=async e=>{try{return await se(e)}catch(n){const t=e.toLowerCase(),i=(await z()).filter(s=>s.active&&(s.id===e||s.title.toLowerCase().includes(t)));if(i.length===1)return i[0];throw n}},se=async e=>{const n=await z(),t=e===void 0?"":String(e).trim();if(t&&!n.some(i=>i.id===t||i.title.toLowerCase().includes(t.toLowerCase())))for(const i of y?[y]:u){const s=await i.driver.unlistedTarget(t);if(s)return{...s,id:K?`${i.id}/${s.id}`:s.id,slice:i,inner:s.id}}return Ee(n,e)},Te=X({surfaces:async()=>{k().charge("targets");const e=await z();return k().push("targets","",!0,`${e.length} surfaces`),e.map(V)},surface:async e=>V(await se(e)),log:re.log,help:()=>P=u.map(e=>be(e.driver)).join(`
+
+`)}),Oe=new Set(Object.keys(re)),ie=(e,n)=>{const t=/^(?:ReferenceError: )?(\w+) is not defined$/.exec(e);return t?Oe.has(t[1])?`${e} \u2014 verbs live on a surface handle, not on the top level: const w = await computer.surface('<id or title>'); await w.${t[1]}(\u2026). computer.surface() with no argument binds the only surface when there is one; computer.surfaces() lists them.`:`${e} \u2014 `+(n?"this call started a FRESH runtime, so nothing declared in an earlier call survives (the server was restarted, or its host reaped it while idle). ":"no call on this runtime declared it at the top level, and nothing is predefined but computer. ")+"If it is a surface handle, bind it: const w = await computer.surface('<id or title>').":e},xe=()=>Ie({computer:Te,console:void 0}),_e=e=>Object.entries(e).flatMap(([n,t])=>t&&typeof t=="object"&&_.has(t)&&R.get(t.id)===t?[`${n} \u2192 ${t.id} \xAB${t.title}\xBB`]:[]),F=pe({name:"computer_repl",label:"Computer REPL",description:We,promptSnippet:"computer_repl runs JavaScript over the GUI in a persistent session: verbs, surface handles, one observation per call.",promptGuidelines:["Plan several actions per computer_repl call \u2014 one call costs one step and one observation whatever it does inside.","Keep surface handles and collected data in top-level declarations; they persist across calls.","Re-find by name inside a loop rather than reusing a ref you read earlier: find() handles re-resolve, raw refs do not survive a re-render."],parameters:f.Object({code:f.String({description:"JavaScript, run as the body of an async function in the persistent runtime. `await` every verb; `return` what you want back. Bind once \u2014 first call: `const w = await computer.surface();` \u2014 then later calls use `w` directly: `for (const e of (await w.observe()).elements.filter(e => e.role === 'checkbox')) await w.click(e.ref);`"}),purpose:f.Optional(f.String({maxLength:500,description:"Briefly describe the intended task when requesting app access. Shown to the user as the agent's stated purpose; it does not narrow the app permission. Do not include secrets."})),interaction:f.Optional(f.Union([f.Literal("background"),f.Literal("desktop")],{description:"Native apps default to background \u2014 macOS: driver mouse/activation/clipboard routes are blocked, including fallback; Windows: limited UIA setText/click/scroll by ref, with automatic focus and real-input fallback disabled; unsupported routes are refused. Use desktop only after explaining effects and obtaining permission covering this call. This option is not user consent."})),platform:f.Optional(f.String({description:'For a HOST that mounts several platforms in one session: the platform this call is about ("browser", "electron", "macos", "windows"). computer.surfaces() and computer.surface() then see only its surfaces, and the observation is of it; the other platforms stay mounted and untouched.'})),target:f.Optional(f.String({description:"Bind this surface before the code runs \u2014 the same string computer.surface() takes (an id, a whole title, or a piece of one). For a HOST that decides which surface a call is about: the code can still bind whatever it likes, and this only settles where a run that binds nothing lands."})),observation:f.Optional(f.Union([f.Literal("full"),f.Literal("brief"),f.Literal("none")],{description:'How much of the end-of-run observation to render. "full" (default) is the element list and the text excerpt; "brief" keeps the head (title, targets, verdict) and drops the listing; "none" leaves only the action journal. The observation is always TAKEN either way \u2014 the change verdict and the stale-ref bookkeeping are unaffected \u2014 and a NOTE line always survives, so a swallowed click is never hidden by this. Reach for it when the window holds content you do not want in the transcript, or when your own `return` value is the answer.'}))}),async execute(e){L=null,P=null;const n=String(e.code??"");if(!n.trim())return O("computer_repl needs code to run.",{error:"bad_args"});const t=String(e.platform??"").trim();if(y=t?u.find(r=>r.id===t)??null:null,t&&!y)return O(`platform ${JSON.stringify(t)} is not mounted in this session (mounted: ${u.map(r=>r.id).join(", ")}).`,{error:"bad_args"});y&&(b=y);const i=e.purpose;for(const r of u)r.driver.interactionMode=e.interaction==="desktop"?"desktop":"background",r.driver.consentPurpose=typeof i=="string"&&i.trim().slice(0,500)||null,r.driver.permissionRefusal=null;const s=e.observation,c=s==="brief"||s==="none"?s:"full",l=[],d=[],E=Date.now(),o=()=>u.reduce((r,a)=>r+a.driver.waitedOnUserMs,0),m=o();let Le=0,U=null,oe=!1,J=null;const ae={fields:0,verified:0,ended:null};x={round:ae,charge(r){if(J??=Date.now(),A)throw new Error("The runtime was reset; this run stops before its next operation.");if(++Le>we)throw new Error(`step cap reached (${we} actions) at ${r} \u2014 split the work across calls, or narrow the loop`);if(Date.now()-E-(o()-m)>ge)throw new Error(`time cap reached (${ge/1e3}s) at ${r} \u2014 the surface may be slower than the code assumes`)},push(r,a,$,T,ue,fe){!ue&&$&&!$e.has(r)&&!Ke.has(r)&&(U=r),!ue&&!$&&!$e.has(r)&&(oe=!0);const he=J===null?void 0:Date.now()-J;J=null,l.push({i:l.length+1,kind:r,target:a,ok:$,note:T,...fe?{outcome:fe}:{},...he!==void 0?{ms:he}:{}})},attach(r){d.push(r)}};const le=!D;D??=xe();let B,h=null;try{const r=String(e.target??"").trim();r&&await Q(V(await Re(r))),B=await Ne(Be(n),D,{filename:"computer_repl",timeout:me})}catch(r){r.code==="ERR_SCRIPT_EXECUTION_TIMEOUT"&&(r=new Error(`the code ran for ${me/1e3}s without awaiting anything and was stopped \u2014 a loop that never awaits blocks the whole runtime; await a verb (or w.wait) inside it`)),h=ie(r instanceof Error?`${r.name==="Error"?"":`${r.name}: `}${r.message}`:String(r),le)}finally{try{await X.drain()}catch(r){h??=ie(r instanceof Error?r.message:String(r),le)}x=null}const ce=Date.now()-E,de=(ce/1e3).toFixed(1),N=r=>({runMs:ce,...r!==void 0?{observeMs:r}:{},steps:l.map(a=>({kind:a.kind,target:a.target,ok:a.ok,...a.outcome?{outcome:a.outcome}:{},...a.ms!==void 0?{ms:a.ms}:{},note:a.note.slice(0,160)}))}),p=[h?`Run STOPPED after ${l.length} steps (${de}s): ${h}`:`Run completed ${l.length} steps in ${de}s.`,...Fe(l)];if(h&&l.length>0&&p.push("Nothing was returned: the throw ended the call before `return`. What survives a stop \u2014 top-level bindings (read them in your next call) and anything already passed to log() (in the journal above)."),h){const r=_e(D);r.length&&p.push(`Still bound: ${r.join(", ")} \u2014 use ${r.length===1?"it":"them"} directly in the next call.`)}if(B!==void 0){const r=Ue(B);p.push(`Returned: ${r.length>ee&&B!==P?`${r.slice(0,ee)}\u2026 (cut at ${ee} of ${r.length} chars \u2014 return a slice, a count or a filtered list instead of the whole thing)`:r}`)}const C=u.filter(r=>r.docsDue&&(r===b||u.length===1)).map(r=>{r.docsDue=!1;const a=!q;return q=!0,{type:"text",text:`
+${a?be(r.driver):He(r.driver)}`,pinned:!0}});G&&(G=!1,C.push({type:"text",text:`
+This computer_repl runtime was restarted (idle), so top-level bindings and refs from before are gone; the API reference you already have still applies \u2014 computer.help() returns it again.`,pinned:!0})),U?b.driver.noteWriteRound(ae):oe&&b.driver.noteContentRead();const Ae=U?{afterAction:U}:{},Z=u.find(r=>r.driver.permissionRefusal)?.driver.permissionRefusal;if(P&&l.length===0){const r=O(p.join(`
+`),{verdict:"unknown",ledgerLines:p.length,timing:N(),...h?{error:"run_stopped"}:{}});return r.content.push(...C),r}if(Z||L){const r=Z||L;h?.includes(r)||p.push(`Observation skipped: ${r}`);const a=O(p.join(`
+`),{verdict:"unknown",ledgerLines:p.length,timing:N(),...h?{error:"run_stopped"}:{},...Z?{permissionRefused:!0}:{ownershipRefused:!0}});return a.content.push(...d,...C),a}try{if(A||I&&(!v.has(I)||!await b.driver.isActiveTarget(_.get(I).inner))){const $=b.driver.takeNoteForResult(),T=O(`${p.join(`
+`)}${$?`
+NOTE: ${$}.`:""}
+Observation skipped: the bound target was released, closed or replaced. List surfaces and bind the intended target before continuing.`,{verdict:"unknown",ledgerLines:p.length,timing:N(),...h?{error:"run_stopped"}:{}});return T.content.push(...d,...C),T}const r=Date.now(),a=await b.driver.observeElements({prefix:p.join(`
+`),...Ae});return a.details.ledgerLines=p.length,a.details.timing=N(Date.now()-r),Me(a,c),a.content.push(...d,...C),h&&(a.details.error="run_stopped"),a}catch(r){if(r.surfaceGone)throw r;const a=r.message,$=h&&h.includes(a)?"for the same reason":`${a} \u2014 observe again before the next action`,T=O(`${p.join(`
+`)}
+(re-observation failed: ${$}.)`,{verdict:"unknown",ledgerLines:p.length,timing:N(),...h?{error:"run_stopped"}:{}});return T.content.push(...d,...C),T}}}),Ce=F.execute.bind(F);F.execute=async e=>{W=!0;let n;te=new Promise(t=>{n=t});try{return await Ce(e)}finally{try{if(A){for(const t of u)await t.driver.releaseInputResources();await S.releaseAll(),A=!1}y=null;for(const t of u)t.driver.endCall(),t.driver.consentPurpose=null,t.driver.permissionRefusal=null}finally{W=!1,n()}}};const De=pe({name:"computer_repl_reset",label:"Computer REPL reset",description:qe,promptSnippet:"computer_repl_reset discards the persistent JavaScript bindings; surfaces are untouched.",parameters:f.Object({}),async execute(e){const n=te;if(W)A=!0;else{for(const t of u)await t.driver.releaseInputResources();await S.releaseAll()}D=null,I=null,R.clear(),g.clear();for(const t of u)t.docsDue=!0;return q=!1,G=!1,e?.waitForIdle&&await n,O("The JavaScript runtime was reset: every binding is gone. The next computer_repl call starts fresh and returns the API reference. No surface was touched \u2014 tabs, windows and apps are as they were.",{reset:!0})}});return[F,De]}export{nt as createReplTools};

package/dist/platform/tools/shot-file.d.ts

@@ -0,0 +1,7 @@
+export declare function checkShotPath(path: string, mimeType: string): void;
+export declare function saveShot(data: string, mimeType: string, path?: string): {
+    path: string;
+    host: string;
+    width?: number;
+    height?: number;
+};

package/dist/platform/tools/shot-file.js

@@ -0,0 +1 @@
+import{existsSync as g,mkdirSync as m,unlinkSync as d,writeFileSync as f}from"node:fs";import{hostname as l,tmpdir as u}from"node:os";import{dirname as w,extname as S,isAbsolute as p,join as s}from"node:path";const a=200,c={"image/jpeg":[".jpg",".jpeg"],"image/png":[".png"]};let o=0;const h=()=>s(u(),"mirasim-computer-use-shots",String(process.pid));function j(t,r){if(!p(t))throw new Error(`screenshot path must be absolute, got "${t}"`);const e=c[r]??[];if(!e.includes(S(t).toLowerCase()))throw new Error(`this surface's screenshots are ${r} \u2014 the path must end in ${e.join(" or ")}`);if(g(t))throw new Error(`${t} already exists \u2014 screenshots never overwrite; pick a new name`)}function x(t,r){if(r==="image/png"&&t.length>=24)return{width:t.readUInt32BE(16),height:t.readUInt32BE(20)};for(let e=2;e+9<t.length;){if(t[e]!==255){e+=1;continue}const n=t[e+1];if(n>=192&&n<=207&&n!==196&&n!==200&&n!==204)return{height:t.readUInt16BE(e+5),width:t.readUInt16BE(e+7)};e+=2+t.readUInt16BE(e+2)}return null}function I(t,r,e){const n=Buffer.from(t,"base64");let i=e;if(!i&&(o+=1,i=s(h(),`${String(o).padStart(4,"0")}${c[r]?.[0]??".img"}`),o>a))try{d(s(h(),`${String(o-a).padStart(4,"0")}${c[r]?.[0]??".img"}`))}catch{}return m(w(i),{recursive:!0}),f(i,n,{flag:e?"wx":"w"}),{path:i,host:l(),...x(n,r)??{}}}export{j as checkShotPath,I as saveShot};

package/dist/platform/tools/validate.d.ts

@@ -0,0 +1,2 @@
+import type { TSchema } from "typebox";
+export declare function argComplaint(schema: TSchema, label: string, params: unknown): string | null;

package/dist/platform/tools/validate.js

@@ -0,0 +1 @@
+import{Value as r}from"typebox/value";const i=3;function g(s,d,l){if(r.Check(s,l))return null;const t=new Map;for(const o of r.Errors(s,l)){const e=o,n=String(e.instancePath??""),a=t.get(n)??{where:n,message:e.message,allowed:[]};if(e.keyword==="const"&&e.params?.allowedValue!==void 0?a.allowed.push(String(e.params.allowedValue)):a.allowed.length||(a.message=e.message),t.set(n,a),t.size>i)break}const m=[...t.values()].slice(0,i).map(o=>{const e=o.allowed.length?`must be one of ${o.allowed.join(", ")}`:o.message;return o.where?`${o.where}: ${e}`:e});return`invalid arguments for ${d} \u2014 ${m.join("; ")||"they do not match the schema"}`}export{g as argComplaint};

package/dist/platform/types.d.ts

@@ -0,0 +1,220 @@
+import type { TSchema, Static } from "typebox";
+export interface PreviewTarget {
+    kind: "browser" | "electron" | "macos";
+    targetId: string;
+    title: string;
+    url: string;
+    appName: string;
+    capture: {
+        type: "cdp";
+        endpoint: string;
+        targetId: string;
+    } | {
+        type: "macos";
+        app: string;
+        pid: number;
+        windowId: number;
+    };
+    pointer?: {
+        x: number;
+        y: number;
+        kind: string;
+        ts: number;
+    };
+}
+export interface PreviewFrame {
+    mimeType: "image/png" | "image/jpeg";
+    data: string;
+    width: number;
+    height: number;
+    capturedAt: number;
+}
+export interface PreviewSink {
+    frame(frame: PreviewFrame): void;
+    status(status: {
+        unavailable: string | null;
+        icon?: string;
+    }): void;
+    gone(): void;
+}
+export interface PreviewStream {
+    tick(): Promise<void>;
+    dispose(): void;
+}
+export declare const TOOL_NAMES: readonly ["computer_repl", "computer_repl_reset"];
+export interface ToolResult {
+    content: Array<{
+        type: "text";
+        text: string;
+    } | {
+        type: "image";
+        data: string;
+        mimeType: string;
+    }>;
+    details: Record<string, unknown>;
+}
+export interface ComputerUseToolSpec<T extends TSchema = TSchema> {
+    name: string;
+    label: string;
+    description: string;
+    promptSnippet: string;
+    promptGuidelines?: string[];
+    parameters: T;
+    execute(params: Static<T>): Promise<ToolResult>;
+}
+export declare function spec<T extends TSchema>(s: ComputerUseToolSpec<T>): ComputerUseToolSpec;
+export declare class SurfaceGoneError extends Error {
+    readonly surfaceGone = true;
+}
+export declare class SurfaceUnreadableError extends Error {
+    readonly surfaceUnreadable = true;
+}
+export declare function text(t: string, details?: Record<string, unknown>): ToolResult;
+export interface TargetInfo {
+    id: string;
+    kind: "tab" | "window" | "app" | "device";
+    title: string;
+    active: boolean;
+    primary?: boolean;
+    own?: boolean;
+    unloaded?: boolean;
+}
+export interface UserConsent {
+    available(): boolean;
+    ask(message: string, meta: Record<string, unknown>): Promise<{
+        allowed: boolean;
+        persist?: "session" | "always";
+        reason?: string;
+    }>;
+    onRevoke(cb: (ids: string[]) => void): void;
+}
+export interface VerbSpec {
+    kind: string;
+    schema: TSchema;
+    readOnly?: boolean;
+}
+export interface Element {
+    ref: string;
+    role: string;
+    name: string;
+    state: string;
+    id?: string;
+    help?: string;
+    rect: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    row?: string;
+    rowBlank?: boolean;
+    coveredBy?: string;
+}
+export interface HeaderChanges {
+    target: boolean;
+    coverage: boolean;
+    omissions: boolean;
+    groups: boolean;
+    ambiguity: boolean;
+}
+export interface TargetIdentity {
+    transport: "cdp";
+    endpoint: string;
+    windowId: string;
+    application?: {
+        name: string;
+        version: string;
+    };
+}
+export interface ObservationCoverage {
+    offscreen: number | null;
+    covered: number | null;
+    capped: number | null;
+    modal?: string;
+}
+export interface Heading {
+    name: string;
+    level: number;
+    rect: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+}
+export interface Snapshot {
+    url: string;
+    title: string;
+    target?: TargetIdentity;
+    elements: Element[];
+    offscreenElements?: Element[];
+    coveredElements?: Element[];
+    groups?: {
+        label: string;
+        members: string[];
+    }[];
+    headings?: Heading[];
+    text: string;
+    meta: {
+        scrollY: number;
+        viewportH: number;
+        docH: number;
+        offscreen?: number;
+        capped?: number;
+        partial?: string;
+        rowsCut?: number;
+        sharedMarks?: string[];
+    };
+    covered?: {
+        count: number;
+        top: string;
+        modal?: string;
+    };
+}
+export interface ViewportPosition {
+    scrollY: number;
+    viewportH: number;
+    docH: number;
+    maxScrollY: number;
+    belowFold: number;
+    atBottom: boolean;
+    screen: number;
+    total: number;
+}
+export interface SnapshotCapture {
+    snapshot: Snapshot;
+    image?: {
+        bytes: Buffer;
+        mimeType: string;
+    };
+}
+export interface ShotScope {
+    ref?: string;
+    region?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    fullPage?: boolean;
+    padding?: number;
+}
+export interface ScopedShot {
+    bytes: Buffer;
+    mimeType: string;
+    note: string;
+    capture?: Record<string, unknown>;
+}
+export type Verdict = "changed" | "frozen" | "unknown";
+export interface OwnershipResource {
+    key: string;
+    mode: "shared" | "exclusive";
+}
+export interface ActResult {
+    ok: boolean;
+    note?: string;
+    journal?: Record<string, unknown>;
+    value?: unknown;
+    verified?: boolean;
+    mayHaveActed?: boolean;
+}

package/dist/platform/types.js

@@ -0,0 +1 @@
+const t=["computer_repl","computer_repl_reset"];function o(e){return e}class n extends Error{surfaceGone=!0}class c extends Error{surfaceUnreadable=!0}function u(e,r={}){return{content:[{type:"text",text:e}],details:r}}export{n as SurfaceGoneError,c as SurfaceUnreadableError,t as TOOL_NAMES,o as spec,u as text};

package/dist/platform/windows/app-gate.d.ts

@@ -0,0 +1,24 @@
+import type { UserConsent } from "../types.ts";
+export declare class WindowsApprovals {
+    readonly file: string | null;
+    constructor(env: NodeJS.ProcessEnv);
+    has(app: string): boolean;
+    list(): string[];
+    add(app: string): void;
+    remove(app: string): void;
+    private read;
+    private write;
+}
+export declare class WindowsAppGate {
+    private readonly consent;
+    private readonly approved;
+    private readonly approvedThisCall;
+    private readonly stored;
+    private readonly hostApps;
+    constructor(consent: UserConsent | null, env: NodeJS.ProcessEnv);
+    cleared(app: string): boolean;
+    endCall(): void;
+    authorize(app: string, title: string, purpose: string | null): Promise<number>;
+}
+export declare class AppRefused extends Error {
+}

package/dist/platform/windows/app-gate.js

@@ -0,0 +1,2 @@
+import{mkdirSync as l,readFileSync as h,writeFileSync as p}from"node:fs";import{dirname as c,join as u}from"node:path";import{computerUseHome as w}from"../home.js";import{approvalsAddCommand as f}from"../cli-command.js";const s=n=>n.trim().toLowerCase();class m{file;constructor(e){const t=e.COMPUTER_USE_WINDOWS_APPROVALS;this.file=t===void 0?u(w(e),"windows-approvals.json"):t.trim()||null}has(e){return this.read().includes(s(e))}list(){return this.read()}add(e){const t=this.read();t.includes(s(e))||this.write([...t,s(e)])}remove(e){const t=this.read();t.includes(s(e))&&this.write(t.filter(r=>r!==s(e)))}read(){if(!this.file)return[];try{const e=JSON.parse(h(this.file,"utf8")).approvedApps;return Array.isArray(e)?e.filter(t=>typeof t=="string").map(s):[]}catch{return[]}}write(e){if(this.file)try{l(c(this.file),{recursive:!0}),p(this.file,JSON.stringify({approvedApps:e},null,2)+`
+`)}catch{}}}class C{consent;approved=new Set;approvedThisCall=new Set;stored;hostApps;constructor(e,t){this.consent=e,this.stored=new m(t),this.hostApps=new Set((t.COMPUTER_USE_HOST_APPS??"").split(",").map(s).filter(Boolean)),e?.onRevoke(r=>{for(const i of r)this.approved.delete(s(i)),this.approvedThisCall.delete(s(i)),this.stored.remove(i)})}cleared(e){const t=s(e);return this.hostApps.has(t)?!1:this.approved.has(t)||this.approvedThisCall.has(t)||this.consent?.available()!==!0?!0:this.stored.has(t)}endCall(){this.approvedThisCall.clear()}async authorize(e,t,r){if(this.hostApps.has(s(e)))throw new a(`${e} is the app this agent runs inside; Computer Use never drives it`);if(this.cleared(e))return 0;const i=Date.now(),o=await this.consent.ask(`Allow Computer Use to see and control ${e} ("${t}")?`,{"mirasim-computer-use/windowsApp":{app:e,title:t,...r?{purpose:r}:{}},persist:["session","always"]}),d=Date.now()-i;if(!o.allowed)throw new a(`permission to use ${e} was not granted${o.reason?` \u2014 ${o.reason}`:""}. Do not retry and do not ask again in this task; continue without it, or tell the user what you could not do`+(this.stored.file!==null?` \u2014 and, if nobody was there to answer (a non-interactive run declines every question), that they can allow it ahead of time with \`${f(e)}\``:""));return o.persist?(this.approved.add(s(e)),o.persist==="always"&&this.stored.add(e)):this.approvedThisCall.add(s(e)),d}}class a extends Error{}export{a as AppRefused,C as WindowsAppGate,m as WindowsApprovals};

package/dist/platform/windows/background-verb.d.ts

@@ -0,0 +1,13 @@
+import type { ActResult } from "../types.ts";
+export interface BackgroundHost {
+    authorized(): Promise<{
+        pid: number;
+        hwnd: number;
+    }>;
+    target(ref: unknown): {
+        key: string;
+    };
+    call<T>(op: string, params?: Record<string, unknown>): Promise<T>;
+    interrupted: Set<string>;
+}
+export declare function performBackground(host: BackgroundHost, kind: string, a: Record<string, any>): Promise<ActResult>;

package/dist/platform/windows/background-verb.js

@@ -0,0 +1 @@
+import{UiaRequestError as s}from"./uia-client.js";async function u(o,n,e){if(!["set_text","click","scroll","uia_action"].includes(n)||typeof e.ref!="string"||e.ref==="window")return{ok:!1,note:"background Windows actions require a control ref: setText, supported click, uiaAction, or a ScrollPattern container. Coordinates, type, keys, hover and activation need desktop interaction with permission; nothing was sent"};if(n==="click"&&((e.button??"left")!=="left"||(e.count??1)!==1))return{ok:!1,note:"background click only supports a single left action by ref; nothing was sent"};if(n==="scroll"&&e.count!==void 0&&(!Number.isInteger(e.count)||e.count<1||e.count>20))return{ok:!1,note:"background scroll count must be an integer from 1 to 20; nothing was sent"};const i=await o.authorized(),r=`${i.pid}:${i.hwnd}`;if(o.interrupted.has(r))return{ok:!1,note:"background actions on this window were stopped after interference or an uncertain outcome. Observe the result; do not replay the action. Desktop interaction requires permission for its foreground/input effects"};const c=o.target(e.ref);try{const t=await o.call("background",{hwnd:i.hwnd,key:c.key,kind:n,...n==="set_text"?{text:String(e.text??"")}:{},...n==="uia_action"?{action:String(e.action??"")}:{},...n==="scroll"?{direction:e.direction??"down",count:e.count??1,...e.to?{to:e.to}:{}}:{}});return t.verified===!1&&n!=="set_text"?(o.interrupted.add(r),{ok:!1,mayHaveActed:!0,...t}):{ok:!0,...t}}catch(t){return t instanceof s?["background_interrupted","background_uncertain","panic"].includes(t.code)?(o.interrupted.add(r),{ok:!1,mayHaveActed:!0,note:t.message}):{ok:!1,note:t.code==="unknown_op"?"this Windows native helper does not support background UIA actions; update @mirasim/computer-use, or the app that bundles it. Nothing was sent":t.message}:(o.interrupted.add(r),{ok:!1,mayHaveActed:!0,note:`background UIA outcome is unknown: ${t.message}. Observe without replaying; background actions on this window are stopped`})}}export{u as performBackground};

package/dist/platform/windows/dates.d.ts

@@ -0,0 +1,2 @@
+export declare function parseDateTime(text: string): Record<string, number> | null;
+export declare function sameDate(shown: string, asked: string): boolean;

package/dist/platform/windows/dates.js

@@ -0,0 +1 @@
+function d(u){const s={},t=/(\d{4})\s*[-/.年]\s*(\d{1,2})\s*[-/.月]\s*(\d{1,2})\s*日?/.exec(u);if(t){const[e,i,o]=[Number(t[1]),Number(t[2]),Number(t[3])],c=new Date(e,i-1,o);if(c.getMonth()!==i-1||c.getDate()!==o)return null;Object.assign(s,{year:e,month:i,day:o})}const r=t?u.slice(t.index+t[0].length):u,n=/(\d{1,2})\s*[:：]\s*(\d{2})(?:\s*[:：]\s*(\d{2}))?/.exec(r);if(n){let e=Number(n[1]);const i=/下午|晚上|\bpm\b|p\.m\./i.test(r),o=/上午|凌晨|早上|\bam\b|a\.m\./i.test(r);i&&e<12&&(e+=12),o&&e===12&&(e=0);const[c,m]=[Number(n[2]),n[3]===void 0?void 0:Number(n[3])];if(e>23||c>59||(m??0)>59)return null;Object.assign(s,{hour:e,minute:c,...m===void 0?{}:{second:m}})}return Object.keys(s).length?s:null}function b(u,s){const t=d(s),r=d(u);return!!t&&!!r&&Object.entries(t).every(([n,e])=>r[n]===e)}export{d as parseDateTime,b as sameDate};

package/dist/platform/windows/driver-apps.d.ts

@@ -0,0 +1,9 @@
+import type { ActResult, TargetInfo } from "../types.ts";
+import { WindowsDriverYield } from "./driver-yield.ts";
+export declare abstract class WindowsDriverApps extends WindowsDriverYield {
+    private apps;
+    private startMenu;
+    protected listApps(query: string): Promise<ActResult>;
+    unlistedTarget(query: string): Promise<TargetInfo | null>;
+    protected launchAndBind(app: string, handle?: string): Promise<ActResult>;
+}

package/dist/platform/windows/driver-apps.js

@@ -0,0 +1 @@
+import{WindowsDriverYield as c}from"./driver-yield.js";import{filterApps as f,LAUNCH_NEEDS_DESKTOP as m,LAUNCH_WAIT_MS as u,launchedBy as g,pickAppeared as A,startAppTarget as k}from"./launch.js";import{UiaRequestError as w}from"./uia-client.js";class v extends c{apps=null;async startMenu(){if(this.apps&&Date.now()-this.apps.at<6e4)return this.apps.list;const t=(await this.call("apps")).apps;return this.apps={at:Date.now(),list:t},t}async listApps(t){let i;try{i=await this.startMenu()}catch(n){if(n instanceof w&&n.code==="unknown_op")return{ok:!1,note:"this Windows native helper predates listApps; update @mirasim/computer-use, or the app that bundles it"};throw n}return f(i,t)}async unlistedTarget(t){return t.trim()?k(await this.startMenu().catch(()=>[]),t):null}async launchAndBind(t,i){if(!t.trim())return{ok:!1,note:"launch_app needs an app: a Start-menu name or id from listApps(), or a program name (notepad, mspaint)"};if(this.interactionMode!=="desktop")return{ok:!1,note:m};const n=await this.yieldToHuman("launch_app",t);if(n)return n;const l=new Set((await this.windows()).map(a=>a.hwnd));let s;try{s=await this.call("launch",{app:t})}catch(a){if(!(a instanceof w))throw a;return{ok:!1,note:a.code==="unknown_op"?"this Windows native helper predates launch_app; update @mirasim/computer-use, or the app that bundles it. Nothing was started":a.message}}const p=g(t,s);let e,d=!0;for(const a=Date.now()+u;!e&&Date.now()<a;){const r=(await this.windows()).filter(o=>!l.has(o.hwnd));e=A(r,p),e||await new Promise(o=>setTimeout(o,250))}if(!e){const a=await this.call("foreground");e=(await this.windows()).find(r=>r.hwnd===a.hwnd&&p(r)),d=!1}if(!e)return{ok:!1,mayHaveActed:!0,note:`started ${s.name}, but no window of it appeared within ${u/1e3} s \u2014 it may still be loading, be waiting behind a prompt, or run without a window. Do not start it again: computer.surfaces() lists the windows`};const h=await this.focusTarget(String(e.hwnd));return h.ok?(this.leftFront={pid:e.pid,at:Date.now()},this.launched={id:i??`app:${t}`,hwnd:e.hwnd},{ok:!0,note:`${d?"started":"brought forward the running"} ${s.name} \u2014 its window "${e.title}" (${e.process}) is now the target, in front of the user; observe() to see it`}):{...h,mayHaveActed:!0}}}export{v as WindowsDriverApps};

package/dist/platform/windows/driver-read.d.ts

@@ -0,0 +1,12 @@
+import type { ScopedShot, ShotScope, Snapshot } from "../types.ts";
+import { WindowsDriverTargets } from "./driver-targets.ts";
+export declare function fieldText(s: string): string;
+export declare abstract class WindowsDriverRead extends WindowsDriverTargets {
+    private walk;
+    protected snapshot(): Promise<Snapshot>;
+    screenshot(): Promise<Buffer>;
+    screenshotScoped(scope: ShotScope): Promise<ScopedShot>;
+    readText(): Promise<string>;
+    readValue(ref: string): Promise<string>;
+    valueMatches(actual: string, expected: string): boolean;
+}

package/dist/platform/windows/driver-read.js

@@ -0,0 +1,3 @@
+import{SurfaceUnreadableError as d}from"../types.js";import{WindowsDriverTargets as w}from"./driver-targets.js";import{buildElements as f,windowText as c}from"./elements.js";import{UiaRequestError as l}from"./uia-client.js";function g(h){return h.replace(/\r\n?/g,`
+`)}class y extends w{async walk(){const e=await this.authorized(),t=await this.call("tree",{hwnd:e.hwnd,max:3e3});return this.frame=t.frame,{row:e,tree:t}}async snapshot(){const{row:e,tree:t}=await this.walk(),{elements:n,offscreen:i,refs:r,capped:o}=f(t);this.refTargets=r,e.minimized?this.pendingNote=`"${e.title}" is minimized \u2014 desktop actions restore it; background actions never do. Some apps (UWP/XAML) expose no usable controls while minimized`:t.dialogs?.length?this.pendingNote=`a dialog is open over "${e.title}" (${t.dialogs.map(s=>`"${s}"`).join(", ")}) \u2014 the window waits on it; its controls are listed last`:t.popups&&(this.pendingNote=`a menu or popup of "${e.title}" is open \u2014 its items are listed last; it closes on a pick, on escape or on a click elsewhere`);const a=n.filter(s=>s.role==="text").map(s=>s.name).join(" \xB7 ");return{url:`windows:${e.process}:${e.hwnd}`,title:t.title||e.title,elements:n,offscreenElements:i,text:a.slice(0,2e3),meta:{scrollY:0,viewportH:t.frame.h,docH:t.frame.h,offscreen:i.length,...o?{capped:o}:{},...t.partial?{partial:t.partial}:{}}}}async screenshot(){const e=await this.authorized();try{const t=await this.call("shot",{hwnd:e.hwnd});return this.frame={x:t.x,y:t.y,w:t.width,h:t.height},this.fullShotFrame=this.frame,Buffer.from(t.png,"base64")}catch(t){throw t instanceof l&&t.code==="minimized"?new d(`"${e.title}" is minimized, so it has no pixels to capture until it is restored (activate)`):t}}async screenshotScoped(e){if(e.fullPage)throw new Error("fullPage screenshots are not supported on Windows; use ref or region for window detail");if(!!e.ref==!!e.region)throw new Error("pick exactly one of ref or region for a Windows scoped screenshot");if(e.padding!==void 0&&(!e.ref||!Number.isFinite(e.padding)||e.padding<0))throw new Error("padding needs a finite number of pixels, 0 or more, with ref");if(e.region&&(![e.region.x,e.region.y,e.region.width,e.region.height].every(Number.isFinite)||!(e.region.width>0)||!(e.region.height>0)))throw new Error("region needs finite x/y and positive width/height");if(e.region&&!this.fullShotFrame)throw new Error("region coordinates use the full screenshot; take w.screenshot() or w.observe({screenshot:true}) first");const t=await this.authorized(),n=e.ref?this.target(e.ref).key:void 0;let i;try{i=await this.call("shotscoped",{hwnd:t.hwnd,...n?{key:n,padding:e.padding??0}:{region:e.region,frame:this.fullShotFrame}})}catch(a){throw a instanceof l&&a.code==="minimized"?new d(`"${t.title}" is minimized, so it has no pixels to capture until it is restored (activate)`):a}const r=this.fullShotFrame,o=!!r&&r.x===i.x&&r.y===i.y&&r.w===i.width&&r.h===i.height;return{bytes:Buffer.from(i.png,"base64"),mimeType:"image/png",note:`${e.ref?`element ${e.ref}`:"region"}, ${i.pixelWidth}\xD7${i.pixelHeight} pixels at (${i.region.x},${i.region.y}) in window ${t.hwnd}${i.trimmed?" \u2014 trimmed to the window":""}`+(r&&!o?" \u2014 window geometry changed since the full screenshot; take a new full screenshot before point actions":""),capture:{target:{pid:t.pid,windowId:t.hwnd},window:{x:i.x,y:i.y,width:i.width,height:i.height},region:i.region,units:"window-pixels",pointsPerPixel:{x:1,y:1},toFullScreenshot:o?{origin:{x:i.region.x,y:i.region.y},scale:{x:1,y:1}}:null,captureBackend:i.method}}}async readText(){return c((await this.walk()).tree)}async readValue(e){return g((await this.call("value",{key:this.target(e).key})).value)}valueMatches(e,t){return e===t.replace(/\r\n?/g,`
+`)}}export{y as WindowsDriverRead,g as fieldText};

package/dist/platform/windows/driver-targets.d.ts

@@ -0,0 +1,58 @@
+import { ComputerUseDriver } from "../driver.ts";
+import type { ActResult, OwnershipResource, TargetInfo } from "../types.ts";
+import { type WindowsAppGate } from "./app-gate.ts";
+import type { RefTarget } from "./elements.ts";
+import type { UiaTransport } from "./uia-client.ts";
+export interface WindowRow {
+    hwnd: number;
+    pid: number;
+    process: string;
+    title: string;
+    class: string;
+    rect: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    minimized: boolean;
+    foreground: boolean;
+}
+export type Frame = {
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+};
+export declare abstract class WindowsDriverTargets extends ComputerUseDriver {
+    protected abstract readonly uia: UiaTransport;
+    protected abstract readonly gate: WindowsAppGate;
+    protected hwnd: number | null;
+    protected row: WindowRow | null;
+    protected frame: Frame | null;
+    protected fullShotFrame: Frame | null;
+    protected refTargets: Map<string, RefTarget>;
+    protected pendingNote: string | null;
+    protected leftFront: {
+        pid: number;
+        at: number;
+    } | null;
+    protected launched: {
+        id: string;
+        hwnd: number;
+    } | null;
+    protected abstract launchAndBind(app: string, handle?: string): Promise<ActResult>;
+    protected takePendingNote(): string | null;
+    protected call<T>(op: string, params?: Record<string, unknown>): Promise<T>;
+    protected windows(): Promise<WindowRow[]>;
+    private ensure;
+    protected authorized(): Promise<WindowRow>;
+    endCall(): void;
+    private bind;
+    listTargets(): Promise<TargetInfo[]>;
+    focusTarget(id: string): Promise<ActResult>;
+    ownershipResources(): Promise<OwnershipResource[]>;
+    currentUrl(): Promise<string>;
+    isActiveTarget(id: string): Promise<boolean>;
+    protected target(ref: unknown): RefTarget;
+}

package/dist/platform/windows/driver-targets.js

@@ -0,0 +1 @@
+import{ComputerUseDriver as i}from"../driver.js";import{SurfaceGoneError as n,SurfaceUnreadableError as r}from"../types.js";import{AppRefused as o}from"./app-gate.js";class d extends i{hwnd=null;row=null;frame=null;fullShotFrame=null;refTargets=new Map;pendingNote=null;leftFront=null;launched=null;takePendingNote(){const t=this.pendingNote;return this.pendingNote=null,t}call(t,e={}){return this.uia.request(t,e)}async windows(){return(await this.call("windows")).windows}async ensure(){if(this.hwnd===null)throw new Error("mirasim-cu: no window is bound, and picking one is the caller's call, not this platform's. Take one with computer.surface('<title or program name>'); computer.surfaces() lists them.");const e=(await this.windows()).find(s=>s.hwnd===this.hwnd);if(!e)throw new n(`the window "${this.row?.title??this.hwnd}" was closed`);return this.row=e,e}async authorized(){const t=await this.ensure();if(t.minimized&&t.process.toLowerCase()==="applicationframehost")throw new r("This minimized app no longer exposes its UIA app identity. Restore it and bind again; no input was sent. The shared ApplicationFrameHost is not an app-access substitute");if(!this.gate.cleared(t.process||"?"))throw this.permissionRefusal=`mirasim-cu: ${t.process||"this app"} is not allowed for this session (any more) \u2014 bind it again with computer.surface(\u2026) to ask the user, or continue without it`,new Error(this.permissionRefusal);return t}endCall(){this.gate.endCall()}bind(t){this.hwnd!==t.hwnd&&(this.hwnd=t.hwnd,this.row=t,this.frame=null,this.fullShotFrame=null,this.refTargets.clear(),this.leftFront=null,this.resetObservationState(),this.pendingSurfaceChange=!0)}async listTargets(){return(await this.windows()).map(t=>({id:String(t.hwnd),kind:"window",title:this.gate.cleared(t.process||"?")?`${t.title} \u2014 ${t.process||"?"}${t.minimized?" (minimized)":""}`:`${t.process||"?"} \u2014 not cleared in this session yet; binding it checks the user's allow-list or asks them`,active:t.hwnd===this.hwnd,own:!1}))}async focusTarget(t){if(t.startsWith("app:")){const s=await this.launchAndBind(t.slice(4),t);if(!s.ok)throw new Error(`mirasim-cu: ${s.note}`);return s}const e=(await this.windows()).find(s=>String(s.hwnd)===t.trim());if(!e)return{ok:!1,note:`no window ${t} \u2014 list them with computer.surfaces()`};if(e.minimized&&e.process.toLowerCase()==="applicationframehost")return{ok:!1,note:"This minimized app no longer exposes its UIA app identity. Restore it and bind again; no input was sent"};try{this.waitedOnUserMs+=await this.gate.authorize(e.process||"?",e.title,this.consentPurpose)}catch(s){throw s instanceof o?(this.permissionRefusal=`mirasim-cu: ${s.message}`,new Error(this.permissionRefusal)):s}return this.bind(e),{ok:!0,note:`bound to "${e.title}" (${e.process}); reads and supported background UIA actions do not activate it; desktop actions do`}}async ownershipResources(){return[{key:`windows:window:${(await this.ensure()).hwnd}`,mode:"exclusive"}]}async currentUrl(){return this.row?`windows:${this.row.process}:${this.row.hwnd}`:""}async isActiveTarget(t){return t.startsWith("app:")?!!this.launched&&this.launched.id===t&&this.launched.hwnd===this.hwnd&&(!this.row||this.gate.cleared(this.row.process||"?")):await super.isActiveTarget(t)&&(!this.row||this.gate.cleared(this.row.process||"?"))}target(t){const e=typeof t=="string"?this.refTargets.get(t):void 0;if(!e)throw new Error(`${String(t)} is not in the latest observation of this window \u2014 observe again`);return e}}export{d as WindowsDriverTargets};

package/dist/platform/windows/driver-yield.d.ts

@@ -0,0 +1,10 @@
+import type { ActResult } from "../types.ts";
+import { WindowsDriverRead } from "./driver-read.ts";
+import type { WindowRow } from "./driver-targets.ts";
+export declare abstract class WindowsDriverYield extends WindowsDriverRead {
+    private humanIdleSeconds;
+    protected takenOver(row: WindowRow): Promise<ActResult | null>;
+    protected yieldToHuman(kind: string, title: string): Promise<ActResult | null>;
+    protected activate(row: WindowRow): Promise<ActResult>;
+    private frontRefusal;
+}

package/dist/platform/windows/driver-yield.js

@@ -0,0 +1 @@
+import{WindowsDriverRead as a}from"./driver-read.js";import{UiaRequestError as s}from"./uia-client.js";const r=2e3,l=1500,d=new Set(["pickerhost","credentialuibroker","consent","logonui","lockapp"]);class w extends a{async humanIdleSeconds(){try{const e=await this.call("humanidle");return typeof e.listening=="boolean"?e.seconds:null}catch(e){if(e instanceof s)return null;throw e}}async takenOver(e){const t=this.leftFront;if(!t)return null;const n=await this.humanIdleSeconds();if(n===null||n*1e3>=Date.now()-t.at)return null;const o=await this.call("foreground");return o.pid===t.pid?null:{ok:!1,note:`the user switched to ${o.process||"another window"} since your last action on "${e.title}" \u2014 they have taken over the screen. Nothing was sent and the window was not brought back. Tell them what is left and wait; once they agree, activate it deliberately (kind:'activate') and continue`}}async yieldToHuman(e,t){const n=Date.now()+l;for(;;){const o=await this.humanIdleSeconds();if(o===null||o*1e3>=r)return null;if(Date.now()>=n)return{ok:!1,note:`${e} was NOT sent: the user touched the keyboard or pointer ${o.toFixed(1)} s ago, and acting now would bring "${t}" to the front under them \u2014 their next keystrokes would land in it. Nothing was sent. Wait and retry; it goes through once they pause for ${r/1e3} s`};await new Promise(i=>setTimeout(i,150))}}async activate(e){const t=await this.call("activate",{hwnd:e.hwnd});return t.foreground?{ok:!0,note:t.method==="already"?`"${e.title}" was already in front`:`brought "${e.title}" to the front${t.restored?" (restored from minimized)":""} \u2014 the user's keyboard now goes to it`,journal:{activation:t.method}}:{ok:!1,note:await this.frontRefusal(e)}}async frontRefusal(e){const t=await this.call("foreground").catch(()=>null),n=`Windows would not bring "${e.title}" to the front; nothing was sent`;return!t||!t.hwnd?`${n} \u2014 no window holds the foreground (the screen may be locked or the desktop switching). Wait, or ask the user to unlock`:d.has(t.process.toLowerCase())?`${n} \u2014 a Windows security prompt (${t.process}${t.title?`: "${t.title}"`:""}) is in front, and Windows lets no other window take the foreground until the user answers it. Do not retry or try to close it: tell the user it is waiting for them`:`${n} \u2014 ${t.process||"another app"} is in front and Windows kept it there (the user may be working in it). Try again, or ask the user to switch to it`}}export{w as WindowsDriverYield};

package/dist/platform/windows/driver.d.ts

@@ -0,0 +1,39 @@
+import type { DiagnosticsAbout } from "../driver-diagnostics.ts";
+import type { ActResult, OwnershipResource, UserConsent, VerbSpec } from "../types.ts";
+import { WindowsAppGate } from "./app-gate.ts";
+import { WindowsDriverApps } from "./driver-apps.ts";
+import { type UiaTransport } from "./uia-client.ts";
+export interface WindowsDriverOptions {
+    interaction?: "background" | "desktop";
+    env?: NodeJS.ProcessEnv;
+    consent?: UserConsent;
+    transport?: UiaTransport;
+}
+export declare class WindowsDriver extends WindowsDriverApps {
+    readonly platform = "windows";
+    readonly screenshotMimeType: string;
+    readonly supportsDiagnostics = true;
+    readonly readsOffscreen = true;
+    protected readonly uia: UiaTransport;
+    protected readonly gate: WindowsAppGate;
+    private readonly interruptedBackground;
+    constructor(opts?: WindowsDriverOptions);
+    get diagnosticsAbout(): DiagnosticsAbout;
+    promptFragment(): string;
+    interactionNotes(): readonly string[];
+    extraVerbs(): VerbSpec[];
+    actionResources(kind: string): Promise<OwnershipResource[]>;
+    perform(kind: string, a: Record<string, any>): Promise<ActResult>;
+    private menuHost;
+    private readonly textHost;
+    private performBackground;
+    private dragPoint;
+    private click;
+    private setText;
+    private verifyText;
+    private scroll;
+    private center;
+    private windowCenter;
+    private toScreen;
+    close(): Promise<void>;
+}

package/dist/platform/windows/driver.js

@@ -0,0 +1,2 @@
+import{WindowsAppGate as p}from"./app-gate.js";import{performBackground as w}from"./background-verb.js";import{paste as m,selectText as g,setDate as y}from"./text-verbs.js";import{sameDate as v}from"./dates.js";import{menuVerb as u}from"./menu-verb.js";import{WindowsDriverApps as k}from"./driver-apps.js";import{fieldText as l}from"./driver-read.js";import{promptFragmentFor as b,WINDOWS_INTERACTION_NOTES as x}from"./prompt-fragment.js";import{UiaHelperProcess as $,UiaRequestError as s}from"./uia-client.js";import{LAUNCH_NEEDS_DESKTOP as _}from"./launch.js";import{WINDOWS_VERBS as S}from"./verbs.js";function N(c,t){return l(c)===t.replace(/\r\n?/g,`
+`)}const d=new Set(["click","hover","set_text","type","keys","scroll","activate","uia_action","drag","launch_app","select_text","paste","menu"]),T=5,f=2500;class B extends k{platform="windows";screenshotMimeType="image/png";supportsDiagnostics=!0;readsOffscreen=!0;uia;gate;interruptedBackground=new Set;constructor(t={}){super(),this.interactionMode=t.interaction??"background",this.gate=new p(t.consent??null,t.env??process.env),this.uia=t.transport??new $({env:t.env??process.env,onLog:e=>this.diagnosticsSink?.push("helper",e),onStart:async e=>{await this.onExecutorStart?.(e)}}),this.uia.onReset=()=>{this.refTargets.clear(),this.fullShotFrame=null,this.resetObservationState()}}get diagnosticsAbout(){return{empty:"complaints from the UI Automation helper (this platform's diagnostics are the helper's own log, never the app's)",header:"the Windows UI Automation helper's own log \u2014 what went wrong inside the platform, not inside the app"}}promptFragment(){return b()}interactionNotes(){return x}extraVerbs(){return S}async actionResources(t){return this.interactionMode==="desktop"&&d.has(t)?[{key:"windows:desktop-input",mode:"exclusive"}]:[]}async perform(t,e){if(t==="list_apps")return this.listApps(typeof e.query=="string"?e.query:"");if(!d.has(t))return{ok:!1,note:`unknown action kind ${t}`};if(this.interactionMode==="background")return t==="launch_app"?{ok:!1,note:_}:t==="drag"?{ok:!1,note:"drag holds the user's real mouse button and moves their pointer, so it needs interaction:'desktop'; explain that and obtain permission first. Nothing was sent"}:t==="menu"?(await this.authorized(),u(this.menuHost(!1),e.path)):t==="paste"||t==="select_text"?{ok:!1,note:`${t} moves the window's keyboard focus${t==="paste"?" and borrows the user's clipboard":""}, so it needs interaction:'desktop'; explain that and obtain permission first. Nothing was sent`}:this.performBackground(t,e);if(t==="launch_app")return this.launchAndBind(String(e.app??""));const o=await this.authorized(),n=(t==="activate"?null:await this.takenOver(o))??await this.yieldToHuman(t,o.title);if(n)return n;const i=await this.activate(o);if(!i.ok||(this.leftFront={pid:o.pid,at:Date.now()},t==="activate"))return i;try{switch(t){case"click":return await this.click(e);case"hover":{const r=e.ref?this.center(this.target(e.ref)):this.toScreen(e.x,e.y);return await this.call("move",r),{ok:!0,note:`moved the real pointer to ${r.x},${r.y} on screen and left it there`}}case"set_text":return await this.setText(e.ref,String(e.text??""));case"type":{const r=await this.call("type",{hwnd:o.hwnd,text:String(e.text??""),verifyMs:f});return{ok:!0,...typeof r.verified=="boolean"?{verified:r.verified}:{},note:r.note??"typed into the focused control; read-back unavailable \u2014 the value was not verified"}}case"keys":return await this.call("key",{chord:String(e.chord??""),count:Number(e.count??1)||1}),{ok:!0};case"scroll":return await this.scroll(e);case"menu":return await u(this.menuHost(!0),e.path);case"select_text":return await g(this.textHost,e);case"paste":return await m(this.textHost,e);case"drag":{const r=this.dragPoint(e.ref,e.x,e.y,"start"),a=this.dragPoint(e.toRef,e.toX,e.toY,"destination");if("ok"in r)return r;if("ok"in a)return a;const h=await this.call("drag",{x:r.x,y:r.y,toX:a.x,toY:a.y,steps:Number(e.steps??20)||20});return{ok:!0,journal:{via:"mouse",steps:h.steps},note:`dragged ${r.what} to ${a.what} with the real pointer (left button held over ${h.steps} moves, then released) \u2014 observe for the result`}}case"uia_action":{const r=await this.call("pattern",{key:this.target(e.ref).key,action:String(e.action??"")});return{ok:!0,journal:{via:r.via,...r.pending?{pending:!0}:{}},note:r.pending?`${r.via} sent; the app is still handling it \u2014 typically a dialog it opened. Observe to see it; do not repeat it`:`${r.via} through UI Automation \u2014 observe for its effect`}}}}catch(r){if(r instanceof s&&r.code==="stale_key")return{ok:!1,note:r.message};if(r instanceof s&&(r.code==="no_action"||r.code==="bad_request"))return{ok:!1,note:`${r.message}; nothing was sent`};if(r instanceof s&&r.code==="unknown_op")return{ok:!1,note:`this Windows native helper predates ${t}; update @mirasim/computer-use, or the app that bundles it. Nothing was sent`};if(r instanceof s&&r.code==="human_input")return{ok:!1,note:`${r.message}. The rest was NOT sent. Do not resend it blindly: observe what landed, and let the user finish \u2014 the next action waits until they pause`};throw r}return{ok:!1,note:`unknown action kind ${t}`}}menuHost(t){return{call:(e,o)=>this.call(e,o),hwnd:()=>this.hwnd,canOpen:t}}textHost={target:t=>this.target(t),call:(t,e)=>this.call(t,e),hwnd:()=>this.hwnd};async performBackground(t,e){return w({authorized:()=>this.authorized(),target:o=>this.target(o),call:(o,n)=>this.call(o,n),interrupted:this.interruptedBackground},t,e)}dragPoint(t,e,o,n){if(typeof e=="number"&&typeof o=="number")return{...this.toScreen(e,o),what:`${e},${o}`};if(typeof t!="string")return{ok:!1,note:`drag needs a ${n}: a ref, or x and y from the latest screenshot. Nothing was sent`};const i=this.target(t);return{...this.center(i),what:`${i.type.toLowerCase()} "${i.name}"`}}async click(t){const e=String(t.button??"left"),o=Math.max(1,Math.min(3,Number(t.count??1)||1));let n;if(t.ref){const i=this.target(t.ref);if(e==="left"&&o===1)try{const r=await this.call("press",{key:i.key});return r.pending?{ok:!0,journal:{via:r.via,pending:!0},note:`pressed through UI Automation (${r.via}); the app is still handling it \u2014 typically a dialog it opened is waiting for an answer. Observe to see it; do not press again`}:{ok:!0,note:`pressed through UI Automation (${r.via})`,journal:{via:r.via}}}catch(r){if(!(r instanceof s)||r.code!=="no_action"&&r.code!=="prefer_mouse")throw r}n=this.center(i)}else n=this.toScreen(t.x,t.y);return await this.call("click",{...n,button:e,count:o}),{ok:!0,note:`real ${e} click${o>1?` \xD7${o}`:""} at ${n.x},${n.y} on screen \u2014 the user's pointer moved there`,journal:{via:"mouse"}}}async setText(t,e){const o=this.target(t);if(o.datetime)return y(this.textHost,o,e);try{return await this.call("setvalue",{key:o.key,text:e}),{ok:!0,...await this.verifyText(o.key,e)}}catch(n){if(!(n instanceof s)||n.code!=="no_value"&&n.code!=="read_only")throw n}return await this.call("focus",{key:o.key}).catch(()=>this.call("click",{...this.center(o),button:"left",count:1})),await this.call("key",{chord:"ctrl+a"}),await this.call("type",{text:e}),{ok:!0,...await this.verifyText(o.key,e)}}async verifyText(t,e){const o=Date.now()+f;for(;;){let n;try{n=(await this.call("value",{key:t})).value}catch{return{note:"write dispatched; the original field could not be read \u2014 the value was not verified"}}if(N(n,e))return{verified:!0,note:"the original field reads back the requested text"};if(v(n,e))return{verified:!0,note:`the original field reads back the requested date as "${l(n)}"`};if(Date.now()>=o)return{verified:!1,note:"the original field did not retain the requested text before verification timed out; do not automatically replay input"};await new Promise(i=>setTimeout(i,Math.min(50,o-Date.now())))}}async scroll(t){const e=t.ref&&t.ref!=="window"?this.center(this.target(t.ref)):this.windowCenter(),o=String(t.direction??"down"),n=t.to?60:Math.max(1,Number(t.count??1)||1)*T,i=t.to?t.to==="bottom":o==="down",r=!t.to&&(o==="left"||o==="right");return await this.call("scroll",r?{...e,dx:o==="right"?n:-n}:{...e,dy:i?n:-n}),{ok:!0,note:`${n} wheel notches ${t.to?`toward the ${t.to}`:o} at ${e.x},${e.y} \u2014 observe to see how far it moved`,journal:{notches:n}}}center(t){return{x:Math.round(t.rect.x+t.rect.w/2),y:Math.round(t.rect.y+t.rect.h/2)}}windowCenter(){const t=this.frame??this.row.rect;return{x:Math.round(t.x+t.w/2),y:Math.round(t.y+t.h/2)}}toScreen(t,e){const o=this.frame??this.row?.rect;if(!o)throw new Error("no frame for this window yet \u2014 observe it first");return{x:Math.round(o.x+Number(t)),y:Math.round(o.y+Number(e))}}async close(){try{await this.uia.close()}finally{await this.disconnected()}}}export{B as WindowsDriver};

package/dist/platform/windows/elements.d.ts

@@ -0,0 +1,66 @@
+import type { Element } from "../types.ts";
+interface UiaNode {
+    key: string;
+    type: string;
+    name: string;
+    rect: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    depth: number;
+    actions: string[];
+    id?: string;
+    disabled?: boolean;
+    offscreen?: boolean;
+    focused?: boolean;
+    value?: string;
+    readonly?: boolean;
+    toggle?: "on" | "off" | "mixed";
+    expanded?: "collapsed" | "expanded" | "partial" | "leaf";
+    selected?: boolean;
+    range?: number;
+    defaultAction?: string;
+    help?: string;
+    datetime?: {
+        shown: string;
+        iso?: string;
+        none?: boolean;
+        calendar?: boolean;
+    };
+}
+export interface UiaTree {
+    title: string;
+    frame: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    elements: UiaNode[];
+    dialogs: string[];
+    popups?: number;
+    partial: string | null;
+}
+export interface RefTarget {
+    key: string;
+    type: string;
+    name: string;
+    actions: string[];
+    rect: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    datetime?: boolean;
+}
+export declare function buildElements(tree: UiaTree): {
+    elements: Element[];
+    offscreen: Element[];
+    refs: Map<string, RefTarget>;
+    capped: number;
+};
+export declare function windowText(tree: UiaTree): string;
+export {};

package/dist/platform/windows/elements.js

@@ -0,0 +1,2 @@
+import{parseDateTime as m}from"./dates.js";const p={Button:"button",SplitButton:"button",CheckBox:"checkbox",RadioButton:"radio",ComboBox:"combobox",Edit:"textbox",Document:"document",Hyperlink:"link",Image:"image",ListItem:"listitem",List:"list",Menu:"menu",MenuBar:"menubar",MenuItem:"menuitem",ProgressBar:"progressbar",ScrollBar:"scrollbar",Slider:"slider",Spinner:"spinbutton",StatusBar:"status",Tab:"tablist",TabItem:"tab",Text:"text",ToolBar:"toolbar",AppBar:"toolbar",ToolTip:"tooltip",Tree:"tree",TreeItem:"treeitem",DataGrid:"grid",DataItem:"row",Table:"table",Header:"header",HeaderItem:"columnheader",Group:"group",Pane:"pane",Window:"window",TitleBar:"titlebar",Thumb:"thumb",Calendar:"calendar",Custom:"custom"},f=new Set(["Button","SplitButton","CheckBox","RadioButton","ComboBox","Edit","Hyperlink","ListItem","MenuItem","Slider","Spinner","TabItem","TreeItem","DataItem","HeaderItem"]),h=new Set(["invoke","toggle","select","expand","value","range"]),b=400;function y(e){if(e.type==="Window"&&e.depth===0||e.type==="TitleBar")return!1;if(e.datetime||e.actions.includes("scroll")||f.has(e.type))return!0;const a=e.actions.some(o=>h.has(o));return(e.type==="Group"||e.type==="Pane"||e.type==="Custom"||e.type==="Document")&&!e.name?!1:a?!0:(e.type==="Text"||e.type==="Image"||e.type==="Hyperlink")&&e.name.trim()!==""}function x(e){return e.type==="Custom"&&e.actions.includes("expand")&&e.actions.includes("value")&&!!e.value&&!!m(e.value)}function g(e){const a=[];if(e.value!==void 0&&e.actions.includes("value")&&!e.readonly){const o=r(e.value,80);a.push(o?`filled:${o}`:"empty")}if(e.toggle&&a.push(e.type==="CheckBox"||e.type==="RadioButton"?e.toggle==="on"?"checked":e.toggle==="mixed"?"mixed":"unchecked":e.toggle==="on"?"pressed":""),(e.expanded==="expanded"||e.expanded==="collapsed")&&a.push(e.expanded),e.selected&&a.push("selected"),e.range!==void 0&&a.push(`value:${e.range}`),e.datetime){const o=e.datetime;a.push(o.none?"empty (its checkbox is off)":o.calendar?`selected:${o.iso??"none"}`:`filled:${r(o.shown,40)}${o.iso?` (${o.iso})`:""}`)}return e.focused&&a.push("focused"),e.disabled&&a.push("disabled"),a.filter(Boolean).join(" ")}function r(e,a){const o=e.replace(/\s+/g," ").trim();return o.length>a?`${o.slice(0,a-1)}\u2026`:o}function T(e){const a=new Map,o=[],i=[];let n=0,u=0;const s=e.frame;for(const t of e.elements){if(!y(t))continue;const c=!t.offscreen&&t.rect.w>0&&t.rect.h>0;if(c&&o.length>=b){n++;continue}const l=`e${++u}`,d={ref:l,role:t.datetime?t.datetime.calendar?"calendar":"datetime":x(t)?"datetime":p[t.type]??t.type.toLowerCase(),name:r(t.name,160),state:g(t),rect:{x:t.rect.x-s.x,y:t.rect.y-s.y,w:t.rect.w,h:t.rect.h},...t.id&&t.id!==t.name&&!/^\d+$/.test(t.id)?{id:t.id}:{},...t.help&&t.help!==t.name?{help:r(t.help,120)}:{}};a.set(l,{key:t.key,type:t.type,name:t.name,actions:t.actions,rect:t.rect,...t.datetime&&!t.datetime.calendar?{datetime:!0}:{}}),(c?o:i).push(d)}return{elements:o,offscreen:i,refs:a,capped:n}}function I(e){const a=[];for(const o of e.elements)for(const i of[o.name,o.actions.includes("value")?o.value:o.datetime?.shown]){const n=i?.trim();n&&a[a.length-1]!==n&&a.push(n)}return a.join(`
+`)}export{T as buildElements,I as windowText};