@mirasim/computer-use 0.0.0-stage → 0.1.0-rc.3

package/dist/platform/macos/driver-read.d.ts

@@ -0,0 +1,47 @@
+import { type ScopedShot, type ShotScope, type Snapshot } from "../types.ts";
+import { MacosDriverApps } from "./driver-apps.ts";
+import type { ElementActs } from "./act-element.ts";
+import { type AxTree, type AxWindow, type MappedTree } from "./elements.ts";
+export declare abstract class MacosDriverRead extends MacosDriverApps {
+    protected abstract readonly acts: ElementActs;
+    protected abstract appNoteThisRound: string | null;
+    protected abstract readonly toldAbout: Set<string>;
+    protected frame: AxWindow | null;
+    protected rects: Map<string, {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    }>;
+    protected offscreenRefs: Set<string>;
+    protected shotScale: number;
+    protected shotTaken: boolean;
+    protected refsByWindow: Map<string, Map<string, string>>;
+    private static readonly MAX_WINDOWS_REMEMBERED;
+    private static readonly MAX_ABSENT_REFS;
+    protected refCount: number;
+    protected appWindowIds: {
+        pid: number;
+        ids: Set<number>;
+        unnumbered: number;
+    } | null;
+    protected walk(maxText?: number): Promise<{
+        tree: AxTree;
+        mapped: MappedTree;
+    }>;
+    private lastWalkBare;
+    protected treeBlind(): boolean;
+    protected lastWalkHadWeb(): boolean;
+    private walkHadWeb;
+    protected takeBareWindow(): boolean;
+    private saidSelfDrawn;
+    private chromeOnly;
+    protected windowUnreadable(locked: boolean): string;
+    protected snapshot(): Promise<Snapshot>;
+    protected noteWindowChanges(pid: number, reading: number | null): Promise<void>;
+    private sayNoWindow;
+    screenshot(): Promise<Buffer>;
+    screenshotScoped(scope: ShotScope): Promise<ScopedShot>;
+    readText(): Promise<string>;
+    readValue(ref: string): Promise<string>;
+}

package/dist/platform/macos/driver-read.js

@@ -0,0 +1 @@
+import{SurfaceUnreadableError as E}from"../types.js";import{MacosDriverApps as b}from"./driver-apps.js";import{AxRequestError as f}from"./ax-client.js";import{mapTree as k,textOf as x}from"./elements.js";import{newWindowsNote as S,noWindowYetNote as W,textRunsNote as N,windowUnreadableNote as v}from"./narrate.js";import{SHOT_MAX_LONG_SIDE as g,TREE_MAX_DEPTH as $,TREE_MAX_ELEMENTS as _,TREE_MAX_TEXT as M,TREE_MAX_VISITED as p,TEXT_PREVIEW as y,TREE_WALK_DEADLINE_MS as T,pinnable as c,unreadable as u}from"./ax-shapes.js";class m extends b{frame=null;rects=new Map;offscreenRefs=new Set;shotScale=1;shotTaken=!1;refsByWindow=new Map;static MAX_WINDOWS_REMEMBERED=12;static MAX_ABSENT_REFS=400;refCount=0;appWindowIds=null;async walk(e=M){const{pid:i,window:n}=await this.ensure(),o={max:p,deadlineMs:T,maxElements:_,maxText:e,depth:$};let s;try{s=await this.call("tree",{pid:i,...n!==void 0?{window:n}:{},...o})}catch(a){if(!(a instanceof f&&a.code==="no_window"&&n!==void 0))throw await this.sayNoWindow(a);this.windowId=null;try{s=await this.call("tree",{pid:i,...o})}catch(d){throw await this.sayNoWindow(d)}}if(s.capped&&this.addNote(`the accessibility walk stopped at its read budget (${s.visited} nodes) before covering the whole window \u2014 what is listed is complete for the part it reached; `+(s.cappedBy==="time"?"the app answered too slowly to finish, so retrying may well reach further":"focus a smaller window or read text")),this.frame=s.window,this.windowId=c(s.window.id),await this.noteWindowChanges(i,this.windowId),!s.window.onScreen&&s.elements.length===0)throw new E(this.windowUnreadable(s.locked));(s.window.w<=0||s.window.h<=0)&&this.addNote("this window reports no frame at all (macOS does not list it: minimised, another Space, or a sleeping display) \u2014 coordinates and element rects from it mean nothing, so act by ref; a screenshot of it will be refused");const w=`${this.app?.bundle??"?"}:${s.window.id}`,l=this.refsByWindow.get(w),t=new Map,h=k(s,a=>{const d=l?.get(a)??`e${++this.refCount}`;return t.set(a,d),d});let r=0;for(const[a,d]of l??[]){if(r>=m.MAX_ABSENT_REFS)break;!t.has(a)&&a.endsWith("|1")&&(t.set(a,d),r++)}this.refsByWindow.delete(w),this.refsByWindow.set(w,t);for(const a of this.refsByWindow.keys()){if(this.refsByWindow.size<=m.MAX_WINDOWS_REMEMBERED)break;this.refsByWindow.delete(a)}return this.acts.keys=h.keys,this.rects=h.rects,this.offscreenRefs=new Set(h.offscreenElements.map(a=>a.ref)),{tree:s,mapped:h}}lastWalkBare=!1;treeBlind(){return this.lastWalkBare}lastWalkHadWeb(){return this.walkHadWeb}walkHadWeb=!1;takeBareWindow(){return!this.lastWalkBare||this.saidSelfDrawn?!1:(this.saidSelfDrawn=!0,!0)}saidSelfDrawn=!1;chromeOnly(e){return!e.some(i=>i.name.trim()!==""&&i.role!=="text"&&i.role!=="image")}windowUnreadable(e){return v(e)}async snapshot(){const{tree:e,mapped:i}=await this.walk(),n=this.app;e.textOverflow&&this.addNote(N(e.textOverflow)),e.folded&&this.addNote(`${e.folded} structural nodes with no name, value or action of their own were folded out of the listing`);const o=this.chromeOnly(i.elements);return this.lastWalkBare=o,this.walkHadWeb=!!e.nudged||i.elements.some(s=>s.role==="webarea"),o||(this.saidSelfDrawn=!1),e.nudged&&i.elements.length<5&&this.addNote(`${n.name} answered the accessibility switch and its window still lists almost nothing \u2014 its web content is not published to accessibility yet. The menu commands still work (menu({})), and a control you can SEE is reachable by position (w.observe({screenshot: true}), then w.click({x, y})). Reading again later may find it awake`),{url:`macos:${n.bundle}:${e.window.id}`,title:`${n.name} \u2014 ${e.window.title||"(untitled window)"}`,elements:i.elements,offscreenElements:i.offscreenElements,text:i.text.length>y?`${i.text.slice(0,y)}\u2026`:i.text,meta:{scrollY:0,viewportH:e.window.h,docH:e.window.h,...e.offscreen?{offscreen:e.offscreen}:{},...e.overflow?{capped:e.overflow}:{},...e.capped?{partial:`the walk stopped at its read budget (${e.visited} nodes, out of ${e.cappedBy==="time"?"time":"nodes"})`}:{},...i.sharedMarks.length?{sharedMarks:i.sharedMarks}:{}}}}async noteWindowChanges(e,i){const n=await this.windows(e).then(r=>r.rows).catch(()=>null);if(!n)return;const o=n.filter(r=>c(r.id)!==null),s=n.length-o.length,w=this.appWindowIds?.pid===e?this.appWindowIds:null,l=new Set(o.map(r=>r.id));if(this.appWindowIds={pid:e,ids:l,unnumbered:s},!w)return;const t=S(e,o.filter(r=>!w.ids.has(r.id)&&r.id!==i),s>w.unnumbered),h=[...w.ids].filter(r=>!l.has(r));(t||h.length||s<w.unnumbered)&&(this.pendingSurfaceChange=!0,this.addNote(t??(h.length?`this app no longer lists window${h.length>1?"s":""} ${h.map(r=>`${e}:${r}`).join(", ")} \u2014 computer.surfaces() lists the remaining targets`:"this app's window list changed: an unnumbered window is no longer listed")))}async sayNoWindow(e){if(!(e instanceof f)||e.code!=="no_window")return e;const i=this.app;if(!i)return e;try{const{windows:n,locked:o}=await this.call("windows",{pid:i.pid});if(o||n.length>0)return e}catch{return e}return new f("no_window",W(i.name),e.ax)}async screenshot(){const{pid:e,window:i}=await this.ensure();let n;try{n=await this.call("shot",{pid:e,...i!==void 0?{window:i}:{},maxLongSide:g})}catch(o){if(!(o instanceof f&&o.code==="no_window"&&i!==void 0))throw u(await this.sayNoWindow(o));this.windowId=null;try{n=await this.call("shot",{pid:e,maxLongSide:g})}catch(s){throw u(await this.sayNoWindow(s))}}return this.shotScale=n.scale||1,this.shotTaken=!0,this.windowId=c(n.id),this.frame={id:n.id,title:this.frame?.title??"",onScreen:!0,x:n.x,y:n.y,w:n.w,h:n.h},await this.noteWindowChanges(e,this.windowId),Buffer.from(n.png,"base64")}async screenshotScoped(e){if(e.fullPage)throw new Error("fullPage screenshots are not supported on macOS; use ref or region for window detail");if(!!e.ref==!!e.region)throw new Error("pick exactly one of ref or region for a macOS scoped screenshot");if(e.padding!==void 0&&(!e.ref||!Number.isFinite(e.padding)||e.padding<0))throw new Error("padding needs a finite number of full-screenshot pixels, 0 or more, with ref");if(e.region&&(![e.region.x,e.region.y,e.region.width,e.region.height].every(Number.isFinite)||!(e.region.width>0)||!(e.region.height>0)))throw new Error("region needs finite x/y and positive width/height");if(e.region&&!this.shotTaken)throw new Error("region coordinates use the full screenshot; take w.screenshot() or w.observe({screenshot:true}) first");const{pid:i,window:n}=await this.ensure(),o=n??this.frame?.id;if(o===void 0||o<0)throw new Error("observe the intended window before taking a scoped screenshot");let s;if(e.ref){const d=this.acts.resolve(e.ref,void 0);if(!("key"in d))throw new Error(d.note??this.staleRefNote(e.ref));if(this.offscreenRefs.has(e.ref))throw new Error(`${e.ref} is outside the captured window; choose a visible control or region`);s=d.key}const w=e.region,l=(e.padding??0)*this.shotScale;if(!Number.isFinite(l))throw new Error("padding is too large for the full screenshot's coordinate scale");let t;try{t=await this.call("shotscoped",{pid:i,window:o,...s?{key:s,padding:l}:{region:{x:w.x*this.shotScale,y:w.y*this.shotScale,width:w.width*this.shotScale,height:w.height*this.shotScale}}})}catch(d){throw d instanceof f&&d.code==="stale_key"&&e.ref?new Error(this.staleRefNote(e.ref)):u(d)}const h=this.frame,r=this.shotTaken&&h?.id===t.id&&[h.x-t.x,h.y-t.y,h.w-t.w,h.h-t.h].every(d=>Math.abs(d)<.5),a={target:{pid:i,windowId:t.id},window:{x:t.x,y:t.y,width:t.w,height:t.h},region:{x:t.region.x,y:t.region.y,width:t.region.w,height:t.region.h},units:"window-points",pointsPerPixel:{x:t.scale,y:t.scaleY},toFullScreenshot:r?{origin:{x:t.region.x/this.shotScale,y:t.region.y/this.shotScale},scale:{x:t.scale/this.shotScale,y:t.scaleY/this.shotScale}}:null};return{bytes:Buffer.from(t.png,"base64"),mimeType:"image/png",capture:a,note:`${e.ref?`element ${e.ref}`:"region"}, ${t.pixelWidth}\xD7${t.pixelHeight} pixels from ${t.region.w}\xD7${t.region.h} window points at (${t.region.x},${t.region.y}) in window ${t.id}${t.trimmed?" \u2014 trimmed to the window":""}`+(this.shotTaken&&!r?" \u2014 window geometry changed since observation; observe again before point actions":"")}}async readText(){const{tree:e,mapped:i}=await this.walk(p),n=[...e.elements,...e.offscreenElements??[]].filter(o=>o.valueCut);return n.length?(await Promise.all(n.map(async o=>{try{const s=(await this.call("value",{key:o.key})).value;s.length>=o.value.length&&(o.value=s,delete o.valueCut)}catch{}})),x(e)):i.text}async readValue(e){const i=this.acts.resolve(e,void 0);if(!("key"in i))throw new Error(i.note??this.staleRefNote(e));try{return String((await this.call("value",{key:i.key})).value??"")}catch(n){throw n instanceof f&&n.code==="stale_key"?new Error(this.staleRefNote(e)):n}}}export{m as MacosDriverRead};

package/dist/platform/macos/driver-targets.d.ts

@@ -0,0 +1,9 @@
+import type { ActResult, TargetInfo } from "../types.ts";
+import { MacosDriverRead } from "./driver-read.ts";
+export declare abstract class MacosDriverTargets extends MacosDriverRead {
+    listTargets(): Promise<TargetInfo[]>;
+    focusTarget(id: string): Promise<ActResult>;
+    unlistedTarget(query: string): Promise<TargetInfo | null>;
+    isActiveTarget(id: string): Promise<boolean>;
+    protected launch(bundle: string): Promise<ActResult>;
+}

package/dist/platform/macos/driver-targets.js

@@ -0,0 +1 @@
+import{MacosDriverRead as l}from"./driver-read.js";import{launchedNote as c,noWindowYetNote as h,panelOnLaunchNote as f,unclearedAppNote as w}from"./narrate.js";import{pinnable as p}from"./ax-shapes.js";import{noAccessibility as $}from"./permissions.js";class v extends l{async listTargets(){await this.refreshGrants();const e=await this.scopedApps(),s=[];for(const t of e){if(!this.cleared(t.bundle)){s.push({id:`${t.pid}`,kind:"app",title:`${t.name} (${t.bundle}) \u2014 ${w()}`,active:!1});continue}const{rows:n,locked:o}=await this.windows(t.pid).catch(()=>({rows:[],locked:!1})),d=n.filter(r=>p(r.id)!==null);for(const r of d)s.push({id:`${t.pid}:${r.id}`,kind:"window",title:`${t.name} (${t.bundle}) \u2014 ${r.title||"(untitled)"}`,active:this.app?.pid===t.pid&&this.windowId===r.id,...r.focused||r.main?{primary:!0}:{}});if(d.length===0){const r=o?" \u2014 the screen is LOCKED; they come back when the user unlocks it":"",u=n.length===0?`no windows${r}`:`${n.length} window(s), none on screen${r}`;s.push({id:`${t.pid}`,kind:"app",title:`${t.name} (${t.bundle}) \u2014 ${u}`,active:this.app?.pid===t.pid})}}let a=[...new Set([...this.fallback??[],...this.approved,...this.asking?this.stored.list():[]])].filter(t=>!this.hostApps.has(t)&&!e.some(n=>n.bundle===t));for(const t of a.length?await this.lookup({bundles:a}):[])s.push({id:`${t.pid}`,kind:"app",title:`${t.name} (${t.bundle}) \u2014 running without a Dock icon`,active:this.app?.pid===t.pid}),a=a.filter(n=>n!==t.bundle);for(const t of await this.installed(a))s.push({id:t.bundle,kind:"app",title:`${t.name} (${t.bundle}) \u2014 not running; binding it starts it`,active:!1});return s}async focusTarget(e){const[s,i]=e.split(":");if(!/^\d+$/.test(s??""))return this.launch(e);const a=Number(s),t=Number.isFinite(a)?(await this.lookup({pids:[a]}))[0]:void 0;if(!t)return{ok:!1,note:`no running app with pid ${s}; call computer.surfaces() for current ids`};await this.authorize(t.bundle,t.name);const{rows:n}=await this.windows(a),o=i===void 0?n.find(d=>d.main)??n[0]:n.find(d=>String(d.id)===i);return i!==void 0&&!o?{ok:!1,note:`${t.name} has no window ${i} any more; call computer.surfaces() for current ids`}:(this.forgetApp(),this.app=t,this.windowId=o?p(o.id):null,this.frame=o??null,{ok:!0,note:`now driving ${t.name}${o?` \u2014 ${o.title||"(untitled)"}`:""} (in the background; the user's focus is untouched)`})}async unlistedTarget(e){const s=e.trim();if(!s||this.fallback&&!this.asking)return null;const i=/^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+){1,}$/.test(s),a=i?s:(await this.appsNamed(s).catch(()=>[])).find(o=>!this.hostApps.has(o));if(!a||this.hostApps.has(a))return null;const t=(await this.lookup({bundles:[a]}))[0];if(t){const o=i?"running without a Dock icon; binding it asks as usual":"running";return{id:`${t.pid}`,kind:"app",title:`${t.name} (${t.bundle}) \u2014 ${o}`,active:!1}}const n=(await this.installed([a]))[0];return n?{id:n.bundle,kind:"app",title:`${n.name} (${n.bundle}) \u2014 not running; binding it starts it`,active:!1}:null}async isActiveTarget(e){return this.app&&e===this.app.bundle||this.app&&/^\d+$/.test(e)&&Number(e)===this.app.pid?!0:super.isActiveTarget(e)}async launch(e){if(!e)return{ok:!1,note:"launch_app needs a bundle id, e.g. com.apple.TextEdit"};await this.authorize(e,()=>this.displayName(e));const s=this.app?.pid,{app:i,ready:a,started:t,windows:n,panel:o}=await this.call("launch",{bundle:e});if(this.app=i,t!==!1&&this.noteStarted(i.pid),s!==i.pid&&this.forgetApp(),!a)return await this.refreshGrants(),{ok:!0,note:this.granted&&!this.granted.accessibility?`${i.name} is running (pid ${i.pid}) and is now the target, but nothing about it can be READ: ${$(!this.granted.screenRecording)}`:`${i.name} is running (pid ${i.pid}) and is now the target, but it is not answering Accessibility yet \u2014 wait({ms:2000}), then observe(); if it still does not answer, the app may not support Accessibility at all`};const d=c(i.name,i.pid,t!==!1);return n===0?{ok:!0,note:`${d}. It is now the target. ${h(i.name)}`}:o?{ok:!0,note:`${d}. It is now the target. ${f(i.name)}`}:{ok:!0,note:`${d}. It is now the target; observe() to see its window`}}}export{v as MacosDriverTargets};

package/dist/platform/macos/driver.d.ts

@@ -0,0 +1,65 @@
+import { MacosDriverTargets } from "./driver-targets.ts";
+import type { DiagnosticsAbout } from "../driver-diagnostics.ts";
+import { ElementActs } from "./act-element.ts";
+import type { ActResult, HeaderChanges, OwnershipResource, PreviewTarget, Snapshot, TargetInfo, UserConsent, VerbSpec } from "../types.ts";
+import { type AxHelperOptions, type AxTransport } from "./ax-client.ts";
+import { MacosApprovals } from "./approvals.ts";
+import { type AppNameResolver } from "./app-names.ts";
+export interface MacosDriverOptions {
+    interaction?: "background" | "desktop";
+    bundle?: string;
+    env?: NodeJS.ProcessEnv;
+    consent?: UserConsent;
+    transport?: AxTransport;
+    appsNamed?: AppNameResolver;
+    helper?: AxHelperOptions;
+}
+export declare class MacosDriver extends MacosDriverTargets {
+    private pointerHeld;
+    releaseInputResources(): Promise<void>;
+    protected previewTarget(): Promise<PreviewTarget | null>;
+    readonly platform = "macos";
+    readonly screenshotMimeType: string;
+    ownershipResources(): Promise<OwnershipResource[]>;
+    actionResources(kind: string, args: Record<string, unknown>): Promise<OwnershipResource[]>;
+    protected readonly ax: AxTransport;
+    protected readonly wantBundle: string | undefined;
+    protected readonly fallback: Set<string> | null;
+    protected readonly hostApps: ReadonlySet<string>;
+    protected readonly consent: UserConsent | null;
+    protected readonly stored: MacosApprovals;
+    protected readonly appsNamed: AppNameResolver;
+    private readonly menu;
+    private keyPid;
+    private typingChannel;
+    private readonly mouse;
+    private lastKeyAt;
+    protected readonly acts: ElementActs;
+    private pendingNote;
+    protected appNoteThisRound: string | null;
+    protected readonly toldAbout: Set<string>;
+    constructor(opts?: MacosDriverOptions);
+    readonly supportsDiagnostics = true;
+    get diagnosticsAbout(): DiagnosticsAbout;
+    private collapsedFrameRisk;
+    protected frozenExtra(): string | null;
+    extraVerbs(): VerbSpec[];
+    protected offscreenNote(count: number): string;
+    protected renderHeadLines(s: Snapshot, targets: TargetInfo[], targetsUnchanged?: boolean, changes?: HeaderChanges): string[];
+    private takeIntro;
+    protected trimSurvivors(): string[];
+    private takeAppNote;
+    protected staleRefCause(): string;
+    explainMiss(query: string): string | null;
+    promptFragment(): string;
+    interactionNotes(): readonly string[];
+    protected addNote(note: string): void;
+    protected takePendingNote(): string | null;
+    perform(kind: string, raw: Record<string, any>): Promise<ActResult>;
+    private performAction;
+    close(): Promise<void>;
+    private forgetHelper;
+    private toScreen;
+    private scroll;
+    protected forgetApp(): void;
+}

package/dist/platform/macos/driver.js

@@ -0,0 +1,2 @@
+import{MacosDriverTargets as u}from"./driver-targets.js";import{MACOS_VERBS as d}from"./extra-verbs.js";import{ElementActs as m}from"./act-element.js";import{AxHelperProcess as f}from"./ax-client.js";import{menuChord as w,menuVerb as y,normalizeChord as l,pasteVerb as k}from"./menu-commands.js";import{COLLAPSED_FRAME_HINT as b,dragVerb as g,hoverVerb as A,PressMemory as S}from"./mouse.js";import{clickVerb as v}from"./click-verb.js";import{selectTextVerb as N}from"./select-text-verb.js";import{scrollVerb as x}from"./scroll-verb.js";import{typeVerb as c}from"./type-verb.js";import{offscreenWindowNote as $,promptFragmentFor as _,selfDrawnWindowNote as M,sharedMarksNote as R,staleRefCauseNote as I}from"./narrate.js";import{SAVE_PANEL_NOTE as E,appNote as T}from"./app-notes.js";import{MACOS_INTERACTION_NOTES as H}from"./interaction-notes.js";import{DesktopInteractionRequired as h,clipboardChord as C}from"./interaction-policy.js";import{PANEL_SERVICE_BUNDLE as P}from"./ax-shapes.js";import{MacosApprovals as F}from"./approvals.js";import{findInstalledApps as O}from"./app-names.js";const p="#save-panel";class st extends u{pointerHeld=!1;async releaseInputResources(){if(!this.pointerHeld)return;const t=await this.perform("pointer_release",{});if(!t.ok)throw new Error(t.note||"Could not release the held pointer")}previewTarget(){const t=this.app,e=this.windowId;return!t||e===null||!this.cleared(t.bundle)?Promise.resolve(null):Promise.resolve({kind:"macos",targetId:`${t.pid}:${e}`,title:this.frame?.title||t.name,appName:t.name,url:`macos:${t.bundle}:${e}`,capture:{type:"macos",app:t.bundle,pid:t.pid,windowId:e}})}platform="macos";screenshotMimeType="image/png";async ownershipResources(){const{pid:t}=await this.ensure(),e=`macos:app:${t}`;return this.windowId===null?[{key:e,mode:"exclusive"}]:[{key:`macos:window:${t}:${this.windowId}`,mode:"exclusive"},{key:e,mode:"shared"}]}async actionResources(t,e){const s=["click","hover","drag","scroll","activate","pointer_move","pointer_press","pointer_release","paste"].includes(t),n=["keys","type","select_text"].includes(t)||t==="menu"&&!!e.path;if(!s&&!n)return[];const{pid:a}=await this.ensure(),i=this.interactionMode==="desktop"?s||t==="keys"||t==="menu":t==="pointer_release";return[{key:`macos:app:${a}`,mode:"exclusive"},...i?[{key:"macos:desktop-input",mode:"exclusive"}]:[]]}ax;wantBundle;fallback;hostApps;consent;stored;appsNamed;menu={checkCommand:async(t,e)=>{if(this.interactionMode==="background"&&((e.mods&8)===0&&/^[cxv]$/i.test(e.cmd)||await this.menu.isFrontmost(t)))throw new h(`menu ${e.path}`)},call:(t,e)=>this.call(t,e),walk:()=>this.walk(),appName:()=>this.app.name,windowId:()=>this.windowId,keyPostedAt:t=>{this.lastKeyAt=t},closeWindow:async t=>{await this.call("closewindow",{pid:t,...this.windowId!==null?{window:this.windowId}:{}}),this.windowId=null},raiseWindow:async t=>this.windowId===null?!1:(await this.call("raisewindow",{pid:t,window:this.windowId})).raised===!0,isFrontmost:async t=>(await this.lookup({pids:[t]}))[0]?.active===!0,refForKey:t=>this.acts.refForKey(t),startedHere:t=>this.startedHere(t),keyPid:t=>this.keyPid(t)};async keyPid(t){let e=await this.call("focused",{pid:t}).then(n=>n.element.pid??t,()=>{});if(e===void 0&&(e=await this.call("panelservice",{pid:t}).then(n=>n.pid||t,()=>t)),typeof e!="number"||e<=0||e===t)return t;const s=(await this.lookup({pids:[e]}))[0];return s&&s.bundle!==P&&await this.authorize(s.bundle,s.name),e}async typingChannel(t){const e=await this.keyPid(t),s=e===t&&!await this.menu.isFrontmost(t);return{to:e,call:(n,a)=>s&&n==="type"?this.call("backgroundtype",{...a,window:this.windowId}):this.call(n,a)}}mouse={call:(t,e)=>this.call(t,e),pid:()=>this.app.pid,appName:()=>this.app.name,rectOf:t=>{const e=this.rects.get(t);return e&&this.frame?{x:this.frame.x+e.x,y:this.frame.y+e.y,w:e.w,h:e.h}:null},toScreen:(t,e)=>this.toScreen(t,e),offWindow:t=>this.offscreenRefs.has(t),walk:async()=>(await this.walk()).mapped.elements,bundle:()=>this.app.bundle,hadWeb:()=>this.lastWalkHadWeb(),pointerAllowed:()=>this.interactionMode==="desktop",flagCollapsedFrameRisk:()=>{this.collapsedFrameRisk=!0},memory:new S};lastKeyAt=0;acts=new m((t,e)=>this.call(t,e));pendingNote=null;appNoteThisRound=null;toldAbout=new Set;constructor(t={}){super(),this.interactionMode=t.interaction??"background";const e=t.env??process.env;this.ax=t.transport??new f({env:e,onLog:n=>this.diagnosticsSink?.push("helper",n),...t.helper,onStart:async n=>{await this.onExecutorStart?.(n),await t.helper?.onStart?.(n)}}),this.ax.nameKeys=this.acts.nameKeys,this.ax.onReset=()=>this.forgetHelper(),this.wantBundle=t.bundle?.trim()||void 0;const s=e.COMPUTER_USE_MACOS_FALLBACK_APPS;this.fallback=s===void 0?null:new Set(s.split(",").map(n=>n.trim()).filter(Boolean)),this.hostApps=new Set((e.COMPUTER_USE_HOST_APPS??"").split(",").map(n=>n.trim()).filter(Boolean)),this.consent=t.consent??null,this.stored=new F(e),this.appsNamed=t.appsNamed??O,this.consent?.onRevoke(n=>this.revoke(n))}supportsDiagnostics=!0;get diagnosticsAbout(){return{empty:"complaints from the accessibility helper (this platform's diagnostics are the helper's own log, never the app's)",header:"the macOS accessibility helper's own log \u2014 what went wrong inside the platform, not inside the app"}}collapsedFrameRisk=!1;frozenExtra(){const t=this.collapsedFrameRisk;return this.collapsedFrameRisk=!1,t&&this.lastWalkHadWeb()?b:null}extraVerbs(){return d}offscreenNote(t){return $(t)}renderHeadLines(t,e,s=!1,n){const a=super.renderHeadLines(t,e,s,n);t.meta.sharedMarks?.length&&a.push(R(t.meta.sharedMarks));const i=this.takeIntro(t);return i&&(a.push(i),this.appNoteThisRound=i),a}takeIntro(t){const e=[];this.takeBareWindow()&&e.push(M(this.app?.name??t.title,this.app?.webShell===!0));const s=this.takeAppNote();return s&&e.push(s),!this.toldAbout.has(p)&&t.elements.some(n=>n.id==="saveAsNameTextField")&&(this.toldAbout.add(p),e.push(E)),e.length?e.join(`
+`):null}trimSurvivors(){const t=this.appNoteThisRound;return this.appNoteThisRound=null,t?[t]:[]}takeAppNote(){const t=this.app?.bundle;return!t||this.toldAbout.has(t)?null:(this.toldAbout.add(t),T(t))}staleRefCause(){return I()}explainMiss(t){if(!this.fallback||this.asking)return null;const e=t.trim();return/^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+){1,}$/.test(e)&&!this.fallback.has(e)?`mirasim-cu: ${this.notAllowed(e)}`:"mirasim-cu: this session drives only a fixed list of apps, and an app outside it is not listed here at all \u2014 so a missing name may be a running app rather than a closed one."}promptFragment(){return _()}interactionNotes(){return H}addNote(t){this.pendingNote===t||this.pendingNote?.endsWith(` Also: ${t}`)||(this.pendingNote=this.pendingNote?`${this.pendingNote} Also: ${t}`:t)}takePendingNote(){const t=this.pendingNote;return this.pendingNote=null,t}async perform(t,e){const s=this.dispatchedActions;try{return await this.performAction(t,e)}catch(n){if(n instanceof h)return{ok:!1,mayHaveActed:this.dispatchedActions>s,note:n.message};throw n}}async performAction(t,e){const s={...e};if(t==="launch_app")return this.launch(String(s.bundle??""));const{pid:n}=await this.ensure();if(this.interactionMode==="background"){if(t==="paste"||t==="keys"&&C(l(String(s.chord??""))))return{ok:!1,note:new h(t).message};if(["keys","type"].includes(t)&&await this.menu.isFrontmost(n))return{ok:!1,note:new h(`${t} in the frontmost app`).message};if(t==="ax_action"){const a=this.acts.resolve(s.ref,s.name);if("key"in a&&a.role==="menuitem")return{ok:!1,note:"Use menu({path}) for menu items so the background policy can check their command and clipboard effects."}}}switch(t){case"click":return v(this.mouse,this.acts,s,Math.max(1,Math.min(3,s.count??1)));case"drag":return g(this.mouse,this.acts,s);case"hover":return A(this.mouse,this.acts,s);case"pointer_move":{const a=this.toScreen(Number(s.x),Number(s.y)),i=await this.call("ptrmove",{pid:n,...a,...s.steps===void 0?{}:{steps:Number(s.steps)}});return{ok:!0,note:`moved the pointer to ${Math.round(Number(s.x))},${Math.round(Number(s.y))} over ${i.steps} motion events${i.dragging?" with the button still held (a drag)":""} and left it there \u2014 the user's cursor is now here`}}case"pointer_state":{const a=await this.call("ptrstate",{}),i=this.frame,r=i&&this.shotTaken?` = ${Math.round((a.x-i.x)/this.shotScale)},${Math.round((a.y-i.y)/this.shotScale)} in this window's screenshot`:" (no screenshot in this session yet, so there is no screenshot-space answer)";return{ok:!0,note:`the pointer is at ${Math.round(a.x)},${Math.round(a.y)} on screen${r}${a.owned?"":" \u2014 it has not been moved in this session, so that is simply where the user's cursor is"}${a.down?", holding a button":""}${a.grabbed?", and the user's physical mouse is disconnected from the cursor until the release":""}`}}case"pointer_press":return this.pointerHeld=!0,{ok:!0,note:`pressed the ${(await this.call("ptrpress",{pid:n,button:String(s.button??"left")})).button} button and is HOLDING it at the pointer's position \u2014 this can bring ${this.app.name} to the front and change keyboard focus; activation was not verified. While held, the user's physical mouse is disconnected from the cursor; always pointer_release in finally (an idle hold also releases after 20 s)`};case"pointer_release":{if(!(await this.call("ptrstate",{})).down)return this.pointerHeld=!1,{ok:!0,verified:!0,note:"no button is held; no mouse-up was posted and the pointer was not moved"};const i=await this.call("ptrrelease",{pid:n,button:String(s.button??"left")});return this.pointerHeld=!1,{ok:!0,note:i.wasHolding?"released the button at the tracked pointer position":"no button was held; posted a real mouse-up at the helper's tracked position, which can move the user's cursor \u2014 this is not a read-only probe"}}case"set_text":{if(s.ref)return this.acts.setValue(s.ref,s.name,String(s.text??""));const a=await this.typingChannel(n);return c(a.call,a.to,this.app.name,String(s.text??""),this.lastKeyAt)}case"type":{const a=await this.typingChannel(n);return c(a.call,a.to,this.app.name,String(s.text??""),this.lastKeyAt)}case"keys":{const a=l(String(s.chord??"")),i=Math.min(Math.max(Math.trunc(Number(s.count))||1,1),20),r=await this.keyPid(n);if(/\bcmd\b/i.test(a)&&r===n)return await w(this.menu,n,a,i);if(/\bcmd\b/i.test(a)){if(!await this.menu.isFrontmost(n))return{ok:!1,note:`${this.app.name}'s focus is in its Open/Save panel (a separate process), and a \u2318-chord there takes effect only with the app frontmost \u2014 activate, then send it again. The panel's fields and buttons still answer by ref in the background (set_text a field, click a button)`};for(let o=0;o<i;o++)await this.call("key",{pid:r,chord:a});return this.lastKeyAt=Date.now(),{ok:!0,note:`${a} was posted to the Open/Save panel that holds ${this.app.name}'s focus (its own process), not run as one of the app's menu commands`}}if(r===n&&!await this.menu.isFrontmost(n)){const o=await this.call("backgroundkey",{pid:n,window:this.windowId,chord:a,count:i});return this.lastKeyAt=o.lastInputAt??Date.now(),{ok:!0,note:`${a} was posted ${i} time(s) to the bound background window; observe whether it took effect`}}for(let o=0;o<i;o++)await this.call("key",{pid:r,chord:a});return this.lastKeyAt=Date.now(),{ok:!0}}case"ax_action":return this.acts.action(s.ref,s.name,String(s.action??""));case"paste":return k(this.menu,n,s);case"select_text":return N(this.acts,s.ref,s.name,{want:String(s.text??""),near:typeof s.near=="string"?s.near:"",caret:String(s.caret??"select")});case"menu":return y(this.menu,n,s.path);case"scroll":return this.scroll(s);case"activate":return await this.call("activate",{pid:n}),{ok:!0,note:`${this.app.name} is now frontmost \u2014 the user's keyboard goes to it until they switch back`};default:return{ok:!1,note:`unknown action kind ${t}`}}}async close(){this.forgetHelper();try{await this.ax.close()}finally{await this.disconnected()}}forgetHelper(){this.pointerHeld=!1,this.frame=null,this.shotScale=1,this.shotTaken=!1,this.resetObservationState(),this.refsByWindow.clear(),this.acts.keys.clear(),this.rects.clear(),this.offscreenRefs.clear(),this.appWindowIds=null}toScreen(t,e){if(this.frame&&!this.shotTaken&&!this.accessRefused)throw new Error("a point is a pixel of the LAST SCREENSHOT, and this session has not taken one \u2014 so there is no scale to map it with, and on a Retina window the guess is off by a factor of two. Take w.observe({screenshot: true}) and read the coordinates off that picture (it carries the pointer's own position as a ring)");const s=this.frame??{x:0,y:0};return{x:Math.round(s.x+t*this.shotScale),y:Math.round(s.y+e*this.shotScale)}}async scroll(t){return await this.ensure(),this.frame||await this.walk(),x(this.mouse,this.acts,this.frame,t)}forgetApp(){this.windowId=null,this.frame=null,this.shotScale=1,this.shotTaken=!1,this.resetObservationState()}}export{st as MacosDriver};

package/dist/platform/macos/element-lines.d.ts

@@ -0,0 +1,12 @@
+import type { AxElement } from "./element-shapes.ts";
+export declare function roleOf(e: AxElement): string;
+export declare const collapse: (s: string) => string;
+export declare const clean: (s: string) => string;
+export declare const SECONDARY_ACTIONS: Record<string, string>;
+export declare function secondaryActions(actions: string[]): string[];
+export declare function ubiquitousMarks(perControl: string[][]): string[];
+export declare const HELP_MAX = 80;
+export declare function helpBeyond(help: string, name: string): string;
+export declare function ubiquitousHelp(helps: string[]): Set<string>;
+export declare function axActionName(input: string): string | null;
+export declare function stateOf(e: AxElement, role: string, focusIsExclusive: boolean, marks: string[], name: string): string;

package/dist/platform/macos/element-lines.js

@@ -0,0 +1 @@
+const l={AXButton:"button",AXPopUpButton:"combobox",AXMenuButton:"button",AXDisclosureTriangle:"button",AXColorWell:"button",AXCheckBox:"checkbox",AXRadioButton:"radio",AXTextField:"textbox",AXTextArea:"textbox",AXSearchField:"searchbox",AXComboBox:"combobox",AXLink:"link",AXMenuItem:"menuitem",AXSlider:"slider",AXIncrementor:"spinbutton",AXTabGroup:"tablist",AXStaticText:"text",AXImage:"image",AXWebArea:"webarea",AXCell:"cell",AXRow:"row"},f={AXSearchField:"searchbox"};function X(t){const e=f[t.subrole];if(e)return e;const o=l[t.role];return o||(t.actions.includes("AXPress")?"button":t.role.replace(/^AX/,"").toLowerCase()||"element")}const i=t=>(t??"").trim().replace(/\s+/g," "),A=t=>i(t).slice(0,80),a=t=>{const e=i(t);return e.length>60?e.slice(0,60)+"\u2026":e},u={AXShowMenu:"menu",AXIncrement:"increment",AXDecrement:"decrement",AXConfirm:"confirm",AXOpen:"open",AXPick:"pick"};function d(t){return Object.entries(u).filter(([e])=>t.includes(e)).map(([,e])=>e)}const b=.9,x=40;function h(t){if(t.length<x)return[];const e=new Map;for(const n of t)for(const c of n)e.set(c,(e.get(c)??0)+1);const o=t.length*b;return Object.values(u).filter(n=>(e.get(n)??0)>=o)}const p=5,m=80,s=t=>t.toLowerCase().replace(/[^\p{L}\p{N}]+/gu,"");function O(t,e){if(!t)return"";if(!e)return t;if(s(t)===s(e))return"";const o=t.toLowerCase().startsWith(e.toLowerCase())?t.slice(e.length).replace(/^[\s:：,，、.。\-—]+/u,"").trim():t;return s(o)===s(e)?"":o}function S(t){const e=new Map;for(const o of t)o&&e.set(o,(e.get(o)??0)+1);return new Set([...e].filter(([,o])=>o>=p).map(([o])=>o))}function w(t){const e=t.trim();if(!e)return null;const o=e.toLowerCase();for(const[n,c]of Object.entries(u))if(c===o||n.toLowerCase()===o)return n;return/^AX[A-Za-z]+$/.test(e)?e:null}function g(t,e,o,n,c){const r=[];return t.enabled||r.push("disabled"),t.runOf&&r.push(`${t.runOf} text runs`),e==="textbox"||e==="combobox"?r.push(t.value?`filled:${a(t.value)}`:"empty"):e==="checkbox"||e==="radio"?r.push(/^(1|true|yes)$/i.test(t.value)?"checked":"unchecked"):t.value&&(e!=="text"||A(t.value)!==c)&&r.push(`value:${a(t.value)}`),t.selected&&e!=="checkbox"&&e!=="radio"&&r.push("selected"),n.length&&r.push(`+${n.join(",")}`),t.focused&&o&&r.push("focused"),r.join(",")}export{m as HELP_MAX,u as SECONDARY_ACTIONS,w as axActionName,A as clean,i as collapse,O as helpBeyond,X as roleOf,d as secondaryActions,g as stateOf,S as ubiquitousHelp,h as ubiquitousMarks};

package/dist/platform/macos/element-shapes.d.ts

@@ -0,0 +1,54 @@
+export interface AxElement {
+    key: string;
+    role: string;
+    subrole: string;
+    title: string;
+    desc: string;
+    value: string;
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+    enabled: boolean;
+    focused: boolean;
+    actions: string[];
+    depth: number;
+    off?: boolean;
+    ident?: string;
+    help?: string;
+    seq?: number;
+    valueCut?: boolean;
+    selected?: boolean;
+    runOf?: number;
+}
+export interface AxWindow {
+    id: number;
+    title: string;
+    onScreen: boolean;
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+}
+export interface AxTree {
+    window: AxWindow;
+    elements: AxElement[];
+    visited: number;
+    capped: boolean;
+    cappedBy?: "nodes" | "time" | "";
+    offscreen: number;
+    offscreenElements?: AxElement[];
+    overflow: number;
+    textOverflow?: number;
+    folded?: number;
+    nudged: boolean;
+    locked: boolean;
+}
+export interface AxMenuItem {
+    key: string;
+    path: string;
+    title: string;
+    enabled: boolean;
+    cmd: string;
+    mods: number;
+}

package/dist/platform/macos/element-text.d.ts

@@ -0,0 +1,3 @@
+import type { AxElement, AxTree } from "./element-shapes.ts";
+export declare function textOf(tree: AxTree): string;
+export declare function foldTextRuns(elements: AxElement[]): AxElement[];

package/dist/platform/macos/element-text.js

@@ -0,0 +1,2 @@
+import{clean as c,collapse as i,roleOf as r}from"./element-lines.js";function h(l){const n=l.offscreenElements??[];if(!n.length)return l.elements;const e=s=>s.seq??Number.POSITIVE_INFINITY;return[...l.elements,...n].sort((s,t)=>e(s)-e(t))}function m(l){const n=[];for(const e of h(l)){const s=r(e),t=c(e.title||e.desc||(s==="text"?e.value:"")),o=e.value.trim(),u=e.valueCut?` (cut at ${e.value.length} chars \u2014 the app has the full value)`:"";s==="text"?o&&n.push(o+u):o&&!(s==="checkbox"||s==="radio")?n.push((t?`${t}: ${o}`:o)+u):t&&n.push(t)}return n.join(`
+`)}const f=80,a=5;function v(l){const n=[];let e=[];const s=()=>{if(e.length){if(e.length<a)n.push(...e);else{const t=e[0],o=i(e.map(u=>u.value||u.title||u.desc).join(""));n.push({...t,value:o.length>f?`${o.slice(0,f)}\u2026`:o,runOf:e.length})}e=[]}};for(const t of l)r(t)==="text"&&!c(t.title||t.desc)?e.push(t):(s(),n.push(t));return s(),n}export{v as foldTextRuns,m as textOf};

package/dist/platform/macos/elements.d.ts

@@ -0,0 +1,26 @@
+import type { Element } from "../types.ts";
+import type { AxTree } from "./element-shapes.ts";
+export type { AxElement, AxMenuItem, AxTree, AxWindow } from "./element-shapes.ts";
+export { axActionName, roleOf, SECONDARY_ACTIONS } from "./element-lines.ts";
+export { textOf } from "./element-text.ts";
+export type RefAllocator = (signature: string) => string;
+export interface MappedTree {
+    elements: Element[];
+    offscreenElements: Element[];
+    keys: Map<string, {
+        key: string;
+        name: string;
+        role: string;
+        actions: string[];
+        enabled: boolean;
+    }>;
+    rects: Map<string, {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    }>;
+    text: string;
+    sharedMarks: string[];
+}
+export declare function mapTree(tree: AxTree, refFor?: RefAllocator): MappedTree;

package/dist/platform/macos/elements.js

@@ -0,0 +1 @@
+import{clean as r,roleOf as B,secondaryActions as _,stateOf as I,ubiquitousHelp as q,ubiquitousMarks as C,helpBeyond as D,HELP_MAX as L}from"./element-lines.js";import{foldTextRuns as W,textOf as Y}from"./element-text.js";import{axActionName as K,roleOf as Q,SECONDARY_ACTIONS as U}from"./element-lines.js";import{textOf as Z}from"./element-text.js";const P=(()=>{let s=0;return()=>`e${++s}`})(),X=new Set(["checkbox","radio","slider","combobox","textbox"]);function F(s,H=P){const w=[],b=[],f=[],k=new Map,M=new Map,$=new Map,N=s.elements.filter(e=>e.focused).length<=1,{x:R,y:O}=s.window;let d=[];const S=new Set(s.elements.filter(e=>B(e)!=="text").map(e=>r(e.title||e.desc)).filter(Boolean)),A=e=>{d=[];const l=[];for(const t of W(e)){const n=B(t);for(;l.length&&l[l.length-1].depth>=t.depth;)l.pop();const h=l.some(c=>c.role==="cell"||c.role==="row");l.push({depth:t.depth,role:n});let o=r(t.title||t.desc||(n==="text"?t.value:""));const a=!o&&n==="textbox"&&h&&t.value?r(t.value):"";n==="text"&&o&&(d=[...d.slice(-2),{name:o,x:t.x,y:t.y,h:t.h}]);const i=!o&&!a&&X.has(n)?[...d].reverse().find(c=>c.x<t.x&&c.y<t.y+t.h&&t.y<c.y+c.h):void 0,x=!!i;if(i&&(o=i.name),n==="text"&&!o||n==="text"&&S.has(o))continue;const p=!o&&t.ident?`#${r(t.ident)}`:"";p&&(o=p),a&&(o=a);const E=x?o:r(t.title||t.desc)||p,u=E?`${n}|${E}`:`${n}|@row${Math.round(t.y-O)}+${Math.round(t.h)}`,g=($.get(u)??0)+1;$.set(u,g);const y=H(`${u}|${g}`),v={x:Math.round(t.x-R),y:Math.round(t.y-O),w:Math.round(t.w),h:Math.round(t.h)};f.push({e:t,role:n,name:o,ref:y,rect:v,marks:_(t.actions)}),k.set(y,{key:t.key,name:o,role:n,actions:t.actions,enabled:t.enabled}),M.set(y,v)}};A(s.elements),A(s.offscreenElements??[]);const m=C(f.filter(e=>!e.e.off&&e.role!=="text").map(e=>e.marks)),T=q(f.filter(e=>!e.e.off&&e.role!=="text").map(e=>r(e.e.help??"")));for(const e of f){const l=m.length?e.marks.filter(x=>!m.includes(x)):e.marks,t=r(e.e.ident??""),n=t&&e.role!=="text"&&e.name&&!e.name.startsWith("#")?t:"",h=r(e.e.help??""),o=e.role!=="text"&&!T.has(h)?D(h,e.name):"",a=o.length>L?`${o.slice(0,L)}\u2026`:o,i={ref:e.ref,role:e.role,name:e.name,state:I(e.e,e.role,N,l,e.name),rect:e.rect,...n?{id:n}:{},...a?{help:a}:{}};(e.e.off?b:w).push(i)}return{elements:w,offscreenElements:b,keys:k,rects:M,text:Y(s),sharedMarks:m}}export{U as SECONDARY_ACTIONS,K as axActionName,F as mapTree,Q as roleOf,Z as textOf};

package/dist/platform/macos/extra-verbs.d.ts

@@ -0,0 +1,2 @@
+import type { VerbSpec } from "../types.ts";
+export declare const MACOS_VERBS: VerbSpec[];

package/dist/platform/macos/extra-verbs.js

@@ -0,0 +1 @@
+import{Type as e}from"typebox";import{SELECT_TEXT_VERB as t}from"./select-text-verb.js";import{SECONDARY_ACTIONS as i}from"./elements.js";import{DRAG_VERB as n}from"./mouse.js";import{TYPE_VERB as a}from"./type-verb.js";const l=[{kind:"activate",schema:e.Object({kind:e.Literal("activate")},{description:"Bring the app's windows to the foreground and redirect the user's keyboard focus; explain this effect and obtain permission before calling."})},{kind:"launch_app",schema:e.Object({kind:e.Literal("launch_app"),bundle:e.String({description:"Bundle id to launch and switch to, e.g. com.apple.TextEdit (never a display name \u2014 names are localised)"})},{description:"Switch this session to another app, launching it if it is not running. The app this session was given needs no such call \u2014 it is launched on first use \u2014 so this is for moving to a DIFFERENT app, and for making a document app that has every window closed open one again."})},{kind:"menu",schema:e.Object({kind:e.Literal("menu"),path:e.Optional(e.String({description:"Menu command to run, exactly as the listing shows it \u2014 menus are in the system language (Fichier \u203A Nouveau on a French Mac), so list first (menu({})) rather than guessing an English name. \u203A > / all separate; the last segment alone also works. Omit to list the commands runnable right now."}))})},n,a,{kind:"pointer_move",schema:e.Object({kind:e.Literal("pointer_move"),x:e.Number({description:"Where to move, in the coordinates of the last screenshot"}),y:e.Number(),steps:e.Optional(e.Number({description:"How many motion events the path is cut into (default 12, max 60). More steps is a slower, more human approach; an app that polls input per frame needs several"}))},{description:"Move the user's real pointer along a path and leave it there; explain the pointer takeover and obtain permission first. While a button is held the path is posted as a drag."})},{kind:"pointer_state",readOnly:!0,schema:e.Object({kind:e.Literal("pointer_state")},{description:"Where the pointer is and what it is holding \u2014 the one READ of the three. Worth asking mid-drag (a press, then moves: nothing else says where the release will land) and after a click, whose rung decides whether the cursor came along. `grabbed` says the user's physical mouse is disconnected from the cursor right now, which only a held button does."})},{kind:"pointer_press",schema:e.Object({kind:e.Literal("pointer_press"),button:e.Optional(e.String({description:"left (default), right, middle"}))},{description:"Hold a real mouse button, potentially changing foreground focus and temporarily disconnecting the physical mouse; explain these effects and obtain permission first. Always release in finally, including on failure."})},{kind:"pointer_release",schema:e.Object({kind:e.Literal("pointer_release"),button:e.Optional(e.String({description:"left (default), right, middle"}))},{description:"Release an authorized mouse hold, including cleanup in background mode; without a held button this posts no input. Use pointerState to inspect pointer state."})},{kind:"ax_action",schema:e.Object({kind:e.Literal("ax_action"),ref:e.String({description:"Element ref from the latest observe() (e.g. e12)"}),name:e.Optional(e.String({description:"Visible name of the element \u2014 identity check + fallback locator"})),action:e.String({description:`The action to run, as the element's [+\u2026] mark names it: ${Object.values(i).join(", ")} (the AX spelling, e.g. AXShowMenu, works too). An element only accepts actions it advertises \u2014 but advertising one is no promise it does anything: read the observation after the act, not the mark.`})},{description:"Run one accessibility action the element itself offers \u2014 expand a popup's menu, pick an item in it, step a stepper. For a plain click use click; to dismiss a sheet, click its own button."})},t,{kind:"paste",schema:e.Object({kind:e.Literal("paste"),text:e.String({description:"The content as plain text (Markdown goes here too)"}),html:e.Optional(e.String({description:"The same content as HTML, for apps that accept rich text"})),ref:e.Optional(e.String({description:"The element you EXPECT to have focus. Not a destination \u2014 the paste follows the app's focus \u2014 but a mismatch is refused instead of landing somewhere else. Pass it whenever you know where you want the text."}))},{description:"Paste at the app's actual focus with the app frontmost, temporarily replacing the system clipboard and attempting to restore it afterwards; explain these effects and obtain permission first. Pass ref to verify focus before the clipboard is changed."})}];export{l as MACOS_VERBS};

package/dist/platform/macos/interaction-notes.d.ts

@@ -0,0 +1,2 @@
+export declare const MACOS_INTERACTION_RULE: string;
+export declare const MACOS_INTERACTION_NOTES: readonly string[];

package/dist/platform/macos/interaction-notes.js

@@ -0,0 +1 @@
+const e="Before an operation that may change the user's foreground, move or hold the real pointer, redirect keyboard focus, or use the system clipboard, explain the target app and the effect to the user and obtain permission. Reuse permission only while it covers the app, effects and operation sequence. App-access permission alone does not authorize interrupting the user. If permission is absent or declined, use a non-disruptive route or stop; do not silently escalate a background attempt to foreground input.",t=[e,"click does not automatically repeat a press whose result is uncertain. click(t, {retry:true}) permits that fallback only for an action you know is harmless to repeat; it does not grant desktop permission.","Each computer_repl call defaults to interaction:background: driver-controlled real pointer, activation and clipboard routes are blocked, including fallback. After obtaining scoped permission, use interaction:desktop for the covered calls. This setting is not proof of consent. AX actions can still cause the app itself to present windows or change state; background is not application isolation.","click (including by ref) can fall back to a real mouse click and raise the app within the same call; obtain permission before using that unrestricted path. An explicit axAction runs only the named accessibility action, without a mouse fallback. Right/middle/double clicks and drag use real mouse events.","activate brings the app's windows forward. hover and pointerMove move the real cursor and leave it there; movement alone does not establish foreground activation. pointerPress holds a real button and temporarily disconnects the physical mouse; always release in finally, including on failure. Without a held button pointerRelease posts no input; pointerState is the read-only state query.","scroll first tries AX paging without moving the pointer; wheel fallback posts system pointer input and requests cursor restoration, and needs an uncovered target. Obtain permission before a scroll that may take that fallback.","paste requires the app frontmost and temporarily replaces the system clipboard, then attempts to restore it. Copy/Cut commands through keys or menu also affect the system clipboard, even in the background; they do not restore it.","observe/find/text/value and surface selection do not deliberately activate the app. setText/selectText and AX actions can change the target app's focus, caret or selection. keys/type can use protected background delivery, but share that app's input state; coordinate with anyone using the same app/window. A menu command in an already-frontmost app may raise the bound window. App actions can themselves open or activate windows.","Open/Save-panel command shortcuts still need the app frontmost; its fields/buttons can be addressed by ref. An instruction to activate, retry with the mouse, or use paste is not permission to do so. A hardware-idle check is not user permission or a lock against another agent."];export{t as MACOS_INTERACTION_NOTES,e as MACOS_INTERACTION_RULE};

package/dist/platform/macos/interaction-policy.d.ts

@@ -0,0 +1,5 @@
+export declare const DESKTOP_INPUT_OPS: Set<string>;
+export declare class DesktopInteractionRequired extends Error {
+    constructor(operation: string);
+}
+export declare function clipboardChord(chord: string): boolean;

package/dist/platform/macos/interaction-policy.js

@@ -0,0 +1 @@
+const r=new Set(["mouse","hover","drag","scroll","activate","raisewindow","clipboardstash","ptrmove","ptrpress"]);class o extends Error{constructor(t){super(`${t} requires desktop interaction; this call is background-only. The desktop operation was not dispatched. Explain the app and foreground/pointer/focus/clipboard effects, obtain permission, then use interaction:'desktop' for the covered sequence. Inspect any preceding AX action before retrying it.`),this.name="DesktopInteractionRequired"}}function n(e){return/(?:\bcmd\b|\bcommand\b|\bmeta\b|⌘)/i.test(e)&&/(?:\+|⌘|⇧|⌥|\s)[cxv]$/i.test(e.trim())}export{r as DESKTOP_INPUT_OPS,o as DesktopInteractionRequired,n as clipboardChord};

package/dist/platform/macos/menu-commands.d.ts

@@ -0,0 +1,22 @@
+import type { ActResult } from "../types.ts";
+import type { AxMenuItem, MappedTree } from "./elements.ts";
+export interface MenuHost {
+    checkCommand?(pid: number, item: AxMenuItem): Promise<void>;
+    call<T>(op: string, params?: Record<string, unknown>): Promise<T>;
+    walk(): Promise<{
+        mapped: MappedTree;
+    }>;
+    appName(): string;
+    windowId(): number | null;
+    keyPostedAt(at: number): void;
+    closeWindow(pid: number): Promise<void>;
+    raiseWindow(pid: number): Promise<boolean>;
+    isFrontmost(pid: number): Promise<boolean>;
+    refForKey(key: string): string | null;
+    keyPid(pid: number): Promise<number>;
+    startedHere(pid: number): boolean;
+}
+export declare function normalizeChord(raw: string): string;
+export declare function menuChord(host: MenuHost, pid: number, chord: string, count?: number): Promise<ActResult>;
+export declare function menuVerb(host: MenuHost, pid: number, rawPath: unknown): Promise<ActResult>;
+export declare function pasteVerb(host: MenuHost, pid: number, a: Record<string, any>): Promise<ActResult>;

package/dist/platform/macos/menu-commands.js

@@ -0,0 +1,5 @@
+const g=4e3,v=250,T=400,y=e=>`${e.mods&4?"\u2303":""}${e.mods&2?"\u2325":""}${e.mods&1?"\u21E7":""}${e.mods&8?"":"\u2318"}${e.cmd.toUpperCase()}`,h=e=>e.cmd.toLowerCase()==="q"&&(e.mods===0||e.mods===2);async function w(e,t){return(await e.call("menu",{pid:t})).items}async function b(e,t,a){const o=Date.now()+4e3;let n=null;for(;;){if(n=(await w(e,t)).find(a)??null,!n||n.enabled||Date.now()>=o)return n;await new Promise(s=>setTimeout(s,250))}}function L(e){const t={"\u2318":"cmd","\u21E7":"shift","\u2325":"alt","\u2303":"ctrl"};if(![...e].some(n=>n in t))return e;const a=[];let o="";for(const n of e)n in t?a.push(t[n]):n!=="+"&&n.trim()&&(o+=n);return[...a,o.toLowerCase()].filter(Boolean).join("+")}async function N(e,t,a,o=1){const n=a.split("+").map(f=>f.trim().toLowerCase()).filter(Boolean),s=n[n.length-1]??"",c=(n.includes("shift")?1:0)|(n.includes("alt")||n.includes("opt")?2:0)|(n.includes("ctrl")?4:0);if(s==="q"&&(c===0||c===2)&&!e.startedHere(t))return{ok:!1,note:`${a} would quit ${e.appName()}, which the user had open \u2014 leave it running`};const d=f=>f.cmd.toLowerCase()===s&&f.mods===c;let r=(await w(e,t)).find(d);if(!await e.isFrontmost(t)&&!(r&&!r.enabled&&r.mods===0&&["a","w"].includes(s)))return m(e,t,a,o);const l=e.windowId();let u=0,i={ok:!0};try{for(let f=0;f<o;f++){if(f>0){if(l!==null){const{windows:p}=await e.call("windows",{pid:t});if(!p.some(k=>k.id===l))return{ok:!1,mayHaveActed:u>0,note:`${u} of ${o} commands were dispatched; the bound window closed, so no more were sent. Observe before retrying`}}r=(await w(e,t)).find(d)}if(r?i=await $(e,t,r,a):(await e.call("key",{pid:await e.keyPid(t),chord:a}),i={ok:!0,note:`${a} was posted; no menu item is bound to it \u2014 observe whether the app handled it`}),!i.ok)return{...i,mayHaveActed:u>0||i.mayHaveActed,note:`${u} of ${o} commands were dispatched. ${i.note??"Observe before retrying"}`};u++,e.keyPostedAt(Date.now())}}catch(f){throw f instanceof Error&&(f.message=`${u} of ${o} commands were dispatched before interruption. ${f.message}`),f}return o===1?i:{...i,note:`${u} of ${o} commands were dispatched. ${i.note??"Observe whether they took effect"}`}}async function m(e,t,a,o){const n=e.windowId();if(n===null)return{ok:!1,note:"a background shortcut needs a bound window \u2014 observe and bind one first"};const s=await e.call("backgroundkey",{pid:t,window:n,chord:a,count:o});return e.keyPostedAt(s.lastInputAt??Date.now()),{ok:!0,note:`${a} was posted${o>1?` ${o} times`:""} to the bound window in the background; `+(s.cleanupVerified===!0?"app-local deactivation was verified":"app-local cleanup was requested")+" and the user's foreground was preserved \u2014 observe whether the command took effect"}}async function A(e,t,a){const o=await w(e,t),n=typeof a=="string"?a.trim():"";if(!n){const r=o.filter(p=>p.enabled&&(!h(p)||e.startedHere(t))).map(p=>`${p.path}${p.cmd?` (${y(p)})`:""}`),l=o.filter(p=>!p.enabled).length,u=await e.isFrontmost(t),i=u?"with the app frontmost":"with the app in the background",f=l?u?`
+(${l} more are disabled right now, with ${e.appName()} already frontmost \u2014 that gate is the app's own: a panel or sheet is up, or the command does not apply to what is open or selected)`:`
+(${l} more need the app's window to be key \u2014 they run after activate)`:"";return{ok:!0,note:`${r.length} menu commands runnable ${i} (listed below \u2014 this verb returns them):
+${r.join(`
+`)}${f}`}}const s=r=>r.toLowerCase().replace(/\s*[›>/]\s*/g," \u203A ").replace(/\s+/g," ").trim(),c=s(n),d=o.find(r=>s(r.path)===c)??o.find(r=>s(r.path).endsWith(` \u203A ${c}`)||s(r.title)===c);return d?$(e,t,d,`menu {path:"${d.path}"}`):{ok:!1,note:`no menu command "${n}" \u2014 menu without a path lists them`}}async function C(e,t,a){const o=String(a.text??""),n=typeof a.html=="string"?a.html:"";if(!o&&!n)return{ok:!1,note:"paste needs text (and optionally html)"};const s=i=>i.cmd.toLowerCase()==="v"&&i.mods===0;let c=(await w(e,t)).find(s)??null;if(!c)return{ok:!1,note:`${e.appName()} has no Paste command bound to \u2318V \u2014 set_text writes a field's whole value instead, and \`menu\` lists what this app does offer`};if(c.enabled){if(!await e.isFrontmost(t))return{ok:!1,note:`${e.appName()} is not frontmost, so it has no key window and a paste has no defined destination \u2014 explain the foreground/clipboard effects and obtain user permission before activating. Nothing was put on the clipboard`}}else{if(!await e.isFrontmost(t))return{ok:!1,note:`${c.path} is disabled while ${e.appName()} is in the background: a paste goes to the app's key window, and an app that is behind has no key window \u2014 explain the foreground/clipboard effects, obtain user permission, then activate and paste. Nothing was put on the clipboard`};if(c=await b(e,t,s),!c?.enabled)return{ok:!1,note:`${e.appName()} is frontmost, but ${c?.path??"its Paste command"} is still disabled after ${4e3/1e3}s \u2014 its key window takes no paste (a panel, or nothing focused that accepts text). Click into the field first. Nothing was put on the clipboard`}}let d=null;try{d=(await e.call("focused",{pid:t})).element}catch{d=null}const r=d?`${d.role}${d.title||d.desc?` "${d.title||d.desc}"`:""}${d.window?` in window "${d.window}"`:""}`:null;if(!r)return{ok:!1,note:`${e.appName()} reports nothing focused, so a paste has nowhere to land \u2014 click into the field first. Nothing was put on the clipboard`};if(typeof a.ref=="string"&&a.ref&&d.key!==void 0){const i=e.refForKey(d.key);if(i&&i!==a.ref)return{ok:!1,note:`focus is on ${i} (${r}), not ${a.ref} \u2014 a paste follows the app's focus, not the ref, so it would land there. Click ${a.ref} first, then paste. Nothing was put on the clipboard`}}const l=(await e.call("clipboardstash",{text:o,...n?{html:n}:{}})).stashed;let u=null;try{await e.call("press",{key:c.key}),await new Promise(i=>setTimeout(i,400))}finally{u=await e.call("clipboardrestore",{}).then(i=>typeof i.restored=="number"?i.restored:null,()=>null)}return{ok:!0,note:`ran ${c.path} at the app's focus \u2014 ${r}${n?", with rich text":""}. `+(u===-1?"The clipboard changed during paste; its newer contents were preserved":u===null?"Clipboard restoration could not be verified":l?"The user's clipboard was put back afterwards (an app that reads the pasteboard late could still see ours)":"Their clipboard was empty before and is empty again")}}async function $(e,t,a,o,n=1){if(await e.checkCommand?.(t,a),h(a)&&!e.startedHere(t))return{ok:!1,note:`${a.path} would quit ${e.appName()}, which the user had open \u2014 leave it running. Close what you opened INSIDE it instead (a document, a panel), or leave the app where it is.`};if(/\bcmd\b/i.test(o)&&!await e.isFrontmost(t)&&!(!a.enabled&&a.mods===0&&["a","w"].includes(a.cmd.toLowerCase())))return m(e,t,o,n);let s=a,c=!1,d=!1;if(await e.isFrontmost(t)&&await e.raiseWindow(t).catch(()=>!1)&&(d=!0,s=(await w(e,t)).find(l=>l.key===a.key)??a),!s.enabled&&await e.isFrontmost(t)&&(s=await b(e,t,l=>l.key===a.key)??a,c=!0),s.enabled){await e.call("press",{key:s.key});const l=c?" \u2014 it became runnable once the app finished coming forward":await e.isFrontmost(t)?" with the app frontmost":" with the app in the background",u=d?` \u2014 the driven window was raised first, so it ran there and not in the one ${e.appName()} had in front`:"";return{ok:!0,note:`ran ${s.path} (${o})${l}${u}`}}const r=s.cmd.toLowerCase();if(s.mods===0&&r==="a"){const{mapped:l}=await e.walk(),u=l.elements.find(p=>/\bfocused\b/.test(p.state??"")),i=u?l.keys.get(u.ref)?.key:void 0,{length:f}=await e.call("selectall",{pid:t,...i?{key:i}:{}});return{ok:!0,note:`${s.path} is disabled while ${e.appName()} is in the background; selected the focused ${u?.role??"element"}'s whole value instead (${f} chars)`}}if(s.mods===0&&r==="w")return await e.closeWindow(t),{ok:!0,note:`${s.path} is disabled while ${e.appName()} is in the background; pressed the window's close button instead \u2014 observe: the app may have opened a save sheet`};if(c){const l=/\bcmd\b/i.test(o)?o:null;return l&&await e.call("key",{pid:await e.keyPid(t),chord:l}),{ok:!!l,note:`${s.path} is disabled even with ${e.appName()} frontmost \u2014 a panel or sheet is up, or the command does not apply to what is focused. `+(l?`${l} was posted to whatever holds the keyboard focus instead (a Save panel's own process included), since a panel can bind it itself \u2014 observe whether it took`:"Dismiss the panel or focus what the command acts on, then run it again")}}return/\bcmd\b/i.test(o)?m(e,t,o,n):{ok:!1,note:`${s.path} is disabled while ${e.appName()} is in the background (macOS validates it against the key window, which an inactive app has none of) \u2014 activate, then ${o}`}}export{N as menuChord,A as menuVerb,L as normalizeChord,C as pasteVerb};

package/dist/platform/macos/mouse.d.ts

@@ -0,0 +1,23 @@
+import type { ActResult, VerbSpec } from "../types.ts";
+import type { ElementActs } from "./act-element.ts";
+import { PressMemory, type MouseHost } from "./press-rungs.ts";
+export { PressMemory, type MouseHost };
+export declare const DRAG_VERB: VerbSpec;
+export type Point = {
+    x: number;
+    y: number;
+};
+export declare const NO_MOUSE: {
+    readonly mouse: "unsupported";
+};
+export declare const centre: (r: {
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+}) => Point;
+export declare function hoverVerb(host: MouseHost, acts: ElementActs, a: Record<string, any>): Promise<ActResult>;
+export declare function outOfReach(what: string, ref: unknown, verb: string): ActResult;
+export declare function dragVerb(host: MouseHost, acts: ElementActs, a: Record<string, any>): Promise<ActResult>;
+export declare const COLLAPSED_FRAME_HINT: string;
+export declare function mouse(host: MouseHost, at: Point, button: string, count: number, what: string, reason?: string, byRef?: boolean): Promise<ActResult>;

package/dist/platform/macos/mouse.js

@@ -0,0 +1 @@
+import{Type as i}from"typebox";import{AxRequestError as p}from"./ax-client.js";import{PressMemory as g}from"./press-rungs.js";const $=400,S={kind:"drag",schema:i.Object({kind:i.Literal("drag"),ref:i.Optional(i.String({description:"Element to drag \u2014 REQUIRED unless x/y are given"})),name:i.Optional(i.String({description:"Visible name of the dragged element"})),x:i.Optional(i.Number({description:"Drag start x in the latest screenshot (instead of ref)"})),y:i.Optional(i.Number({description:"Drag start y"})),toRef:i.Optional(i.String({description:"Element to drop onto \u2014 REQUIRED unless toX/toY are given"})),toX:i.Optional(i.Number({description:"Destination x (instead of toRef)"})),toY:i.Optional(i.Number({description:"Destination y"}))},{description:"Drag with the user's real pointer and potentially bring the app/window forward; explain these effects and obtain permission first. Needs a start (ref or x/y) AND a destination (toRef or toX/toY)."})},k={mouse:"unsupported"},u=e=>({x:Math.round(e.x+e.w/2),y:Math.round(e.y+e.h/2)}),h=(e,a=!1)=>a?`\u2014 another window covered the point, so ${e} was requested to come to the front first; real pointer input was posted and cursor restoration requested`:`\u2014 real pointer input was posted and cursor restoration requested; this can bring ${e} to the front and change keyboard focus, but dispatch alone does not verify activation`;async function R(e,a,t){if(typeof t.ref=="string"&&t.ref&&e.offWindow(t.ref))return x(`${t.name?`"${String(t.name)}"`:String(t.ref)}`,t.ref,"hover");const o=typeof t.x=="number"&&typeof t.y=="number",n=o?null:a.resolve(t.ref,t.name);if(n&&!("key"in n))return n;const r=o?e.toScreen(t.x,t.y):(()=>{const s=e.rectOf(String(t.ref));return s?u(s):null})(),c=o?"the point":`${n.role} "${n.name}"`;return r?(await e.call("hover",{pid:e.pid(),...r}),{ok:!0,note:`moved the user's real pointer onto ${c} and left it there; no app activation was requested. Observe: controls that only exist while hovered${e.hadWeb()?" (a web view's included)":""} are in the tree now, with real frames.`}):{ok:!1,note:`${c} has no frame on this walk, so there is nowhere to hover \u2014 observe and act on a listed control`}}function x(e,a,t){return{ok:!1,note:`${e} is not on screen \u2014 the window lists it but does not show it, so there is no point to ${t}. It is still the right element: run one of the actions it offers (axAction({ref:'${String(a)}', action:\u2026})), or scroll it into view first and then act by ref`}}async function D(e,a,t){const o=await w(e,a,t.ref,t.name,t.x,t.y,"start");if("ok"in o)return o;const n=await w(e,a,t.toRef,void 0,t.toX,t.toY,"destination");if("ok"in n)return n;try{await e.call("drag",{pid:e.pid(),x:o.x,y:o.y,toX:n.x,toY:n.y})}catch(r){if(r instanceof p&&r.code==="no_window")return y(r.message);if(r instanceof p&&r.code==="unsupported")return m("drag");throw r}return{ok:!0,note:`dragged ${o.what} to ${n.what} with mouse events ${h(e.appName())}`}}async function w(e,a,t,o,n,r,c){if(typeof n=="number"&&typeof r=="number")return{...e.toScreen(n,r),what:`${n},${r}`};if(typeof t!="string")return{ok:!1,note:`drag needs a ${c}: a ref or x/y`};const s=a.resolve(t,o);if(!("key"in s))return s;const f=e.rectOf(t);return f?{...u(f),what:`${s.role} "${s.name}"`}:{ok:!1,note:`${s.role} "${s.name}" has no frame on this walk \u2014 observe and try again`}}const N="NOTE: the click went to a ref whose frame a web view may have COLLAPSED \u2014 such a control is only revealed on hover, so the click may have landed elsewhere: hover it, re-observe, and act on the fresh ref, or screenshot and click by x/y.";async function A(e,a,t,o,n,r,c=!1){c&&e.flagCollapsedFrameRisk?.();const s=()=>e.call("mouse",{pid:e.pid(),x:a.x,y:a.y,button:t,count:o});let f=!1;try{await s()}catch(l){if(l instanceof p&&l.code==="unsupported")return m(o>1?`${o}\xD7 click`:`${t} click`);if(!(l instanceof p&&l.code==="no_window"))throw l;await e.call("activate",{pid:e.pid()}),await new Promise(d=>setTimeout(d,$)),f=!0;try{await s()}catch(d){if(d instanceof p&&d.code==="no_window")return y(d.message);throw d}}const v=o>1?`${o}\xD7 clicked`:t==="left"?"clicked":`${t}-clicked`;return{ok:!0,note:`${r?`${r}; `:""}${v} ${n} with a mouse event at its centre ${h(e.appName(),f)}`}}const m=e=>({ok:!1,journal:k,note:`this helper build predates mouse events, so a ${e} is not available until the app's next update (a dev checkout compiles the helper from source); AXPress, keys and set_text are unaffected`}),y=e=>({ok:!1,note:`${e} even after activate brought the app forward \u2014 a mouse event lands on whatever is under the point, so the window is not on screen there (minimised, another Space, or a stale frame): observe again and act on what is listed`});export{N as COLLAPSED_FRAME_HINT,S as DRAG_VERB,k as NO_MOUSE,g as PressMemory,u as centre,D as dragVerb,R as hoverVerb,A as mouse,x as outOfReach};

package/dist/platform/macos/narrate.d.ts

@@ -0,0 +1,20 @@
+import type { AxWindowRow } from "./ax-shapes.ts";
+export declare function notAllowedNote(bundle: string, fallback: Set<string> | null): string;
+export declare function consentQuestion(name: string, bundle: string): string;
+export declare function declinedNote(name: string, bundle: string, reason?: string, allowAhead?: string): string;
+export declare function unclearedAppNote(): string;
+export declare function windowUnreadableNote(locked: boolean): string;
+export declare function launchedNote(name: string, pid: number, started: boolean): string;
+export declare function panelOnLaunchNote(name: string): string;
+export declare function noWindowYetNote(name: string): string;
+export declare const NAME_AN_APP: string;
+export declare function ownAppNote(bundle: string): string;
+export declare function backgroundOnlyNote(bundle: string, pid: number): string;
+export declare function offscreenWindowNote(count: number): string;
+export declare function textRunsNote(count: number): string;
+export declare function selfDrawnWindowNote(appName: string, webShell?: boolean): string;
+export declare function sharedMarksNote(marks: string[]): string;
+export declare function appExitedNote(name: string, op: string): string;
+export declare function staleRefCauseNote(): string;
+export { promptFragmentFor } from "./prompt-fragment.ts";
+export declare function newWindowsNote(pid: number, fresh: AxWindowRow[], unnumberedGrew: boolean): string | null;

package/dist/platform/macos/narrate.js

@@ -0,0 +1 @@
+function a(e,t){return`${e} is not one of the apps this session may drive, and its host cannot ask the user to allow more (it does not support MCP elicitation)${t?.size?`; allowed: ${[...t].join(", ")}`:""}`}function r(e,t){return`Allow Computer Use to use "${e}" (${t})?`}function h(e,t,n,o){return`permission to drive ${e} (${t}) was not granted${n?` \u2014 ${n}`:""}. Do not retry and do not ask again in this task; continue without it, or tell the user what you could not do`+(o?` \u2014 and, if nobody was there to answer (a non-interactive run declines every question), that they can allow it ahead of time with \`${o}\``:"")}function d(){return"not allowed yet; binding it asks the user"}function c(e){return e?"the screen is LOCKED \u2014 while it is, macOS reports every window as off-screen with no frame and no controls, so the accessibility tree cannot be read. This app HAS controls; the tree cannot see them until the user unlocks the screen. A screenshot may still answer from the window's backing store (screenshot()); if it does not, report the locked screen and stop \u2014 every other app reads the same while it is locked.":"this window is not on screen \u2014 minimised, on another Space, or its display is asleep \u2014 so macOS reports it with no frame and no controls. That is not an app without controls: a screenshot may still answer (screenshot()), and computer.surfaces() lists the windows that ARE on screen; bringing this one back is the user's call."}function l(e,t,n){return n?`${e} was not running: it was launched in the background (pid ${t}), the user's front window untouched \u2014 what follows is a fresh start, not a window the user had open`:`${e} was already running (pid ${t}), so its windows are the user's own \u2014 opened before this session, not by it`}function u(e){return`${e} came up asking which document to open, not showing one: that panel is a remote view (openAndSavePanelService), so drive it BY REF \u2014 the app's menu commands do not reach it, and a \u2318-chord for the panel itself (\u2318\u21E7G Go To Folder) needs the app frontmost. Its Cancel button leaves the app running with no document open`}function p(e){return`${e} has no window open: a document app with everything closed is a real state, not a failure, so the next observation will be empty. menu({}) lists its commands \u2014 File \u203A New (named in the system language) or the like opens one; an app with no menu bar (a toolbar-only agent such as Screenshot) is opened from outside, not from here`}const w="mirasim-cu: no app was named, and naming one is the caller's call, not this platform's. Take one with computer.surface('<bundle id or app name>') \u2014 binding a quit app starts it; computer.surfaces() lists what this session can drive.";function m(e){return`${e} is the app this agent runs inside, so it does not drive it \u2014 an observation of it would contain this session's own UI. Drive it through its own CLI instead, and name another bundle id here.`}function f(e,t){return`${e} is running (pid ${t}) but it is a BACKGROUND-ONLY process (LSBackgroundOnly): no windows, no accessibility tree, nothing to drive. If you meant an app with a UI, name that bundle id instead.`}function y(e){return`${e} outside the window \u2014 NOT unreachable: they are in this observation's element set, so find('<name>') returns one and you can act on it by ref without scrolling. Only the POINTER needs them on screen \u2014 click/hover refuse a ref the window does not show and name the action to run instead. Scrolling buys no TEXT in web content either: text() already has what scrolled away, so read it before scrolling to find a phrase`}function g(e){return`${e} static text runs past this observation's text budget \u2014 the element list keeps the window's CONTROLS complete and only a leading slice of its text; text() returns all of it (a long transcript or document is normal here, and no control is missing because of it)`}function b(e,t=!1){return`${e} publishes only this window's own chrome to accessibility, so there is no control tree to act on. ${t?"Its interface is Chromium, and Chromium publishes its tree LATER under continued accessibility traffic \u2014 so this may well read normally after an act or two. Use the picture now, and take one ordinary observe() again before deciding the tree is never coming":"This is NOT an app with nothing in it and no later read will improve: everything you can see is drawn by the app itself"}. What works here is the picture \u2014 observe({screenshot: true}), then click({x, y}) with coordinates read off it, hover({x, y}) to place the pointer (an app that routes shortcuts by pointer position needs it there), and keys at the app's own keyboard focus. Two things will mislead you: menu({}) lists only the app- and window-level commands macOS owns, never this app's own features, and the change verdict is computed from this same tree \u2014 it will say frozen after an action that worked, so judge by a fresh screenshot instead`}function v(e){return`NOTE: every control below also offers ${e.join(", ")} (ax_action) \u2014 listed here rather than on each line because they all carry it, so it says nothing about which control to act on. An action being offered is not a promise it works; the observation after the act is what says whether anything happened.`}function x(e,t){return`${e} exited while computer-use was mid-call (${t}) \u2014 it is no longer running. Whatever the action was, the app is gone: this is not a failed call to retry. computer.surfaces() lists what is left; binding this app again starts a fresh one.`}function $(){return"the control it named is gone from the window, or the target window changed"}import{promptFragmentFor as T}from"./prompt-fragment.js";function k(e,t,n){const o="If it is a sheet or alert it BLOCKS the window you are reading, so acting here may do nothing until it is answered; a small untitled window right after a rename or a menu command may instead be a transient editor or popover that goes away by itself";return t.length?`this app opened a window it did not have before \u2014 ${t.map(i=>`"${i.title||"(untitled)"}" (focus ${e}:${i.id})`).join(", ")}. ${o}; focus it to see what it asks`:n?`this app opened a window it did not have before \u2014 macOS has not numbered it yet, so it has no target id to focus until the next read. ${o}`:null}export{w as NAME_AN_APP,x as appExitedNote,f as backgroundOnlyNote,r as consentQuestion,h as declinedNote,l as launchedNote,k as newWindowsNote,p as noWindowYetNote,a as notAllowedNote,y as offscreenWindowNote,m as ownAppNote,u as panelOnLaunchNote,T as promptFragmentFor,b as selfDrawnWindowNote,v as sharedMarksNote,$ as staleRefCauseNote,g as textRunsNote,d as unclearedAppNote,c as windowUnreadableNote};

package/dist/platform/macos/permissions.d.ts

@@ -0,0 +1,5 @@
+export declare const grantRoute: (pane: string) => string;
+export declare const noAccessibility: (pixelsAlsoDenied: boolean) => string;
+export declare const NO_SCREEN_RECORDING: string;
+export declare const ACT_OPS: Set<string>;
+export declare const NO_ACCESSIBILITY_ACT: string;

package/dist/platform/macos/permissions.js

@@ -0,0 +1 @@
+const t=e=>`Grant "${e}" to the app that launched this \u2014 on its own permissions page if it has one, otherwise System Settings \u2192 Privacy & Security \u2192 ${e}, tick that app (not the helper).`,n=e=>`Accessibility is not granted, so the control tree cannot be read. This is NOT an app without controls \u2014 no app will read any better until the grant lands, and nothing about this app is wrong. ${t("Accessibility")} `+(e?"Screen Recording is missing as well, so pixels are not a way around this one \u2014 nothing can be read until a grant lands.":"Pixels are a separate grant: screenshot() may still answer while this one is missing."),s=`Screen Recording is not granted, so this window cannot be photographed. The app is fine and the control tree is unaffected \u2014 read it with observe() instead. ${t("Screen Recording")}`,a=new Set(["press","setvalue","selectrange","focus","key","backgroundkey","backgroundtype","type","scroll","mouse","drag","selectall","closewindow","at","clipboardstash","clipboardrestore"]),i=`Accessibility is not granted, so nothing can be pressed, typed or scrolled. The action did NOT fail because the app ignored it or because the ref was stale, and no app will answer until the grant lands \u2014 pixels are no way around it either, since a screenshot cannot act. ${t("Accessibility")}`;export{a as ACT_OPS,i as NO_ACCESSIBILITY_ACT,s as NO_SCREEN_RECORDING,t as grantRoute,n as noAccessibility};

package/dist/platform/macos/press-rungs.d.ts

@@ -0,0 +1,49 @@
+import type { ActResult, Element } from "../types.ts";
+import type { AxCall, ElementActs } from "./act-element.ts";
+export interface MouseHost {
+    flagCollapsedFrameRisk?(): void;
+    call: AxCall;
+    pid(): number;
+    appName(): string;
+    rectOf(ref: string): {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    } | null;
+    toScreen(x: number, y: number): {
+        x: number;
+        y: number;
+    };
+    offWindow(ref: string): boolean;
+    walk(): Promise<Element[]>;
+    bundle(): string;
+    hadWeb(): boolean;
+    pointerAllowed(): boolean;
+    memory: PressMemory;
+}
+export declare const PRESS_SETTLE_MS = 800;
+export declare class PressMemory {
+    static readonly THRESHOLD = 2;
+    private readonly ignored;
+    count(app: string): number;
+    ignores(app: string): boolean;
+    noteIgnored(app: string): void;
+    noteHonoured(app: string): void;
+    hasHonoured(app: string): boolean;
+    private readonly honoured;
+    private readonly keyWorks;
+    keyActivates(app: string): boolean;
+    noteKeyWorked(app: string): void;
+    private redo;
+    armRedo(app: string, what: string): void;
+    takeRedo(app: string, what: string): boolean;
+}
+declare const SURFACE_GONE_MSG = "the window this press acted on is gone (the app has no window now), which is as changed as a surface gets";
+export declare function evidence(host: MouseHost): Promise<string>;
+export declare function surfaceGone(sig: string): boolean;
+export { SURFACE_GONE_MSG };
+export declare function changedSince(host: MouseHost, before: string): Promise<string | null>;
+export declare const keyRoute: (role: string) => boolean;
+export declare function activateByKey(host: MouseHost, acts: ElementActs, ref: unknown, name: unknown, before: string | null): Promise<ActResult | null>;
+export declare function activateByAncestor(host: MouseHost, acts: ElementActs, ref: unknown, name: unknown, before: string | null): Promise<ActResult | null>;

package/dist/platform/macos/press-rungs.js

@@ -0,0 +1,5 @@
+import{AxRequestError as u}from"./ax-client.js";import{roleOf as p}from"./elements.js";const h=800,g=60;class m{static THRESHOLD=2;ignored=new Map;count(n){return this.ignored.get(n)??0}ignores(n){return this.count(n)>=m.THRESHOLD}noteIgnored(n){this.ignored.set(n,this.count(n)+1)}noteHonoured(n){this.ignored.delete(n),this.honoured.add(n)}hasHonoured(n){return this.honoured.has(n)}honoured=new Set;keyWorks=new Set;keyActivates(n){return this.keyWorks.has(n)}noteKeyWorked(n){this.keyWorks.add(n)}redo=null;armRedo(n,a){this.redo=`${n}
+${a}`}takeRedo(n,a){const i=this.redo===`${n}
+${a}`;return this.redo=null,i}}const S=e=>e.map(n=>`${n.role}|${n.name}|${n.state.replace(/,?\bfocused\b/,"")}`).join(`
+`),f="\0gone",E="the window this press acted on is gone (the app has no window now), which is as changed as a surface gets";async function x(e){const n=e.call("windows",{pid:e.pid()}).then(o=>({rows:o.windows}),o=>({err:o}));let a;try{a=await e.walk()}catch(o){const t=await n;if(!(o instanceof u&&o.code==="no_window"&&"rows"in t&&t.rows.length===0))throw o;return f}const i=await n;if("err"in i)throw i.err;return`${S(a)}
+#${i.rows.map(o=>`${o.id}|${o.title}|${o.focused}`).join(",")}`}function P(e){return e===f}async function k(e,n){const a=Date.now()+h;for(;;){const i=await x(e);if(i!==n)return i;if(Date.now()>=a)return null;await new Promise(o=>setTimeout(o,g))}}const y=e=>/^(textbox|textarea|searchbox|combobox)$/.test(e)?null:/^(button|checkbox|radio)$/.test(e)?"Space":void 0,R=e=>y(e)!==void 0,b=h;async function T(e,n){const a=Date.now()+b;for(;;){if((await e.call("focus",{key:n})).focused===!0)return!0;if(Date.now()>=a)return!1;await new Promise(i=>setTimeout(i,40))}}async function _(e,n,a,i,o){if(o===null)return null;const t=n.resolve(a,i);if(!("key"in t))return null;const s=y(t.role);if(s===void 0)return null;try{if(!await T(e,t.key))return null;if(s===null)return e.memory.noteKeyWorked(e.bundle()),{ok:!0,note:`AXPress on ${t.name?`"${t.name}"`:`the ${t.role}`} changed nothing, so it was FOCUSED instead \u2014 which is what a click on a field does. The caret is in it now (keys and type go here); no key was sent, because on a field every key either types or commits`};await e.call("key",{pid:e.pid(),chord:s})}catch(r){if(r instanceof u)return null;throw r}return await k(e,o)===null?null:(e.memory.noteKeyWorked(e.bundle()),{ok:!0,note:`AXPress on ${t.name?`"${t.name}"`:`the ${t.role}`} changed nothing, so it was focused and ${s} was sent to ${e.appName()} instead \u2014 that DID change the window, and unlike a mouse click it left the app where it was, so the user's front window, keyboard and pointer are untouched`})}async function v(e,n,a,i,o){if(o===null)return null;const t=n.resolve(a,i);if(!("key"in t))return null;let s;try{s=(await e.call("ancestors",{key:t.key})).ancestors??[]}catch(r){if(r instanceof u)return null;throw r}for(const r of s){const c=r.actions.includes("AXPress")?"press":r.selectable?"select":"";if(!c)continue;if(c==="select"&&r.selected){const l=r.name?`the ${r.role.replace(/^AX/,"").toLowerCase()} "${r.name}"`:"its row";return{ok:!0,note:`${t.name?`"${t.name}"`:"this element"} carries no click of its own and ${l} is already selected \u2014 which is what a single click here means, so nothing more was done`}}try{await e.call(c,c==="press"?{key:r.key,action:"AXPress"}:{key:r.key})}catch(l){if(l instanceof u)continue;throw l}const d=c==="select"&&await e.call("ancestors",{key:t.key}).then(l=>l.ancestors?.[s.indexOf(r)]?.selected===!0,()=>!1);if(!d&&await k(e,o)===null)continue;e.memory.noteHonoured(e.bundle());const w=p({role:r.role,actions:r.actions}),$=r.name?`the ${w} "${r.name}"`:`its containing ${w}`;return{ok:!0,note:`${t.name?`"${t.name}"`:"this element"} carries no click of its own, so ${c==="press"?"AXPress ran on":"the selection was set on"} ${$} instead \u2014 ${d?"it reads back selected":"that DID change the window"}, and unlike a mouse click it left ${e.appName()} where it was, so the user's front window, keyboard and pointer are untouched`}}return null}export{h as PRESS_SETTLE_MS,m as PressMemory,E as SURFACE_GONE_MSG,v as activateByAncestor,_ as activateByKey,k as changedSince,x as evidence,R as keyRoute,P as surfaceGone};

package/dist/platform/macos/preview-stream.d.ts

@@ -0,0 +1,15 @@
+import type { PreviewSink, PreviewStream, PreviewTarget } from "../types.ts";
+export declare class MacosPreviewStream implements PreviewStream {
+    private readonly target;
+    private readonly sink;
+    private readonly maxSize;
+    private readonly env;
+    private disposed;
+    private stop;
+    constructor(target: PreviewTarget, sink: PreviewSink, maxSize: number, env?: NodeJS.ProcessEnv);
+    start(): void;
+    private event;
+    private unavailable;
+    tick(): Promise<void>;
+    dispose(): void;
+}

package/dist/platform/macos/preview-stream.js

@@ -0,0 +1 @@
+import{AxHelperProcess as r}from"./ax-client.js";import{CaptureClient as o}from"./capture-client.js";const p=16*1024*1024;class c{target;sink;maxSize;env;disposed=!1;stop=null;constructor(t,s,i,a=process.env){this.target=t,this.sink=s,this.maxSize=i,this.env=a}start(){const t=this.target.capture;if(this.stop||this.disposed||t.type!=="macos")return;const s=e=>this.event(e),i=o.fromEnv(this.env);if(i){const e=new AbortController;this.stop=()=>e.abort(),i.preview(t.pid,t.windowId,s,e.signal).catch(n=>{e.signal.aborted||this.unavailable(n.message)});return}const a=new r({env:{...this.env,COMPUTER_USE_PREVIEW_WORKER:"1"},captureLocal:!0,onEvent:s});this.stop=()=>{a.close()},a.request("previewstart",{pid:t.pid,window:t.windowId,maxLongSide:this.maxSize}).then(e=>this.event({event:"previewReady",...e})).catch(e=>this.unavailable(e.message))}event(t){this.disposed||(t.event==="previewError"||t.event==="previewUnavailable"?this.unavailable(typeof t.message=="string"?t.message:"Application capture stopped."):t.event==="previewReady"?this.sink.status({unavailable:null,...typeof t.icon=="string"?{icon:t.icon}:{}}):t.event==="previewFrame"&&typeof t.data=="string"&&t.data.length<p&&Number(t.w)>0&&Number(t.h)>0&&this.sink.frame({mimeType:"image/jpeg",data:t.data,width:Number(t.w),height:Number(t.h),capturedAt:typeof t.capturedAt=="number"?t.capturedAt:Date.now()}))}unavailable(t){this.disposed||this.sink.status({unavailable:t})}async tick(){this.start()}dispose(){this.disposed=!0,this.stop?.(),this.stop=null}}export{c as MacosPreviewStream};

package/dist/platform/macos/prompt-fragment.d.ts

@@ -0,0 +1 @@
+export declare function promptFragmentFor(): string;

package/dist/platform/macos/prompt-fragment.js

@@ -0,0 +1 @@
+import{MACOS_INTERACTION_RULE as e}from"./interaction-notes.js";function a(){return e+" Grounding: observations are text lists of one macOS application window's controls with refs, read from the app's accessibility tree. Act by ref from the latest observation. Clicks and set_text go through the app's own accessibility actions and work while the app stays in the BACKGROUND \u2014 the user keeps their keyboard and pointer. When an app accepts a click and ignores it (Electron web content does), the control is focused and sent one key instead, still in the background; only if that changes nothing is the click redone as a real mouse event, which can bring the app to the front \u2014 the result says which of the three acted. That last one waits if the user is typing or moving the pointer right then, and says so instead of acting: click again and it goes through once they stop. Right/middle clicks, double clicks and drag are always mouse events. A control you can SEE and cannot name \u2014 self-drawn, with no node of its own in the tree \u2014 is reachable by POSITION: take w.observe({screenshot: true}) and w.click({x, y}) with the coordinates read off that picture. It is not a raw mouse event: the point is hit-tested against the tree first, so when something pressable IS under it the click goes through that element's own AXPress and the app stays behind, exactly as a click by ref would; only a point with nothing pressable under it becomes a real mouse event that can bring the app forward. What it gives up is identity \u2014 there is no ref to go stale and no name to disagree \u2014 so it acts on whatever is at that spot now: read the coordinates off a fresh picture, never off an old one. A \u2318 shortcut (keys cmd+\u2026) uses accessibility equivalents or guarded key events to the bound window in the background; `menu` lists the app's commands or runs one by name. Document shortcuts such as Bold and Save can work without activate; observe whether the app handled them. Remote Open/Save panels still need the app frontmost for \u2318 shortcuts, and paste still needs the app's real focus. An interrupted background shortcut says so \u2014 observe before retrying. `activate` deliberately brings the app to the front and takes the user's keyboard. Plain keys and typing in a native background window use the same protected delivery. Interrupted input reports how much was posted; read the field before continuing, since replaying a partial operation can duplicate input. set_text fills a field by ref; a field that commits on Enter says so in its result (w.keys('Enter') commits). Where set_text does not stick \u2014 a file being renamed, a search field that filters as you type \u2014 focus the field (click, or selectText with caret) and w.type({text}) sends real keystrokes. Background input is bound to the driven window; remote-panel keys and typing go to the panel's own process. A web-content field's VALUE can be its placeholder, so a field whose value is exactly its own label is probably empty \u2014 accessibility cannot tell you which, so do not treat it as a draft to preserve. A read that looks PARTIAL is usually not: text() carries a web-content window's whole transcript even where it has scrolled out of sight (measured \u2014 a wheel burst changed the window and returned identical text), so scrolling to read more is wasted. When a read and the window seem to disagree, take a screenshot and look, rather than scrolling to find out. Text that is still ARRIVING reads as fragments: a streaming answer puts one AX node per token, so text() comes back with the words broken across lines and the element list churns on every read. That settles by itself \u2014 wait for what marks the end (w.wait({control_gone:'Stop'}) beats a fixed sleep), then read once. A long document or transcript is not a tree problem: to LOCATE a phrase, use the app's own Find \u2014 keys cmd+f, fill the find field, keys Enter \u2014 which is exact where a list is not. Verify typed input by read-back; an app can reformat or ignore it. Apps and windows are surfaces: computer.surfaces() lists them as handles, and acting through a handle switches to it. The listing carries the apps with a Dock presence; a menu-bar-only app (a status item, the screen-capture toolbar) is not in it but IS drivable \u2014 name its bundle id. Use text() to read all text in the window."}export{a as promptFragmentFor};