@github/computer-use-mcp 0.1.27 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +11 -11
- package/dist/main.js +10 -10
- package/dist/prebuilds/darwin-arm64/computer.node +0 -0
- package/dist/prebuilds/darwin-x64/computer.node +0 -0
- package/dist/prebuilds/linux-arm64/computer.node +0 -0
- package/dist/prebuilds/linux-x64/computer.node +0 -0
- package/dist/prebuilds/win32-arm64/computer.node +0 -0
- package/dist/prebuilds/win32-x64/computer.node +0 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{McpServer as
|
|
1
|
+
import{McpServer as Yt}from"@modelcontextprotocol/sdk/server/mcp.js";import{ElicitResultSchema as St}from"@modelcontextprotocol/sdk/types.js";import{z as p}from"zod";var j=class{knownApplications=new Map;allowedAppIds=new Set;accessActive=!1;allowAll=!1;hostWindowId=null;rememberApplications(t){this.knownApplications.clear();for(let e of t)this.knownApplications.set(e.id,e);if(this.allowAll)for(let e of t)this.allowedAppIds.add(e.id)}getKnownApplications(t){return t.map(e=>{let i=this.knownApplications.get(e);if(!i)throw new Error(`Unknown application id '${e}'. Call list_applications first.`);return i})}setHostWindowId(t){this.hostWindowId=t}tryGetKnownApplication(t){return this.knownApplications.get(t)}allKnownApplications(){return[...this.knownApplications.values()]}allowApplications(t,e=!1,i){this.accessActive=!0;for(let s of t)this.allowedAppIds.add(s);if(e){this.allowAll=!0;for(let s of this.knownApplications.keys())this.allowedAppIds.add(s)}return i&&(this.hostWindowId=i),this.getState()}areAllowedForAccess(t){return this.allowAll||t.every(e=>this.allowedAppIds.has(e))}getState(){return{accessActive:this.accessActive,allowedAppIds:[...this.allowedAppIds],allowAll:this.allowAll,hostWindowId:this.hostWindowId}}};function C(u){let t=new Map,e=[...u].sort((i,s)=>`${i.applicationName}\0${i.title}\0${i.windowId}`.localeCompare(`${s.applicationName}\0${s.title}\0${s.windowId}`));for(let i of e){let s=i.applicationName||i.title||i.applicationId,o=t.get(i.applicationId);o||(o={id:i.applicationId,displayName:s,applicationNames:i.applicationNames,windows:[]},t.set(i.applicationId,o)),o.windows.push({windowId:i.windowId,title:i.title,displayId:i.displayId,isMinimized:i.isMinimized})}return[...t.values()].sort((i,s)=>`${i.displayName}\0${i.id}`.localeCompare(`${s.displayName}\0${s.id}`))}function ot(u){return u.toLowerCase().split(/[^a-z0-9]+/).filter(t=>t.length>0)}function it(u){let t=new Set;for(let e of u)for(let i of ot(e))t.add(i);return t}function nt(u){return u.trim().toLowerCase()}function st(u){let t=new Set;for(let e of u){let i=nt(e);i&&t.add(i)}return t}var B=class{appRecords=new Map;updateApplications(t){for(let e of t){let i=[e.displayName,...e.applicationNames??[]],s=this.appRecords.get(e.id);if(s){for(let o of it(i))s.tokenBag.add(o);for(let o of st(i))s.exactNames.add(o);s.displayName=e.displayName}else this.appRecords.set(e.id,{appId:e.id,displayName:e.displayName,tokenBag:it(i),exactNames:st(i)})}}resolve(t){let e=this.appRecords.get(t);if(e)return[e];let i=nt(t);if(i){let n=[];for(let r of this.appRecords.values())r.exactNames.has(i)&&n.push(r);if(n.length>0)return n}let s=ot(t);if(s.length===0)return[];let o=[];for(let n of this.appRecords.values())s.every(r=>n.tokenBag.has(r))&&o.push(n);return o}allRecords(){return[...this.appRecords.values()]}};var It=new Set(["ctrl","control","shift","alt","option","super","meta","cmd","command","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","0","1","2","3","4","5","6","7","8","9","return","enter","tab","space","backspace","delete","forwarddelete","escape","esc","up","down","left","right","home","end","pageup","pagedown","insert","capslock","f1","f2","f3","f4","f5","f6","f7","f8","f9","f10","f11","f12","-","minus","=","equal","plus","[","]","\\",";","'",",",".","/","`"]),kt={arrowup:"up",arrowdown:"down",arrowleft:"left",arrowright:"right",page_up:"pageup",page_down:"pagedown",backslash:"\\",semicolon:";",slash:"/",grave:"`",bracketleft:"[",bracketright:"]",super_l:"super",gui:"super",win:"super",windows:"super",caps_lock:"capslock",caps:"capslock",del:"delete"};function Q(u){u==="+"?u="plus":u.length>2&&u.endsWith("++")&&(u=u.slice(0,-1)+"plus");let t=u.split("+").map(i=>i.trim().toLowerCase()).filter(Boolean);if(t.length===0)throw new Error("Key combo must contain at least one key.");return t.map(i=>{let s=kt[i]??i;if(!It.has(s))throw new Error(`Unknown key "${i}" in combo "${u}".`);return s}).join("+")}import*as A from"fs/promises";import*as at from"os";import*as H from"path";var V="<!-- LOG -->",F=class{logDir;logPath;startTime=performance.now();queue=Promise.resolve();constructor(){this.logDir=H.join(at.homedir(),".copilot","logs","computer-use"),this.logPath=H.join(this.logDir,`${this.timestamp()}.html`)}timestamp(){let t=new Date,e=(i,s=2)=>String(i).padStart(s,"0");return`${t.getFullYear()}${e(t.getMonth()+1)}${e(t.getDate())}-${e(t.getHours())}${e(t.getMinutes())}${e(t.getSeconds())}.${e(Math.floor(t.getMilliseconds()/10))}`}elapsed(){return`${((performance.now()-this.startTime)/1e3).toFixed(2)}s`}insert(t){return this.queue=this.queue.then(()=>this.write(t)).catch(()=>this.write(t)).catch(()=>{}),this.queue}async write(t){let e=!0;try{await A.access(this.logPath)}catch{e=!1}e||(await A.mkdir(this.logDir,{recursive:!0}),await A.writeFile(this.logPath,`<!DOCTYPE html>
|
|
2
2
|
<html>
|
|
3
3
|
<head>
|
|
4
4
|
<meta charset="utf-8">
|
|
@@ -22,19 +22,19 @@ import{McpServer as Kt}from"@modelcontextprotocol/sdk/server/mcp.js";import{Elic
|
|
|
22
22
|
<body>
|
|
23
23
|
<table>
|
|
24
24
|
<tr><th>Time</th><th>Level</th><th>Source</th><th>Action</th><th>Details</th></tr>
|
|
25
|
-
${
|
|
25
|
+
${V}
|
|
26
26
|
</table>
|
|
27
27
|
</body>
|
|
28
28
|
</html>
|
|
29
|
-
`));let i=await A.readFile(this.logPath,"utf-8");await A.writeFile(this.logPath,i.replace(
|
|
29
|
+
`));let i=await A.readFile(this.logPath,"utf-8");await A.writeFile(this.logPath,i.replace(V,t+V))}rowHtml(t,e,i,s){return`<tr><td class="time">${this.elapsed()}</td><td class="level level-${i}">${i}</td><td class="source">${s}</td><td class="action">${this.escapeHtml(t)}</td><td>${e}</td></tr>
|
|
30
30
|
`}escapeHtml(t){return t.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/"/g,""")}renderMarkdown(t){let e=t.split(`
|
|
31
|
-
`),i=[],s=[],o=()=>{s.length>0&&(i.push(`<ul style="margin:0;padding-left:1.2em">${s.map(n=>`<li>${this.escapeHtml(n)}</li>`).join("")}</ul>`),s=[])};for(let n of e)n.startsWith("- ")?s.push(n.slice(2)):(o(),n.length>0&&(i.length>0&&i.push("<br>"),i.push(this.escapeHtml(n))));return o(),i.join("")}log(t,e,i,s){this.insert(this.rowHtml(i,this.renderMarkdown(s),t,e))}logScreenshot(t,e,i,s){if(i){let o=s===void 0?"png":"jpg",n=`${this.timestamp()}.${o}`;A.mkdir(this.logDir,{recursive:!0}).then(()=>A.writeFile(F.join(this.logDir,n),i)),this.insert(this.rowHtml(t,`${this.escapeHtml(e)}<br><img src="${n}">`,"info","Computer"))}else this.insert(this.rowHtml(t,this.escapeHtml(e),"error","Computer"))}};var B=class{computer;logger;constructor(t,e){this.computer=t,this.logger=e}log(t,e){this.logger.log("info","Computer",t,e)}async checkPermissions(t){return this.computer.checkPermissions(t)}async requestPermissions(t){return this.computer.requestPermissions(t)}async capabilities(){let t=await this.computer.capabilities();return this.log("capabilities",JSON.stringify(t)),t}async click(t,e,i,s,o=""){await this.computer.click(t,e,i,s,o),this.log("click",`(${t}, ${e}) button=${i} count=${s}${o?` display=${o}`:""}`)}async move(t,e,i=""){await this.computer.move(t,e,i),this.log("move",`(${t}, ${e})${i?` display=${i}`:""}`)}async drag(t,e,i,s,o=""){await this.computer.drag(t,e,i,s,o),this.log("drag",`(${t}, ${e}) \u2192 (${i}, ${s})${o?` display=${o}`:""}`)}async mouseDown(t,e,i=""){await this.computer.mouseDown(t,e,i),this.log("mouseDown",`(${t}, ${e})${i?` display=${i}`:""}`)}async mouseUp(t,e,i=""){await this.computer.mouseUp(t,e,i),this.log("mouseUp",`(${t}, ${e})${i?` display=${i}`:""}`)}async type(t){await this.computer.type(t),this.log("type",`"${t}"`)}async key(t){await this.computer.key(t),this.log("key",t)}async scroll(t,e,i,s,o=""){await this.computer.scroll(t,e,i,s,o),this.log("scroll",`(${t}, ${e}) dx=${i} dy=${s}${o?` display=${o}`:""}`)}async cursorPosition(t=""){let e=await this.computer.cursorPosition(t);return this.log("cursorPosition",`(${e.x}, ${e.y})${t?` display=${t}`:""}`),e}async display(t=""){let e=await this.computer.display(t);return this.log("display",`${e.width}x${e.height}${t?` display=${t}`:""}`),e}async listDisplays(){let t=await this.computer.listDisplays(),e=t.map(i=>`${i.isPrimary?"*":""}${i.displayId} "${i.label}" ${i.width}x${i.height}`).join("; ");return this.log("listDisplays",`${t.length} displays${e?`: ${e}`:""}`),t}async listWindows(){let t=await this.computer.listWindows(),e=new Map;for(let s of t){let o=s.displayId||"(unknown)",n=e.get(o)??{visible:0,minimized:0};s.isMinimized?n.minimized+=1:n.visible+=1,e.set(o,n)}let i=[...e.entries()].sort(([s],[o])=>s.localeCompare(o)).map(([s,o])=>`${s}: ${o.visible} visible, ${o.minimized} minimized`).join("; ");return this.log("listWindows",`${t.length} windows${i?`: ${i}`:""}`),t}async getActiveWindow(){let t=await this.computer.getActiveWindow();return this.log("getActiveWindow",t?`${t.windowId} title="${t.title}"`:"null"),t}async activateApplication(t){let e=await this.computer.activateApplication(t);return this.log("activateApplication",`${t} -> ${e}`),e}async concealApplication(t){let e=await this.computer.concealApplication(t);return this.log("concealApplication",`${t} -> ${e}`),e}async restoreApplication(t){let e=await this.computer.restoreApplication(t);return this.log("restoreApplication",`${t} -> ${e}`),e}async activateWindow(t){let e=await this.computer.activateWindow(t);return this.log("activateWindow",`${t} -> ${e}`),e}async concealWindow(t){let e=await this.computer.concealWindow(t);return this.log("concealWindow",`${t} -> ${e}`),e}async restoreWindow(t){let e=await this.computer.restoreWindow(t);return this.log("restoreWindow",`${t} -> ${e}`),e}async getClipboard(){let t=await this.computer.getClipboard(),e=t.slice(0,200)+(t.length>200?"\u2026":"");return this.log("getClipboard",`"${e}"`),t}async setClipboard(t){await this.computer.setClipboard(t);let e=t.slice(0,200)+(t.length>200?"\u2026":"");this.log("setClipboard",`"${e}" (${t.length} chars)`)}lock(t){return this.computer.lock(t)}unlock(){this.computer.unlock()}async prepareForInput(t,e,i){let s=await this.computer.prepareForInput(t,e,i),o=i?` @ (${i.x},${i.y})`:"";return this.log("prepareForInput",`${t} -> ${s} (${e.length} apps)${o}`),s}async screenshot(t,e,i,s,o,n,r){let c=await this.computer.screenshot(t,e,i,s,o,n,r),a=s&&o?` (${s}x${o})`:"",l=n?` crop=[${n.join(",")}]`:"",u=`display=${t||"(primary)"}`,f=r===void 0?"":` quality=${r}`,h=`${u}${a}${l}${f}`;return this.logger.logScreenshot("screenshot",h,c,r),c}};import{createRequire as xt}from"module";import{dirname as _t,join as St}from"path";import{fileURLToPath as Pt}from"url";var Wt=xt(import.meta.url),Ct=_t(Pt(import.meta.url)),z=class p{constructor(t,e){this.native=t;t.setLogger?.((i,s,o)=>{e?.log(this.nativeLogLevel(i),"Driver",s,o)})}native;static create(t){let e=St(Ct,"prebuilds",`${process.platform}-${process.arch}`,"computer.node"),i;try{i=Wt(e)}catch(s){let o=s instanceof Error?s.message:String(s);throw new Error(`Native computer bindings not available for ${process.platform}-${process.arch}: ${o} (path: ${e})`,{cause:s})}return new p(i,t)}async checkPermissions(t){return this.native.checkPermissions(t)}async requestPermissions(t){this.native.requestPermissions(t)}async click(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,c]=this.toNative(n,t,e);this.native.click(r,c,i,s)}nativeLogLevel(t){switch(t){case 0:return"trace";case 1:return"debug";case 2:return"info";case 3:return"warn";case 4:return"error";default:return"info"}}async move(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.move(o,n)}async drag(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e),this.assertPointInBounds(n,i,s);let[r,c]=this.toNative(n,t,e),[a,l]=this.toNative(n,i,s);this.native.drag(r,c,a,l)}async mouseDown(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseDown(o,n)}async mouseUp(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseUp(o,n)}async type(t){this.native.type(t)}async key(t){this.native.key(t)}async scroll(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,c]=this.toNative(n,t,e);this.native.scroll(r,c,i,s)}async cursorPosition(t=""){let e=this.native.cursorPosition(),i=this.resolveCoordinateSpace(t),[s,o]=this.fromNative(i,e.x,e.y);return{x:s,y:o}}async display(t=""){let e=this.resolveCoordinateSpace(t);return{width:e.targetWidth,height:e.targetHeight}}async screenshot(t,e,i,s,o,n,r){let c=this.resolveCoordinateSpace(t,s,o),a=r??-1;if(n){this.assertRegionEdgeInBounds(c,n[0],n[1]),this.assertRegionEdgeInBounds(c,n[2],n[3]);let[l,u,f,h]=this.toNativeCrop(c,n[0],n[1],n[2],n[3]);return this.native.screenshot(t,e,i,0,0,l,u,f,h,a)}return this.native.screenshot(t,e,i,c.targetWidth,c.targetHeight,0,0,0,0,a)}async capabilities(){return{...this.native.capabilities(),canGetActiveWindow:typeof this.native.getActiveWindow=="function"}}async listDisplays(){return this.native.listDisplays().map(e=>{let{targetWidth:i,targetHeight:s}=this.targetSizeForDisplay(e.bounds.width,e.bounds.height);return{displayId:e.displayId,label:e.label,width:i,height:s,isPrimary:e.isPrimary}})}async listWindows(){return this.native.listWindows()}async getActiveWindow(){return this.native.getActiveWindow?this.native.getActiveWindow():null}async activateApplication(t){return this.native.activateApplication(t)}async concealApplication(t){return this.native.concealApplication(t)}async restoreApplication(t){return this.native.restoreApplication(t)}async activateWindow(t){return this.native.activateWindow(t)}async concealWindow(t){return this.native.concealWindow(t)}async restoreWindow(t){return this.native.restoreWindow(t)}async getClipboard(){return this.native.getClipboard()}async setClipboard(t){this.native.setClipboard(t)}lock(t){return this.native.lock?.(t)??!1}unlock(){this.native.unlock?.()}async prepareForInput(t,e,i){let s=Number.NaN,o=Number.NaN;if(i){let n=this.resolveCoordinateSpace(t);this.assertPointInBounds(n,i.x,i.y),[s,o]=this.toNative(n,i.x,i.y)}return this.native.prepareForInput(t,e,i?.blockedWindowIds??[],s,o)}resolveDisplay(t){let e=this.native.listDisplays();return t!==""?e.find(i=>i.displayId===t):e.find(i=>i.isPrimary)??e[0]}targetSizeForDisplay(t,e,i=0,s=0){if(i>0&&s>0)return{targetWidth:i,targetHeight:s};let r=Math.min(1,1568/Math.max(t,e),Math.sqrt(115e4/(t*e)));return{targetWidth:Math.floor(t*r),targetHeight:Math.floor(e*r)}}resolveCoordinateSpace(t,e=0,i=0){let s=this.resolveDisplay(t),o=s?.bounds.x??0,n=s?.bounds.y??0,r=s?void 0:this.native.display(),c=s?.bounds.width??r?.width??1,a=s?.bounds.height??r?.height??1,{targetWidth:l,targetHeight:u}=this.targetSizeForDisplay(c,a,e,i);return{originX:o,originY:n,displayWidth:c,displayHeight:a,scaleX:l/c,scaleY:u/a,targetWidth:l,targetHeight:u}}toNative(t,e,i){return[t.originX+e/t.scaleX,t.originY+i/t.scaleY]}toNativeCrop(t,e,i,s,o){return[Math.floor(e/t.scaleX),Math.floor(i/t.scaleY),Math.ceil(s/t.scaleX),Math.ceil(o/t.scaleY)]}assertPointInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>=t.targetWidth||i>=t.targetHeight)throw new RangeError(`Coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}assertRegionEdgeInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>t.targetWidth||i>t.targetHeight)throw new RangeError(`Crop coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}fromNative(t,e,i){return[Math.round((e-t.originX)*t.scaleX),Math.round((i-t.originY)*t.scaleY)]}};var C=.8,Dt=250,S={readOnlyHint:!0,destructiveHint:!1,openWorldHint:!1},b={readOnlyHint:!1,destructiveHint:!0,openWorldHint:!0},V={readOnlyHint:!1,destructiveHint:!1,openWorldHint:!1},w=" Requires `request_access`.",at=" If the target app is missing, call `list_applications` then `request_access`.",T=d.coerce.number().transform(Math.round),M=d.object({x:T.describe("Horizontal pixel coordinate."),y:T.describe("Vertical pixel coordinate.")}),Nt=d.object({x1:T.describe("Left edge of the region in screenshot pixel coordinates."),y1:T.describe("Top edge of the region in screenshot pixel coordinates."),x2:T.describe("Right edge of the region in screenshot pixel coordinates."),y2:T.describe("Bottom edge of the region in screenshot pixel coordinates.")}).refine(({x1:p,x2:t})=>t>p,{message:"x2 must be greater than x1.",path:["x2"]}).refine(({y1:p,y2:t})=>t>p,{message:"y2 must be greater than y1.",path:["y2"]}),Rt=d.string().min(1).describe("Application name or stable app id from `list_applications`. Examples: 'Google Chrome', 'Microsoft Outlook', 'app.windows.abc123'."),qt=d.object({display_id:d.string().optional().describe("Display id to select. Omit or pass an empty string to use the default display.")}),Mt=d.object({display_id:d.string().optional().describe("Optional display id to filter by. When provided, only apps with at least one non-minimized window on that display are returned.")}),pt=M.optional().describe("{x, y} pixel coordinate. Clicks at current cursor position if omitted."),dt=M.describe("{x, y} pixel coordinate to move the cursor to."),ut=M.describe("{x, y} pixel coordinate to start the drag from."),ht=M.describe("{x, y} pixel coordinate to drag to."),mt=d.string().describe("The text to type."),gt=d.string().describe("Key combo to press (e.g. 'Return', 'ctrl+s', 'alt+Tab')."),wt=d.enum(["up","down","left","right"]).describe("Direction to scroll."),yt=M.optional().describe("{x, y} pixel coordinate to scroll at. Scrolls at current cursor position if omitted."),ft=d.number().int().nonnegative().max(100).optional().describe("Number of scroll clicks (default 3)."),Lt=d.number().nonnegative().max(100).describe("Seconds to wait (max 100). Use after a screenshot shows the UI is not in the expected state yet, but an operation may still finish."),bt=d.string().describe("The text to copy to the clipboard."),jt=d.object({action:d.enum(["left_click","right_click","middle_click","double_click","triple_click"]),coordinate:pt}),Et=d.object({action:d.literal("mouse_move"),coordinate:dt}),Ht=d.object({action:d.literal("left_click_drag"),start_coordinate:ut,coordinate:ht}),Ft=d.object({action:d.literal("left_mouse_down")}),Ut=d.object({action:d.literal("left_mouse_up")}),Bt=d.object({action:d.literal("type"),text:mt}),zt=d.object({action:d.literal("key"),text:gt}),Ot=d.object({action:d.literal("scroll"),scroll_direction:wt,coordinate:yt,scroll_amount:ft}),Yt=d.object({action:d.literal("set_clipboard"),text:bt}),Xt=d.discriminatedUnion("action",[jt,Et,Ht,Ft,Ut,Bt,zt,Ot,Yt]),Gt=d.object({actions:d.array(Xt).min(1).describe("Ordered actions. Do not include waits; call the separate `wait` tool only after observing a screenshot that is not ready yet.")});function rt(p,t){let e=p.length===0?["","Computer Use runs on your actual desktop and can send mouse and keyboard input.","","Computer Use wants to control your desktop for this session.","","Apps that are not allowed may be hidden."]:["","Computer Use runs on your actual desktop and can send mouse and keyboard input to the apps you share.","","Computer Use wants to control these apps:","",...p.map(i=>`- ${i}`),"","Apps that are not allowed may be hidden."];return t&&e.push("","Reason:",t),e.join(`
|
|
32
|
-
`)}
|
|
33
|
-
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:`Grant ${e.join(" and ")} permission${e.length>1?"s":""}`,oneOf:[{const:"open",title:"1. Open System Settings"},{const:"done",title:"2. I've granted the permissions"},{const:"skip",title:"3. Skip (things may not work)"}]}},required:["action"]}}),
|
|
34
|
-
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:"Confirm permissions",oneOf:[{const:"done",title:"1. I've granted the permissions"},{const:"skip",title:"2. Skip (things may not work)"}]}},required:["action"]}});if((a.action==="accept"?a.content?.action:void 0)!=="done")break}else if(c!=="done")break;if(i=await t.checkPermissions("accessibility"),s=await t.checkPermissions("screen"),!i||!s){let a=[];throw i||a.push("Accessibility"),s||a.push("Screen Recording"),new Error(`${a.join(" and ")} permission${a.length>1?"s are":" is"} still not detected. This usually means you need to restart your terminal after granting the permission. Please quit and reopen your terminal, then try again.`)}}break}}}async activeDisplayId(t){return this.selectedDisplay!==""?this.selectedDisplay:(await t.listDisplays()).find(i=>i.isPrimary)?.displayId??""}async requireAccess(){let t=this.accessStore.getState();if(!t.accessActive)throw new Error("No access session is active. Call request_access first to start a session.");let e=await this.computer();if(t.allowAll&&(await this.capabilities()).canListWindows){let{windows:s}=await this.filterHostWindow(e),o=W(s);this.accessStore.rememberApplications(o),this.appNameResolver.updateApplications(o),t=this.accessStore.getState()}return{computer:e,allowedAppIds:t.allowedAppIds,hostWindowId:t.hostWindowId}}async hostWindowId(t){let e=this.accessStore.getState().hostWindowId;if(e)return e;if(!(await this.capabilities()).canListWindows)return null;let s=await t.getActiveWindow();return s&&(await t.listWindows()).some(n=>n.windowId===s.windowId)?s.windowId:null}async showHostWindow(t){let e=await this.hostWindowId(t);if(e)try{await t.activateWindow(e)}catch{}}async filterHostWindow(t,e){let i=e??await t.listWindows(),s=await this.hostWindowId(t);return{windows:s?i.filter(o=>o.windowId!==s):i,hostWindowId:s}}async prepareForInput(t,e,i){let s=await this.capabilities();if(!s.canListWindows)return;let o=await this.activeDisplayId(t),n=i&&s.canHitTest;if(e.length===0&&!n)return;if(!await t.prepareForInput(o,e,n?i:void 0))throw n?new Error("Input blocked: a disallowed app at the target could not be hidden. Call `list_applications` to see what's there, then `request_access` to allow it."):new Error("No allowed visible windows are available on the selected display.")}async validateActiveWindow(t,e){if(!(await this.capabilities()).canGetActiveWindow)return;let s=await t.getActiveWindow();if(s&&!e.includes(s.applicationId))throw new Error(`Keyboard input blocked: the focused application ("${s.applicationName}") is not allowed. Click an allowed application first, or call \`request_access\` to allow it.`)}async list_displays(){let t=await this.computer(),e=await t.listDisplays(),i=await this.activeDisplayId(t);return this.toJson({displays:e,selectedDisplay:i})}async select_display({display_id:t}){let e=t??"",i=await this.computer(),s=await i.listDisplays();if(e!==""&&!s.some(r=>r.displayId===e))throw new Error(`Unknown display id '${e}'. Call list_displays first.`);this.selectedDisplay=e;let o=await this.activeDisplayId(i),n=s.find(r=>r.displayId===o)??null;return this.toJson({selectedDisplay:o,status:"selected",message:e===""?"Using the default display for screenshots, zoom, cursor position, and coordinate-based actions.":`Selected display '${n?.label??e}' for screenshots, zoom, cursor position, and coordinate-based actions.`})}async list_applications({display_id:t}={}){let e=await this.computer(),{windows:i}=await this.filterHostWindow(e),s=W(i);this.accessStore.rememberApplications(s),this.appNameResolver.updateApplications(s);let o=t??"";if(o!==""&&!(await e.listDisplays()).some(l=>l.displayId===o))throw new Error(`Unknown display id '${o}'. Call list_displays first.`);let n=o===""?i:i.filter(a=>!a.isMinimized&&a.displayId===o),r=W(n),c=await this.activeDisplayId(e);return this.toJson({selectedDisplay:c,applications:r})}async request_access({apps:t,reason:e}){this.logEvent("request_access",`Start: apps=[${t.join(", ")}]${e?` reason="${e}"`:""}`),await this.requestPermission();let i={const:"allow",title:"Allow"},s={const:"allow_all",title:"Allow all apps (don't ask again)"},o={const:"deny",title:"Deny (Esc)"},n,r=async(m,y,D)=>{this.logEvent("request_access",`End: allowed=${m} allowAll=${y.allowAll} selectedDisplay=${this.selectedDisplay||"(default)"} hostWindowId=${y.hostWindowId??"null"} message="${D}"`);let $=n??(y.allowAll?void 0:y.allowedAppIds.map(v=>{let x=this.accessStore.tryGetKnownApplication(v);return{appId:v,name:x?.displayName,displayIds:vt(x?.windows??[])}})),N=this.toJson({allowed:m,allowAll:y.allowAll,...$?{allowedApps:$}:{},selectedDisplay:this.selectedDisplay||void 0,message:D});if(!m)return N;let P=await this.computer();this.lock();let G=y.hostWindowId?[y.hostWindowId]:[];return this.addScreenshot(N,P,y.allowedAppIds,G)};if(this.options.yolo){let m=this.accessStore.allowApplications([],!0);return await r(!0,m,"YOLO mode is enabled. Auto-allowing full desktop access for this session.")}let c=await this.capabilities();if(!this.server.server.getClientCapabilities()?.elicitation?.form){let m=this.accessStore.allowApplications([],!0);return await r(!0,m,"Elicitation is not supported by this client. Auto-allowing full desktop access for this session.")}let l,u,f,h,Y;if(c.canListWindows){let m=await this.computer(),y=this.accessStore.getState().hostWindowId,D=await m.listWindows(),$=y?D.filter(g=>g.windowId!==y):D;Y=$;let N=W($);this.accessStore.rememberApplications(N),this.appNameResolver.updateApplications(N);let P=t&&t.length>0?"":await this.activeDisplayId(m);P&&(this.selectedDisplay=P);let G=new Set(y?[y]:[]),v;if(t&&t.length>0){let g=new Map;for(let k of t){let q=this.appNameResolver.resolve(k);if(q.length===0)throw new Error(`No matching application found for '${k}'. Call list_applications to all running apps and their IDs.`);for(let _ of q)if(!g.has(_.appId)){let j=this.accessStore.tryGetKnownApplication(_.appId);g.set(_.appId,j??{id:_.appId,displayName:_.displayName,windows:[]})}}v=[...g.values()];let At=new Set(g.keys()),L=new Map;for(let k of $)At.has(k.applicationId)&&!k.isMinimized&&k.displayId&&L.set(k.displayId,(L.get(k.displayId)??0)+1);if(L.size>0){let k="",q=0;for(let[_,j]of L)j>q&&(k=_,q=j);this.selectedDisplay=k}n=lt(v,$)}else v=W($.filter(g=>!g.isMinimized&&(P===""||g.displayId===P)&&!G.has(g.windowId)));let x=v.map(g=>g.id),K=this.accessStore.getState(),R=v.length;u=["allow","allow_all"],f=x.length===0?K.allowAll||K.accessActive:this.accessStore.areAllowedForAccess(x),h=x.length===0?K.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.":R===1?`'${v[0].displayName}' is already shared for this session.`:"The requested apps are already shared for this session.",l={choices:[i,s,o],message:rt(v.map(g=>g.displayName),e),denyMessage:R===0?"The user declined desktop access.":R===1?`The user declined to share '${v[0].displayName}'.`:"The user declined to share the requested apps.",allow:g=>this.accessStore.allowApplications(x,g==="allow_all"),allowedMessage:g=>g==="allow_all"?"All current and future apps are allowed for the rest of this session, so you do not need to call request_access again.":R===0?"Desktop access is active for this session.":R===1?`Access session started for '${v[0].displayName}'.`:"Access session started for the requested apps."}}else{let m=this.accessStore.getState();u=["allow_all"],f=m.allowAll||m.accessActive,h=m.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.",l={choices:[s,o],message:rt([],e),denyMessage:"The user declined desktop access.",allow:()=>this.accessStore.allowApplications([],!0),allowedMessage:()=>"Access session started for the desktop. All current and future apps are allowed for the rest of this session, so you do not need to call request_access again."}}if(f){let m=this.accessStore.getState();return await r(!0,m,h)}this.accessStore.getState().hostWindowId&&await this.showHostWindow(await this.computer());let X=await this.server.server.request({method:"elicitation/create",params:{mode:"form",message:l.message,requestedSchema:{type:"object",properties:{decision:{type:"string",title:"Allow access for this session?",oneOf:l.choices}},required:["decision"]}}},Tt),kt=await this.computer(),Z=await this.hostWindowId(kt);Z&&this.accessStore.setHostWindowId(Z);let I=X.action==="accept"?X.content?.decision:void 0;if(X.action!=="accept"||!I||typeof I!="string"||!u.includes(I)){let m=this.accessStore.getState(),y=typeof I=="string"&&I!==o.const?`The user did not approve this access request. User response: ${I}`:l.denyMessage;return await r(!1,m,y)}let tt=l.allow(I);return tt.allowAll&&Y&&(n=lt(this.accessStore.allKnownApplications(),Y)),await r(!0,tt,l.allowedMessage(I))}async screenshot(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[];return this.captureScreenshot(t,e,s)}logScreenshotFilter(t,e){let i=r=>{let c=this.accessStore.tryGetKnownApplication(r);return c?`${c.displayName} (${r})`:r},s=r=>{let c=r.includes("|")?r.split("|")[1]:r;return i(c)},o=(r,c)=>c.length===1?`${r}: ${c[0]}`:`${r}:
|
|
35
|
-
${
|
|
31
|
+
`),i=[],s=[],o=()=>{s.length>0&&(i.push(`<ul style="margin:0;padding-left:1.2em">${s.map(n=>`<li>${this.escapeHtml(n)}</li>`).join("")}</ul>`),s=[])};for(let n of e)n.startsWith("- ")?s.push(n.slice(2)):(o(),n.length>0&&(i.length>0&&i.push("<br>"),i.push(this.escapeHtml(n))));return o(),i.join("")}log(t,e,i,s){this.insert(this.rowHtml(i,this.renderMarkdown(s),t,e))}logScreenshot(t,e,i,s){if(i){let o=s===void 0?"png":"jpg",n=`${this.timestamp()}.${o}`;A.mkdir(this.logDir,{recursive:!0}).then(()=>A.writeFile(H.join(this.logDir,n),i)),this.insert(this.rowHtml(t,`${this.escapeHtml(e)}<br><img src="${n}">`,"info","Computer"))}else this.insert(this.rowHtml(t,this.escapeHtml(e),"error","Computer"))}};var U=class{computer;logger;constructor(t,e){this.computer=t,this.logger=e}log(t,e){this.logger.log("info","Computer",t,e)}async checkPermissions(t){return this.computer.checkPermissions(t)}async requestPermissions(t){return this.computer.requestPermissions(t)}async capabilities(){let t=await this.computer.capabilities();return this.log("capabilities",JSON.stringify(t)),t}async click(t,e,i,s,o=""){await this.computer.click(t,e,i,s,o),this.log("click",`(${t}, ${e}) button=${i} count=${s}${o?` display=${o}`:""}`)}async move(t,e,i=""){await this.computer.move(t,e,i),this.log("move",`(${t}, ${e})${i?` display=${i}`:""}`)}async drag(t,e,i,s,o=""){await this.computer.drag(t,e,i,s,o),this.log("drag",`(${t}, ${e}) \u2192 (${i}, ${s})${o?` display=${o}`:""}`)}async mouseDown(t,e,i=""){await this.computer.mouseDown(t,e,i),this.log("mouseDown",`(${t}, ${e})${i?` display=${i}`:""}`)}async mouseUp(t,e,i=""){await this.computer.mouseUp(t,e,i),this.log("mouseUp",`(${t}, ${e})${i?` display=${i}`:""}`)}async type(t){await this.computer.type(t),this.log("type",`"${t}"`)}async key(t){await this.computer.key(t),this.log("key",t)}async scroll(t,e,i,s,o=""){await this.computer.scroll(t,e,i,s,o),this.log("scroll",`(${t}, ${e}) dx=${i} dy=${s}${o?` display=${o}`:""}`)}async cursorPosition(t=""){let e=await this.computer.cursorPosition(t);return this.log("cursorPosition",`(${e.x}, ${e.y})${t?` display=${t}`:""}`),e}async display(t=""){let e=await this.computer.display(t);return this.log("display",`${e.width}x${e.height}${t?` display=${t}`:""}`),e}async listDisplays(){let t=await this.computer.listDisplays(),e=t.map(i=>`${i.isPrimary?"*":""}${i.displayId} "${i.label}" ${i.width}x${i.height}`).join("; ");return this.log("listDisplays",`${t.length} displays${e?`: ${e}`:""}`),t}async listWindows(){let t=await this.computer.listWindows(),e=new Map;for(let s of t){let o=s.displayId||"(unknown)",n=e.get(o)??{visible:0,minimized:0};s.isMinimized?n.minimized+=1:n.visible+=1,e.set(o,n)}let i=[...e.entries()].sort(([s],[o])=>s.localeCompare(o)).map(([s,o])=>`${s}: ${o.visible} visible, ${o.minimized} minimized`).join("; ");return this.log("listWindows",`${t.length} windows${i?`: ${i}`:""}`),t}async windowAtPoint(t,e,i){let s=await this.computer.windowAtPoint(t,e,i);return this.log("windowAtPoint",`display=${t||"(primary)"} (${e}, ${i}) -> ${s?`${s.windowId} title="${s.title}"`:"null"}`),s}async getActiveWindow(){let t=await this.computer.getActiveWindow();return this.log("getActiveWindow",t?`${t.windowId} title="${t.title}"`:"null"),t}async activateApplication(t){let e=await this.computer.activateApplication(t);return this.log("activateApplication",`${t} -> ${e}`),e}async concealApplication(t){let e=await this.computer.concealApplication(t);return this.log("concealApplication",`${t} -> ${e}`),e}async restoreApplication(t){let e=await this.computer.restoreApplication(t);return this.log("restoreApplication",`${t} -> ${e}`),e}async activateWindow(t){let e=await this.computer.activateWindow(t);return this.log("activateWindow",`${t} -> ${e}`),e}async concealWindow(t){let e=await this.computer.concealWindow(t);return this.log("concealWindow",`${t} -> ${e}`),e}async restoreWindow(t){let e=await this.computer.restoreWindow(t);return this.log("restoreWindow",`${t} -> ${e}`),e}async getClipboard(){let t=await this.computer.getClipboard(),e=t.slice(0,200)+(t.length>200?"\u2026":"");return this.log("getClipboard",`"${e}"`),t}async setClipboard(t){await this.computer.setClipboard(t);let e=t.slice(0,200)+(t.length>200?"\u2026":"");this.log("setClipboard",`"${e}" (${t.length} chars)`)}lock(t){return this.computer.lock(t)}unlock(){this.computer.unlock()}async prepareForInput(t,e,i){let s=await this.computer.prepareForInput(t,e,i),o=i?` @ (${i.x},${i.y})`:"";return this.log("prepareForInput",`${t} -> ${s} (${e.length} apps)${o}`),s}async screenshot(t,e,i,s,o,n,r){let l=await this.computer.screenshot(t,e,i,s,o,n,r),a=s&&o?` (${s}x${o})`:"",c=n?` crop=[${n.join(",")}]`:"",d=`display=${t||"(primary)"}`,b=r===void 0?"":` quality=${r}`,f=`${d}${a}${c}${b}`;return this.logger.logScreenshot("screenshot",f,l,r),l}};import{createRequire as $t}from"module";import{dirname as xt,join as Pt}from"path";import{fileURLToPath as Wt}from"url";var z=class u{constructor(t,e){this.native=t;t.setLogger?.((i,s,o)=>{e?.log(this.nativeLogLevel(i),"Driver",s,o)})}native;static create(t){let e=Pt(xt(Wt(import.meta.url)),"prebuilds",`${process.platform}-${process.arch}`,"computer.node"),i;try{i=$t(import.meta.url)(e)}catch(s){let o=s instanceof Error?s.message:String(s);throw new Error(`Native computer bindings not available for ${process.platform}-${process.arch}: ${o} (path: ${e})`,{cause:s})}return new u(i,t)}async checkPermissions(t){return this.native.checkPermissions(t)}async requestPermissions(t){this.native.requestPermissions(t)}async click(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,l]=this.toNative(n,t,e);this.native.click(r,l,i,s)}nativeLogLevel(t){switch(t){case 0:return"trace";case 1:return"debug";case 2:return"info";case 3:return"warn";case 4:return"error";default:return"info"}}async move(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.move(o,n)}async drag(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e),this.assertPointInBounds(n,i,s);let[r,l]=this.toNative(n,t,e),[a,c]=this.toNative(n,i,s);this.native.drag(r,l,a,c)}async mouseDown(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseDown(o,n)}async mouseUp(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseUp(o,n)}async type(t){this.native.type(t)}async key(t){this.native.key(t)}async scroll(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,l]=this.toNative(n,t,e);this.native.scroll(r,l,i,s)}async cursorPosition(t=""){let e=this.native.cursorPosition(),i=this.resolveCoordinateSpace(t),[s,o]=this.fromNative(i,e.x,e.y);return{x:s,y:o}}async display(t=""){let e=this.resolveCoordinateSpace(t);return{width:e.targetWidth,height:e.targetHeight}}async screenshot(t,e,i,s,o,n,r){let l=this.resolveCoordinateSpace(t,s,o),a=r??-1;if(n){this.assertRegionEdgeInBounds(l,n[0],n[1]),this.assertRegionEdgeInBounds(l,n[2],n[3]);let[c,d,b,f]=this.toNativeCrop(l,n[0],n[1],n[2],n[3]);return this.native.screenshot(t,e,i,0,0,c,d,b,f,a)}return this.native.screenshot(t,e,i,l.targetWidth,l.targetHeight,0,0,0,0,a)}async capabilities(){let t=this.native.capabilities();return{...t,canGetActiveWindow:typeof this.native.getActiveWindow=="function",canLocateWindowAtPoint:typeof this.native.windowAtPoint=="function"&&!!t.canLocateWindowAtPoint}}async listDisplays(){return this.native.listDisplays().map(e=>{let{targetWidth:i,targetHeight:s}=this.targetSizeForDisplay(e.bounds.width,e.bounds.height);return{displayId:e.displayId,label:e.label,width:i,height:s,isPrimary:e.isPrimary}})}async listWindows(){return this.native.listWindows()}async windowAtPoint(t,e,i){if(!this.native.windowAtPoint)return null;let s=this.resolveCoordinateSpace(t);this.assertPointInBounds(s,e,i);let[o,n]=this.toNative(s,e,i);return this.native.windowAtPoint(t,o,n)}async getActiveWindow(){return this.native.getActiveWindow?this.native.getActiveWindow():null}async activateApplication(t){return this.native.activateApplication(t)}async concealApplication(t){return this.native.concealApplication(t)}async restoreApplication(t){return this.native.restoreApplication(t)}async activateWindow(t){return this.native.activateWindow(t)}async concealWindow(t){return this.native.concealWindow(t)}async restoreWindow(t){return this.native.restoreWindow(t)}async getClipboard(){return this.native.getClipboard()}async setClipboard(t){this.native.setClipboard(t)}lock(t){return this.native.lock?.(t)??!1}unlock(){this.native.unlock?.()}async prepareForInput(t,e,i){let s=Number.NaN,o=Number.NaN;if(i){let n=this.resolveCoordinateSpace(t);this.assertPointInBounds(n,i.x,i.y),[s,o]=this.toNative(n,i.x,i.y)}return this.native.prepareForInput(t,e,i?.blockedWindowIds??[],s,o)}resolveDisplay(t){let e=this.native.listDisplays();return t!==""?e.find(i=>i.displayId===t):e.find(i=>i.isPrimary)??e[0]}targetSizeForDisplay(t,e,i=0,s=0){if(i>0&&s>0)return{targetWidth:i,targetHeight:s};let r=Math.min(1,1568/Math.max(t,e),Math.sqrt(115e4/(t*e)));return{targetWidth:Math.floor(t*r),targetHeight:Math.floor(e*r)}}resolveCoordinateSpace(t,e=0,i=0){let s=this.resolveDisplay(t),o=s?.bounds.x??0,n=s?.bounds.y??0,r=s?void 0:this.native.display(),l=s?.bounds.width??r?.width??1,a=s?.bounds.height??r?.height??1,{targetWidth:c,targetHeight:d}=this.targetSizeForDisplay(l,a,e,i);return{originX:o,originY:n,displayWidth:l,displayHeight:a,scaleX:c/l,scaleY:d/a,targetWidth:c,targetHeight:d}}toNative(t,e,i){return[t.originX+e/t.scaleX,t.originY+i/t.scaleY]}toNativeCrop(t,e,i,s,o){return[Math.floor(e/t.scaleX),Math.floor(i/t.scaleY),Math.ceil(s/t.scaleX),Math.ceil(o/t.scaleY)]}assertPointInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>=t.targetWidth||i>=t.targetHeight)throw new RangeError(`Coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}assertRegionEdgeInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>t.targetWidth||i>t.targetHeight)throw new RangeError(`Crop coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}fromNative(t,e,i){return[Math.round((e-t.originX)*t.scaleX),Math.round((i-t.originY)*t.scaleY)]}};var _=.8,Ct=250,W={readOnlyHint:!0,destructiveHint:!1,openWorldHint:!1},I={readOnlyHint:!1,destructiveHint:!0,openWorldHint:!0},Z={readOnlyHint:!1,destructiveHint:!1,openWorldHint:!1},w=" Requires `request_access`.",rt=" If the target app is missing, call `list_applications` then `request_access`.",T=p.coerce.number().transform(Math.round),q=p.object({x:T.describe("Horizontal pixel coordinate."),y:T.describe("Vertical pixel coordinate.")}),_t=p.object({x1:T.describe("Left edge of the region in screenshot pixel coordinates."),y1:T.describe("Top edge of the region in screenshot pixel coordinates."),x2:T.describe("Right edge of the region in screenshot pixel coordinates."),y2:T.describe("Bottom edge of the region in screenshot pixel coordinates.")}).refine(({x1:u,x2:t})=>t>u,{message:"x2 must be greater than x1.",path:["x2"]}).refine(({y1:u,y2:t})=>t>u,{message:"y2 must be greater than y1.",path:["y2"]}),Tt=p.string().min(1).describe("Application name or stable app id from `list_applications`. Examples: 'Google Chrome', 'Microsoft Outlook', 'app.windows.abc123'."),Nt=p.object({display_id:p.string().optional().describe("Display id to select. Omit or pass an empty string to use the default display.")}),Dt=p.object({display_id:p.string().optional().describe("Optional display id to filter by. When provided, only apps with at least one non-minimized window on that display are returned.")}),lt=q.optional().describe("{x, y} pixel coordinate. Clicks at current cursor position if omitted."),ct=p.enum(["left","right","middle"]).optional().describe("Mouse button to click (default left)."),pt=p.coerce.number().int().min(1).max(3).optional().describe("Number of clicks: 1 single, 2 double, 3 triple (default 1)."),dt=q.describe("{x, y} pixel coordinate to move the cursor to."),ut=q.describe("{x, y} pixel coordinate to start the drag from."),ht=q.describe("{x, y} pixel coordinate to drag to."),mt=p.string().describe("The text to type."),gt=p.string().describe("Key combo to press (e.g. 'Return', 'ctrl+s', 'alt+Tab')."),wt=p.enum(["up","down","left","right"]).describe("Direction to scroll."),yt=q.optional().describe("{x, y} pixel coordinate to scroll at. Scrolls at current cursor position if omitted."),bt=p.number().int().nonnegative().max(100).optional().describe("Number of scroll clicks (default 3)."),Mt=p.number().nonnegative().max(100).describe("Seconds to wait (max 100). Use after a screenshot shows the UI is not in the expected state yet, but an operation may still finish."),ft=p.string().describe("The text to copy to the clipboard."),Rt=p.object({action:p.literal("click"),coordinate:lt,button:ct,count:pt}),qt=p.object({action:p.literal("mouse_move"),coordinate:dt}),Lt=p.object({action:p.literal("left_click_drag"),start_coordinate:ut,coordinate:ht}),Et=p.object({action:p.literal("left_mouse_down")}),jt=p.object({action:p.literal("left_mouse_up")}),Bt=p.object({action:p.literal("type"),text:mt}),Ht=p.object({action:p.literal("key"),text:gt}),Ft=p.object({action:p.literal("scroll"),scroll_direction:wt,coordinate:yt,scroll_amount:bt}),Ut=p.object({action:p.literal("set_clipboard"),text:ft}),zt=p.discriminatedUnion("action",[Rt,qt,Lt,Et,jt,Bt,Ht,Ft,Ut]),Ot=p.object({actions:p.array(zt).min(1).describe("Ordered actions. Do not include waits; call the separate `wait` tool only after observing a screenshot that is not ready yet.")}),O=class u{constructor(t={}){this.options=t;process.env.DEBUG&&(this.logger=new F)}options;_computer;_caps;_locked=null;_safetyTimer;static SAFETY_TIMEOUT_MS=180*1e3;server;logger;accessStore=new j;appNameResolver=new B;lastUnsharedWindowsNote=null;selectedDisplay="";toText(t,e){let i=[{type:"text",text:t}];return e?{content:i,isError:!0}:{content:i}}toJson(t){return this.toText(JSON.stringify(t,null,2))}toImage(t,e,i){return t?{content:[{type:"image",data:t.toString("base64"),mimeType:this.imageMimeType(i),_meta:{screenshot:!0}}]}:this.toText(e,!0)}buildAccessMessage(t,e){let i=t.length===0?["","Computer Use runs on your actual desktop and can send mouse and keyboard input.","","Computer Use wants to control your desktop for this session.","","Apps that are not allowed may be hidden."]:["","Computer Use runs on your actual desktop and can send mouse and keyboard input to the apps you share.","","Computer Use wants to control these apps:","",...t.map(s=>`- ${s}`),"","Apps that are not allowed may be hidden."];return e&&i.push("","Reason:",e),i.join(`
|
|
32
|
+
`)}imageMimeType(t){return t!==void 0?"image/jpeg":"image/png"}collectDisplayIds(t,e){let i=new Set;for(let s of t)!s.isMinimized&&s.displayId&&(!e||s.applicationId===e)&&i.add(s.displayId);return[...i]}buildAllowedAppsInfo(t,e){return t.map(i=>({appId:i.id,name:i.displayName,displayIds:this.collectDisplayIds(e,i.id)}))}connect(t){this.server=t;let e=p.object({method:p.literal("notifications/copilot"),params:p.object({type:p.string()}).passthrough()});t.server.setNotificationHandler(e,s=>{switch(s.params.type){case"assistant.turn_start":this._locked===null&&(this._locked=!1),this.unlock();break;case"assistant.turn_end":this.unlock();break}}),t.registerTool("list_displays",{description:"List available displays and the currently selected display used for screenshots, zoom, cursor position, and coordinate-based actions.",annotations:W},()=>this.list_displays()),t.registerTool("select_display",{description:"Select the active display used for screenshots, zoom, cursor position, and coordinate-based actions. Omit display_id or pass an empty string to use the default display.",inputSchema:Nt.shape,annotations:Z},s=>this.select_display(s)),t.registerTool("list_applications",{description:"List running apps and `selectedDisplay`. Use for discovery only; `request_access` can resolve named apps directly.",inputSchema:Dt.shape,annotations:W},s=>this.list_applications(s)),t.registerTool("request_access",{description:"Request app access by friendly name or appId; pass [] for all visible apps. For a named app, call this directly; it finds the app and selects its display. Returns JSON (`allowed`, `allowAll`, `message`, `allowedApps?`, `selectedDisplay?`) and, when allowed, a screenshot. `allowAll=true` covers future apps for this session.",inputSchema:{apps:p.array(Tt).describe('Applications to allow. Each entry can be a friendly name (e.g. "Outlook") or a stable appId, if already known. Pass an empty array to allow all apps on the selected display.'),reason:p.string().min(1).optional().describe("Optional reason text shown in the access dialog.")},annotations:Z},(s,o)=>this.request_access(s,o.signal)),t.registerTool("screenshot",{description:"Capture the current filtered screen. Use returned post-action screenshots when available. "+rt+w,annotations:W},()=>this.screenshot()),t.registerTool("cursor_position",{description:"Get the current cursor position in pixel coordinates. Returns {x, y}."+w,annotations:W},()=>this.cursor_position());let i={coordinate:lt,button:ct,count:pt};t.registerTool("click",{description:"Click a mouse button. Defaults to a single left click; use `button` for right/middle click and `count` 2 or 3 for double/triple-click. Optionally move to a coordinate first."+w,inputSchema:i,annotations:I},s=>this.click(s)),t.registerTool("mouse_move",{description:"Move the mouse cursor to the specified pixel coordinate."+w,inputSchema:{coordinate:dt},annotations:I},s=>this.mouse_move(s)),t.registerTool("left_click_drag",{description:"Click and drag from a start coordinate to an end coordinate."+w,inputSchema:{start_coordinate:ut,coordinate:ht},annotations:I},s=>this.left_click_drag(s)),t.registerTool("left_mouse_down",{description:"Press and hold the left mouse button at the current cursor position."+w,annotations:I},()=>this.left_mouse_down()),t.registerTool("left_mouse_up",{description:"Release the left mouse button at the current cursor position."+w,annotations:I},()=>this.left_mouse_up()),t.registerTool("type",{description:"Type a string for text input fields. Use `key` for physical-key controls."+w,inputSchema:{text:mt},annotations:I},s=>this.type(s)),t.registerTool("key",{description:"Press a key or combo, e.g. 'Return', 'ctrl+s', 'alt+Tab', '1'."+w,inputSchema:{text:gt},annotations:I},s=>this.key(s)),t.registerTool("scroll",{description:"Scroll the screen in a given direction at an optional coordinate."+w,inputSchema:{scroll_direction:wt,coordinate:yt,scroll_amount:bt},annotations:I},s=>this.scroll(s)),t.registerTool("wait",{description:"Pause, then return an updated screenshot if access is active. Use after a screenshot shows the UI is not ready yet.",inputSchema:{duration:Mt},annotations:{...W,idempotentHint:!0}},s=>this.wait(s)),t.registerTool("get_clipboard",{description:"Get the current text contents of the system clipboard."+w,annotations:W},()=>this.get_clipboard()),t.registerTool("set_clipboard",{description:"Set the system clipboard to the specified text."+w,inputSchema:{text:ft},annotations:Z},s=>this.set_clipboard(s)),t.registerTool("zoom",{description:"Capture a filtered screen region at full resolution."+rt+w,inputSchema:{region:_t.describe("{x1, y1, x2, y2} coordinates defining top-left and bottom-right corners of the region to capture.")},annotations:W},s=>this.zoom(s)),t.registerTool("batch",{description:"Run ordered actions and return one final screenshot after a built-in settle delay. Use for predictable sequences; wait is intentionally not supported. Supported actions: click, mouse_move, left_click_drag, left_mouse_down, left_mouse_up, type, key, scroll, set_clipboard.",inputSchema:Ot.shape,annotations:I},s=>this.batch(s))}lock(){this._locked===null||!this._computer||(this._locked||(this._locked=this._computer.lock(()=>{this.unlock(),this.server.server.notification({method:"notifications/copilot",params:{type:"user.abort"}}).catch(()=>{});let t=this.accessStore.getState().hostWindowId;t&&this._computer&&this._computer.activateWindow(t).catch(()=>{})})),this._safetyTimer&&clearTimeout(this._safetyTimer),this._safetyTimer=setTimeout(()=>this.unlock(),u.SAFETY_TIMEOUT_MS))}unlock(){this._locked===null||!this._computer||(this._safetyTimer&&(clearTimeout(this._safetyTimer),this._safetyTimer=void 0),this._locked&&this._computer.unlock(),this._locked=!1)}async computer(){return this._computer||(this._computer=z.create(this.logger),this.logger&&(this._computer=new U(this._computer,this.logger))),this._computer}logEvent(t,e){this.logger?.log("info","Server",t,e)}async capabilities(){if(!this._caps){let t=await this.computer();this._caps=await t.capabilities()}return this._caps}async requestPermission(){switch(process.platform){case"linux":{await(await this.computer()).display();break}case"darwin":{let t=await this.computer(),e=[],i=await t.checkPermissions("accessibility");i||e.push("Accessibility");let s=await t.checkPermissions("screen");if(s||e.push("Screen Recording"),e.length>0){if(!this.server.server.getClientCapabilities()?.elicitation?.form)break;let n=["",`Computer Use needs the following macOS permission${e.length>1?"s":""} to control your desktop:`,"",...e.map(a=>`\u2022 ${a}`),"",'Clicking "Open System Settings" will open the relevant settings page(s).',`Enable the permission${e.length>1?"s":""} for your terminal app, then click "I've granted the permissions".`,"","Note: You may need to restart your terminal after granting permissions for them to take effect."];await this.showHostWindow(t);let r=await this.server.server.elicitInput({mode:"form",message:n.join(`
|
|
33
|
+
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:`Grant ${e.join(" and ")} permission${e.length>1?"s":""}`,oneOf:[{const:"open",title:"1. Open System Settings"},{const:"done",title:"2. I've granted the permissions"},{const:"skip",title:"3. Skip (things may not work)"}]}},required:["action"]}}),l=r.action==="accept"?r.content?.action:void 0;if(l==="open"){i||await t.requestPermissions("accessibility"),s||await t.requestPermissions("screen"),await this.showHostWindow(t);let a=await this.server.server.elicitInput({mode:"form",message:["","System Settings has been opened.","",`Enable ${e.join(" and ")} for your terminal app.`,e.length>1?"Both settings pages have been opened \u2014 check each one.":"","",`After granting permissions, click "I've granted the permissions" below.`,"If the toggle was already on, try removing and re-adding your terminal app."].filter(Boolean).join(`
|
|
34
|
+
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:"Confirm permissions",oneOf:[{const:"done",title:"1. I've granted the permissions"},{const:"skip",title:"2. Skip (things may not work)"}]}},required:["action"]}});if((a.action==="accept"?a.content?.action:void 0)!=="done")break}else if(l!=="done")break;if(i=await t.checkPermissions("accessibility"),s=await t.checkPermissions("screen"),!i||!s){let a=[];throw i||a.push("Accessibility"),s||a.push("Screen Recording"),new Error(`${a.join(" and ")} permission${a.length>1?"s are":" is"} still not detected. This usually means you need to restart your terminal after granting the permission. Please quit and reopen your terminal, then try again.`)}}break}}}async activeDisplayId(t){return this.selectedDisplay!==""?this.selectedDisplay:(await t.listDisplays()).find(i=>i.isPrimary)?.displayId??""}async requireAccess(){let t=this.accessStore.getState();if(!t.accessActive)throw new Error("No access session is active. Call request_access first to start a session.");let e=await this.computer();if(t.allowAll&&(await this.capabilities()).canListWindows){let{windows:s}=await this.filterHostWindow(e),o=C(s);this.accessStore.rememberApplications(o),this.appNameResolver.updateApplications(o),t=this.accessStore.getState()}return{computer:e,allowedAppIds:t.allowedAppIds,hostWindowId:t.hostWindowId}}async hostWindowId(t){let e=this.accessStore.getState().hostWindowId;if(e)return e;if(!(await this.capabilities()).canListWindows)return null;let s=await t.getActiveWindow();return s&&(await t.listWindows()).some(n=>n.windowId===s.windowId)?s.windowId:null}async showHostWindow(t){let e=await this.hostWindowId(t);if(e)try{await t.activateWindow(e)}catch{}}async filterHostWindow(t,e){let i=e??await t.listWindows(),s=await this.hostWindowId(t);return{windows:s?i.filter(o=>o.windowId!==s):i,hostWindowId:s}}async prepareForInput(t,e,i){let s=await this.capabilities();if(!s.canListWindows)return;let o=await this.activeDisplayId(t),n=i&&s.canHitTest;if(e.length===0&&!n)return;if(!await t.prepareForInput(o,e,n?i:void 0)){if(n)throw new Error("Input blocked: a disallowed app at the target could not be hidden. Call `list_applications` to see what's there, then `request_access` to allow it.");if(s.canGetActiveWindow){let l=await t.getActiveWindow();if(l&&!e.includes(l.applicationId))throw new Error(this.keyboardInputBlockedMessage(l))}throw new Error("Keyboard input blocked: an allowed application could not be focused. Click an allowed application first, or call `request_access` to allow it.")}}keyboardInputBlockedMessage(t){return`Keyboard input blocked: the focused application ("${t.applicationName}") is not allowed. Click an allowed application first, or call \`request_access\` to allow it.`}async validateActiveWindow(t,e){if(!(await this.capabilities()).canGetActiveWindow)return;let s=await t.getActiveWindow();if(s&&!e.includes(s.applicationId))throw new Error(this.keyboardInputBlockedMessage(s))}pointerInputBlockedMessage(t){return`Pointer input blocked: the focused application ("${t.applicationName}") is not allowed. Focus an allowed application first, or call \`request_access\` to allow it.`}async validatePointerInput(t,e,i){if(e.length===0)return;let s=await this.capabilities();if(s.canGetActiveWindow){let l=await t.getActiveWindow();if(l&&!e.includes(l.applicationId))throw new Error(this.pointerInputBlockedMessage(l))}if(!s.canLocateWindowAtPoint)return;let o=await this.activeDisplayId(t),n=await t.windowAtPoint(o,i.x,i.y);if(!n||e.includes(n.applicationId))return;let r=n.applicationName||n.title||n.applicationId;throw new Error(`Pointer input blocked: the target location is covered by disallowed application ("${r}"). Call \`list_applications\` to see what's there, then \`request_access\` to allow it.`)}async list_displays(){let t=await this.computer(),e=await t.listDisplays(),i=await this.activeDisplayId(t);return this.toJson({displays:e,selectedDisplay:i})}async select_display({display_id:t}){let e=t??"",i=await this.computer(),s=await i.listDisplays();if(e!==""&&!s.some(r=>r.displayId===e))throw new Error(`Unknown display id '${e}'. Call list_displays first.`);this.selectedDisplay=e;let o=await this.activeDisplayId(i),n=s.find(r=>r.displayId===o)??null;return this.toJson({selectedDisplay:o,status:"selected",message:e===""?"Using the default display for screenshots, zoom, cursor position, and coordinate-based actions.":`Selected display '${n?.label??e}' for screenshots, zoom, cursor position, and coordinate-based actions.`})}async list_applications({display_id:t}={}){let e=await this.computer(),{windows:i}=await this.filterHostWindow(e),s=C(i);this.accessStore.rememberApplications(s),this.appNameResolver.updateApplications(s);let o=t??"";if(o!==""&&!(await e.listDisplays()).some(c=>c.displayId===o))throw new Error(`Unknown display id '${o}'. Call list_displays first.`);let n=o===""?i:i.filter(a=>!a.isMinimized&&a.displayId===o),r=C(n),l=await this.activeDisplayId(e);return this.toJson({selectedDisplay:l,applications:r})}async request_access({apps:t,reason:e},i){this.logEvent("request_access",`Start: apps=[${t.join(", ")}]${e?` reason="${e}"`:""}`),await this.requestPermission();let s={const:"allow",title:"Allow"},o={const:"allow_all",title:"Allow all apps (don't ask again)"},n={const:"deny",title:"Deny (Esc)"},r,l=async(h,g,N)=>{this.logEvent("request_access",`End: allowed=${h} allowAll=${g.allowAll} selectedDisplay=${this.selectedDisplay||"(default)"} hostWindowId=${g.hostWindowId??"null"} message="${N}"`);let $=r??(g.allowAll?void 0:g.allowedAppIds.map(y=>{let x=this.accessStore.tryGetKnownApplication(y);return{appId:y,name:x?.displayName,displayIds:this.collectDisplayIds(x?.windows??[])}})),D=this.toJson({allowed:h,allowAll:g.allowAll,...$?{allowedApps:$}:{},selectedDisplay:this.selectedDisplay||void 0,message:N});if(!h)return D;let S=await this.computer();this.lock();let K=g.hostWindowId?[g.hostWindowId]:[];return this.addScreenshot(D,S,g.allowedAppIds,K)};if(this.options.yolo){let h=this.accessStore.allowApplications([],!0);return await l(!0,h,"YOLO mode is enabled. Auto-allowing full desktop access for this session.")}let a=await this.capabilities();if(!this.server.server.getClientCapabilities()?.elicitation?.form){let h=this.accessStore.allowApplications([],!0);return await l(!0,h,"Elicitation is not supported by this client. Auto-allowing full desktop access for this session.")}let d,b,f,Y,G;if(a.canListWindows){let h=await this.computer(),g=this.accessStore.getState().hostWindowId,N=await h.listWindows(),$=g?N.filter(m=>m.windowId!==g):N;G=$;let D=C($);this.accessStore.rememberApplications(D),this.appNameResolver.updateApplications(D);let S=t&&t.length>0?"":await this.activeDisplayId(h);S&&(this.selectedDisplay=S);let K=new Set(g?[g]:[]),y;if(t&&t.length>0){let m=new Map;for(let v of t){let R=this.appNameResolver.resolve(v);if(R.length===0)throw new Error(`No matching application found for '${v}'. Call list_applications to all running apps and their IDs.`);for(let P of R)if(!m.has(P.appId)){let E=this.accessStore.tryGetKnownApplication(P.appId);m.set(P.appId,E??{id:P.appId,displayName:P.displayName,windows:[]})}}y=[...m.values()];let At=new Set(m.keys()),L=new Map;for(let v of $)At.has(v.applicationId)&&!v.isMinimized&&v.displayId&&L.set(v.displayId,(L.get(v.displayId)??0)+1);if(L.size>0){let v="",R=0;for(let[P,E]of L)E>R&&(v=P,R=E);this.selectedDisplay=v}r=this.buildAllowedAppsInfo(y,$)}else y=C($.filter(m=>!m.isMinimized&&(S===""||m.displayId===S)&&!K.has(m.windowId)));let x=y.map(m=>m.id),J=this.accessStore.getState(),M=y.length;b=["allow","allow_all"],f=x.length===0?J.allowAll||J.accessActive:this.accessStore.areAllowedForAccess(x),Y=x.length===0?J.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.":M===1?`'${y[0].displayName}' is already shared for this session.`:"The requested apps are already shared for this session.",d={choices:[s,o,n],message:this.buildAccessMessage(y.map(m=>m.displayName),e),denyMessage:M===0?"The user declined desktop access.":M===1?`The user declined to share '${y[0].displayName}'.`:"The user declined to share the requested apps.",allow:m=>this.accessStore.allowApplications(x,m==="allow_all"),allowedMessage:m=>m==="allow_all"?"All current and future apps are allowed for the rest of this session, so you do not need to call request_access again.":M===0?"Desktop access is active for this session.":M===1?`Access session started for '${y[0].displayName}'.`:"Access session started for the requested apps."}}else{let h=this.accessStore.getState();b=["allow_all"],f=h.allowAll||h.accessActive,Y=h.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.",d={choices:[o,n],message:this.buildAccessMessage([],e),denyMessage:"The user declined desktop access.",allow:()=>this.accessStore.allowApplications([],!0),allowedMessage:()=>"Access session started for the desktop. All current and future apps are allowed for the rest of this session, so you do not need to call request_access again."}}if(f){let h=this.accessStore.getState();return await l(!0,h,Y)}this.accessStore.getState().hostWindowId&&await this.showHostWindow(await this.computer());let X=await this.server.server.request({method:"elicitation/create",params:{mode:"form",message:d.message,requestedSchema:{type:"object",properties:{decision:{type:"string",title:"Allow access for this session?",oneOf:d.choices}},required:["decision"]}}},St,{signal:i,timeout:300*1e3}),vt=await this.computer(),tt=await this.hostWindowId(vt);tt&&this.accessStore.setHostWindowId(tt);let k=X.action==="accept"?X.content?.decision:void 0;if(X.action!=="accept"||!k||typeof k!="string"||!b.includes(k)){let h=this.accessStore.getState(),g=typeof k=="string"&&k!==n.const?`The user did not approve this access request. User response: ${k}`:d.denyMessage;return await l(!1,h,g)}let et=d.allow(k);return et.allowAll&&G&&(r=this.buildAllowedAppsInfo(this.accessStore.allKnownApplications(),G)),await l(!0,et,d.allowedMessage(k))}async screenshot(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[];return this.captureScreenshot(t,e,s)}logScreenshotFilter(t,e){let i=r=>{let l=this.accessStore.tryGetKnownApplication(r);return l?`${l.displayName} (${r})`:r},s=r=>{let l=r.includes("|")?r.split("|")[1]:r;return i(l)},o=(r,l)=>l.length===1?`${r}: ${l[0]}`:`${r}:
|
|
35
|
+
${l.map(a=>`- ${a}`).join(`
|
|
36
36
|
`)}`,n=[t.length?o("allowed",t.map(i)):"allowed: (all apps)"];e.length&&n.push(o("blocked",e.map(s))),this.logEvent("screenshot filter",n.join(`
|
|
37
|
-
`))}async captureScreenshot(t,e,i){this.logScreenshotFilter(e,i);let s=await t.screenshot(this.selectedDisplay,e,i,0,0,void 0,
|
|
37
|
+
`))}async captureScreenshot(t,e,i){this.logScreenshotFilter(e,i);let s=await t.screenshot(this.selectedDisplay,e,i,0,0,void 0,_),o=this.toImage(s,"Screenshot failed",_);if(s){let n=await this.unsharedWindowsNote(t,e,i);this.appendUnsharedWindowsNote(o,n,!0)}return o}async addScreenshot(t,e,i,s){await new Promise(r=>setTimeout(r,Ct)),this.logScreenshotFilter(i,s);let o=null;try{o=await e.screenshot(this.selectedDisplay,i,s,0,0,void 0,_)}catch{}if(!o)return t.content.push({type:"text",text:"Screenshot capture failed"}),t;t.content.push({type:"image",data:o.toString("base64"),mimeType:this.imageMimeType(_),_meta:{screenshot:!0}});let n=await this.unsharedWindowsNote(e,i,s);return this.appendUnsharedWindowsNote(t,n,!1),t}appendUnsharedWindowsNote(t,e,i){if(!e){this.lastUnsharedWindowsNote=null;return}(i||e!==this.lastUnsharedWindowsNote)&&t.content.push({type:"text",text:e}),this.lastUnsharedWindowsNote=e}async unsharedWindowsNote(t,e,i){if(!(await this.capabilities()).canListWindows)return null;let o=await this.activeDisplayId(t),n=new Set(e),r=new Set(i),{windows:l}=await this.filterHostWindow(t),a=l.filter(d=>!d.isMinimized&&(o===""||d.displayId===o)&&!n.has(d.applicationId)&&!r.has(d.windowId));if(a.length===0)return null;let c=a.length===1?"window":"windows";return`${a.length} other ${c} on this display ${a.length===1?"is":"are"} from an unshared app. The screenshot may not show everything that's running. Use \`list_applications\` to see what else is there if you need to.`}async cursor_position(){let e=await(await this.computer()).cursorPosition(this.selectedDisplay);return this.toText(`${e.x},${e.y}`)}async _click(t,e,i,s,o,n,r){let l;switch(n){case 1:l="";break;case 2:l="double ";break;case 3:l="triple ";break;default:throw new Error(`Invalid click count '${n}'. Expected 1, 2, or 3.`)}let a=r??await t.cursorPosition(s);await this.prepareForInput(t,e,{x:a.x,y:a.y,blockedWindowIds:i}),await this.validatePointerInput(t,e,a),r&&await t.move(a.x,a.y,s),await t.click(a.x,a.y,o,n,s);let c=r?` at (${r.x},${r.y})`:"",d=o==="left"?"":`${o} `;return`${l}${d}click${c}`}async click({coordinate:t,button:e,count:i}){let{computer:s,allowedAppIds:o,hostWindowId:n}=await this.requireAccess();this.lock();let r=n?[n]:[],l=await this._click(s,o,r,this.selectedDisplay,e??"left",i??1,t);return this.addScreenshot(this.toText(l),s,o,r)}async mouse_move({coordinate:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];return await this.prepareForInput(e,i,{x:t.x,y:t.y,blockedWindowIds:o}),await this.validatePointerInput(e,i,t),await e.move(t.x,t.y,this.selectedDisplay),this.addScreenshot(this.toText(`Moved to (${t.x},${t.y})`),e,i,o)}async left_click_drag({start_coordinate:t,coordinate:e}){let{computer:i,allowedAppIds:s,hostWindowId:o}=await this.requireAccess();this.lock();let n=o?[o]:[];return await this.prepareForInput(i,s,{x:t.x,y:t.y,blockedWindowIds:n}),await this.prepareForInput(i,s,{x:e.x,y:e.y,blockedWindowIds:n}),await this.validatePointerInput(i,s,t),await this.validatePointerInput(i,s,e),await i.drag(t.x,t.y,e.x,e.y,this.selectedDisplay),this.addScreenshot(this.toText(`Dragged (${t.x},${t.y}) -> (${e.x},${e.y})`),i,s,n)}async left_mouse_down(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[],o=await t.cursorPosition(this.selectedDisplay);return await this.prepareForInput(t,e,{x:o.x,y:o.y,blockedWindowIds:s}),await this.validatePointerInput(t,e,o),await t.mouseDown(o.x,o.y,this.selectedDisplay),this.addScreenshot(this.toText(`Mouse down at (${o.x},${o.y})`),t,e,s)}async left_mouse_up(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[],o=await t.cursorPosition(this.selectedDisplay);return await this.prepareForInput(t,e,{x:o.x,y:o.y,blockedWindowIds:s}),await this.validatePointerInput(t,e,o),await t.mouseUp(o.x,o.y,this.selectedDisplay),this.addScreenshot(this.toText(`Mouse up at (${o.x},${o.y})`),t,e,s)}async type({text:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];return await this.prepareForInput(e,i),await this.validateActiveWindow(e,i),await e.type(t),this.addScreenshot(this.toText(`Typed ${t.length} chars`),e,i,o)}async key({text:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];await this.prepareForInput(e,i),await this.validateActiveWindow(e,i);let n=Q(t);return await e.key(n),this.addScreenshot(this.toText(`Pressed ${n}`),e,i,o)}async scroll({scroll_direction:t,coordinate:e,scroll_amount:i}){let{computer:s,allowedAppIds:o,hostWindowId:n}=await this.requireAccess();this.lock();let r=n?[n]:[],l=i??3,a=await s.cursorPosition(this.selectedDisplay),c=e?.x??a.x,d=e?.y??a.y;await this.prepareForInput(s,o,{x:c,y:d,blockedWindowIds:r}),await this.validatePointerInput(s,o,{x:c,y:d});let b=t==="left"?-l:t==="right"?l:0,f=t==="down"?l:t==="up"?-l:0;return await s.scroll(c,d,b,f,this.selectedDisplay),this.addScreenshot(this.toText(`Scrolled ${t} ${l} at (${c},${d})`),s,o,r)}async wait({duration:t}){this.logEvent("wait",`duration=${String(t)}s`),await new Promise(r=>setTimeout(r,t*1e3));let e=this.toText(`Waited ${t}s`);if(!this.accessStore.getState().accessActive)return e;let{computer:i,allowedAppIds:s,hostWindowId:o}=await this.requireAccess(),n=o?[o]:[];return this.addScreenshot(e,i,s,n)}async get_clipboard(){let{computer:t}=await this.requireAccess();this.lock();let e=await t.getClipboard();return this.toText(e)}async set_clipboard({text:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];return await e.setClipboard(t),this.addScreenshot(this.toText(`Clipboard set (${t.length} chars)`),e,i,o)}async zoom({region:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[],n=await e.screenshot(this.selectedDisplay,i,o,0,0,[t.x1,t.y1,t.x2,t.y2],_);return this.toImage(n,"Zoom screenshot failed",_)}async batch({actions:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock(),this.logEvent("batch",`${t.length} actions: ${t.map(l=>l.action).join(", ")}`);let o=this.selectedDisplay,n=s?[s]:[],r=[];for(let l=0;l<t.length;l++){let a=t[l];try{switch(a.action){case"click":{let c=await this._click(e,i,n,o,a.button??"left",a.count??1,a.coordinate);r.push(`[${l}] ${c}`);break}case"mouse_move":await this.prepareForInput(e,i,{x:a.coordinate.x,y:a.coordinate.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,a.coordinate),await e.move(a.coordinate.x,a.coordinate.y,o),r.push(`[${l}] moved to (${a.coordinate.x},${a.coordinate.y})`);break;case"left_click_drag":await this.prepareForInput(e,i,{x:a.start_coordinate.x,y:a.start_coordinate.y,blockedWindowIds:n}),await this.prepareForInput(e,i,{x:a.coordinate.x,y:a.coordinate.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,a.start_coordinate),await this.validatePointerInput(e,i,a.coordinate),await e.drag(a.start_coordinate.x,a.start_coordinate.y,a.coordinate.x,a.coordinate.y,o),r.push(`[${l}] dragged (${a.start_coordinate.x},${a.start_coordinate.y}) -> (${a.coordinate.x},${a.coordinate.y})`);break;case"left_mouse_down":{let c=await e.cursorPosition(o);await this.prepareForInput(e,i,{x:c.x,y:c.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,c),await e.mouseDown(c.x,c.y,o),r.push(`[${l}] mouse down at (${c.x},${c.y})`);break}case"left_mouse_up":{let c=await e.cursorPosition(o);await this.prepareForInput(e,i,{x:c.x,y:c.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,c),await e.mouseUp(c.x,c.y,o),r.push(`[${l}] mouse up at (${c.x},${c.y})`);break}case"type":await this.prepareForInput(e,i),await this.validateActiveWindow(e,i),await e.type(a.text),r.push(`[${l}] typed ${a.text.length} chars`);break;case"key":{await this.prepareForInput(e,i),await this.validateActiveWindow(e,i);let c=Q(a.text);await e.key(c),r.push(`[${l}] key ${a.text}`);break}case"scroll":{let c=a.scroll_amount??3,d=a.coordinate??await e.cursorPosition(o);await this.prepareForInput(e,i,{x:d.x,y:d.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,d);let b=a.scroll_direction==="left"?-c:a.scroll_direction==="right"?c:0,f=a.scroll_direction==="down"?c:a.scroll_direction==="up"?-c:0;await e.scroll(d.x,d.y,b,f,o),r.push(`[${l}] scroll ${a.scroll_direction}${a.coordinate?` at (${d.x},${d.y})`:""}`);break}case"set_clipboard":await e.setClipboard(a.text),r.push(`[${l}] clipboard set`);break}}catch(c){return r.push(`[${l}] ${a.action}: FAILED - ${c instanceof Error?c.message:String(c)}`),this.toText(r.join(`
|
|
38
38
|
`),!0)}}return this.addScreenshot(this.toText(r.join(`
|
|
39
|
-
`)),e,i,n)}};function
|
|
40
|
-
`)});return new O(
|
|
39
|
+
`)),e,i,n)}};function he(u={}){let t={name:"computer-use",version:"1.0.0"},e=["This MCP server provides desktop automation tools (mouse, keyboard, screenshots, clipboard).",u.yolo?"YOLO mode is enabled. Call `request_access` once to auto-allow all current and future apps for this session.":"Before using access-gated tools (screenshot, click, type, clipboard, etc.), you MUST call `request_access` to start an access session.","For a named target app, call `request_access` directly; it can find the app, select its display, and return the first screenshot. Use `list_applications` only for discovery or ambiguity.","Calling `request_access` with an empty apps array allows all visible apps on the selected display. If none are visible, it falls back to desktop access and empty desktop screenshots until apps are allowed.","Screenshots and zoom captures are composited to show ONLY the windows of allowed applications (plus system UI like the Dock). Disallowed app windows are not visible.","For speed, use returned screenshots instead of post-action `screenshot`, and batch predictable actions. Returned screenshots already include a small settle delay; do not add waits just for screenshot timing.","Prefer keyboard shortcuts, `type`, and `key` over visual menu navigation when reliable.","If `request_access` returns `allowAll=true`, all current and future apps are allowed for the rest of this session and you do not need to call `request_access` again.","Otherwise, if you launch a new app during the session, call `request_access` again to add it. Use `list_applications` first if you need to discover the app or its display."],i=new Yt(t,{instructions:e.join(`
|
|
40
|
+
`)});return new O(u).connect(i),i}export{he as createServer};
|
package/dist/main.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{StdioServerTransport as
|
|
2
|
+
import{StdioServerTransport as Xt}from"@modelcontextprotocol/sdk/server/stdio.js";import{McpServer as Gt}from"@modelcontextprotocol/sdk/server/mcp.js";import{ElicitResultSchema as Ct}from"@modelcontextprotocol/sdk/types.js";import{z as p}from"zod";var j=class{knownApplications=new Map;allowedAppIds=new Set;accessActive=!1;allowAll=!1;hostWindowId=null;rememberApplications(t){this.knownApplications.clear();for(let e of t)this.knownApplications.set(e.id,e);if(this.allowAll)for(let e of t)this.allowedAppIds.add(e.id)}getKnownApplications(t){return t.map(e=>{let i=this.knownApplications.get(e);if(!i)throw new Error(`Unknown application id '${e}'. Call list_applications first.`);return i})}setHostWindowId(t){this.hostWindowId=t}tryGetKnownApplication(t){return this.knownApplications.get(t)}allKnownApplications(){return[...this.knownApplications.values()]}allowApplications(t,e=!1,i){this.accessActive=!0;for(let s of t)this.allowedAppIds.add(s);if(e){this.allowAll=!0;for(let s of this.knownApplications.keys())this.allowedAppIds.add(s)}return i&&(this.hostWindowId=i),this.getState()}areAllowedForAccess(t){return this.allowAll||t.every(e=>this.allowedAppIds.has(e))}getState(){return{accessActive:this.accessActive,allowedAppIds:[...this.allowedAppIds],allowAll:this.allowAll,hostWindowId:this.hostWindowId}}};function C(d){let t=new Map,e=[...d].sort((i,s)=>`${i.applicationName}\0${i.title}\0${i.windowId}`.localeCompare(`${s.applicationName}\0${s.title}\0${s.windowId}`));for(let i of e){let s=i.applicationName||i.title||i.applicationId,o=t.get(i.applicationId);o||(o={id:i.applicationId,displayName:s,applicationNames:i.applicationNames,windows:[]},t.set(i.applicationId,o)),o.windows.push({windowId:i.windowId,title:i.title,displayId:i.displayId,isMinimized:i.isMinimized})}return[...t.values()].sort((i,s)=>`${i.displayName}\0${i.id}`.localeCompare(`${s.displayName}\0${s.id}`))}function ot(d){return d.toLowerCase().split(/[^a-z0-9]+/).filter(t=>t.length>0)}function it(d){let t=new Set;for(let e of d)for(let i of ot(e))t.add(i);return t}function nt(d){return d.trim().toLowerCase()}function st(d){let t=new Set;for(let e of d){let i=nt(e);i&&t.add(i)}return t}var B=class{appRecords=new Map;updateApplications(t){for(let e of t){let i=[e.displayName,...e.applicationNames??[]],s=this.appRecords.get(e.id);if(s){for(let o of it(i))s.tokenBag.add(o);for(let o of st(i))s.exactNames.add(o);s.displayName=e.displayName}else this.appRecords.set(e.id,{appId:e.id,displayName:e.displayName,tokenBag:it(i),exactNames:st(i)})}}resolve(t){let e=this.appRecords.get(t);if(e)return[e];let i=nt(t);if(i){let n=[];for(let r of this.appRecords.values())r.exactNames.has(i)&&n.push(r);if(n.length>0)return n}let s=ot(t);if(s.length===0)return[];let o=[];for(let n of this.appRecords.values())s.every(r=>n.tokenBag.has(r))&&o.push(n);return o}allRecords(){return[...this.appRecords.values()]}};var kt=new Set(["ctrl","control","shift","alt","option","super","meta","cmd","command","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","0","1","2","3","4","5","6","7","8","9","return","enter","tab","space","backspace","delete","forwarddelete","escape","esc","up","down","left","right","home","end","pageup","pagedown","insert","capslock","f1","f2","f3","f4","f5","f6","f7","f8","f9","f10","f11","f12","-","minus","=","equal","plus","[","]","\\",";","'",",",".","/","`"]),$t={arrowup:"up",arrowdown:"down",arrowleft:"left",arrowright:"right",page_up:"pageup",page_down:"pagedown",backslash:"\\",semicolon:";",slash:"/",grave:"`",bracketleft:"[",bracketright:"]",super_l:"super",gui:"super",win:"super",windows:"super",caps_lock:"capslock",caps:"capslock",del:"delete"};function Q(d){d==="+"?d="plus":d.length>2&&d.endsWith("++")&&(d=d.slice(0,-1)+"plus");let t=d.split("+").map(i=>i.trim().toLowerCase()).filter(Boolean);if(t.length===0)throw new Error("Key combo must contain at least one key.");return t.map(i=>{let s=$t[i]??i;if(!kt.has(s))throw new Error(`Unknown key "${i}" in combo "${d}".`);return s}).join("+")}import*as A from"fs/promises";import*as at from"os";import*as H from"path";var V="<!-- LOG -->",U=class{logDir;logPath;startTime=performance.now();queue=Promise.resolve();constructor(){this.logDir=H.join(at.homedir(),".copilot","logs","computer-use"),this.logPath=H.join(this.logDir,`${this.timestamp()}.html`)}timestamp(){let t=new Date,e=(i,s=2)=>String(i).padStart(s,"0");return`${t.getFullYear()}${e(t.getMonth()+1)}${e(t.getDate())}-${e(t.getHours())}${e(t.getMinutes())}${e(t.getSeconds())}.${e(Math.floor(t.getMilliseconds()/10))}`}elapsed(){return`${((performance.now()-this.startTime)/1e3).toFixed(2)}s`}insert(t){return this.queue=this.queue.then(()=>this.write(t)).catch(()=>this.write(t)).catch(()=>{}),this.queue}async write(t){let e=!0;try{await A.access(this.logPath)}catch{e=!1}e||(await A.mkdir(this.logDir,{recursive:!0}),await A.writeFile(this.logPath,`<!DOCTYPE html>
|
|
3
3
|
<html>
|
|
4
4
|
<head>
|
|
5
5
|
<meta charset="utf-8">
|
|
@@ -23,20 +23,20 @@ import{StdioServerTransport as Qt}from"@modelcontextprotocol/sdk/server/stdio.js
|
|
|
23
23
|
<body>
|
|
24
24
|
<table>
|
|
25
25
|
<tr><th>Time</th><th>Level</th><th>Source</th><th>Action</th><th>Details</th></tr>
|
|
26
|
-
${
|
|
26
|
+
${V}
|
|
27
27
|
</table>
|
|
28
28
|
</body>
|
|
29
29
|
</html>
|
|
30
|
-
`));let i=await A.readFile(this.logPath,"utf-8");await A.writeFile(this.logPath,i.replace(
|
|
30
|
+
`));let i=await A.readFile(this.logPath,"utf-8");await A.writeFile(this.logPath,i.replace(V,t+V))}rowHtml(t,e,i,s){return`<tr><td class="time">${this.elapsed()}</td><td class="level level-${i}">${i}</td><td class="source">${s}</td><td class="action">${this.escapeHtml(t)}</td><td>${e}</td></tr>
|
|
31
31
|
`}escapeHtml(t){return t.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/"/g,""")}renderMarkdown(t){let e=t.split(`
|
|
32
|
-
`),i=[],s=[],o=()=>{s.length>0&&(i.push(`<ul style="margin:0;padding-left:1.2em">${s.map(n=>`<li>${this.escapeHtml(n)}</li>`).join("")}</ul>`),s=[])};for(let n of e)n.startsWith("- ")?s.push(n.slice(2)):(o(),n.length>0&&(i.length>0&&i.push("<br>"),i.push(this.escapeHtml(n))));return o(),i.join("")}log(t,e,i,s){this.insert(this.rowHtml(i,this.renderMarkdown(s),t,e))}logScreenshot(t,e,i,s){if(i){let o=s===void 0?"png":"jpg",n=`${this.timestamp()}.${o}`;A.mkdir(this.logDir,{recursive:!0}).then(()=>A.writeFile(U.join(this.logDir,n),i)),this.insert(this.rowHtml(t,`${this.escapeHtml(e)}<br><img src="${n}">`,"info","Computer"))}else this.insert(this.rowHtml(t,this.escapeHtml(e),"error","Computer"))}};var B=class{computer;logger;constructor(t,e){this.computer=t,this.logger=e}log(t,e){this.logger.log("info","Computer",t,e)}async checkPermissions(t){return this.computer.checkPermissions(t)}async requestPermissions(t){return this.computer.requestPermissions(t)}async capabilities(){let t=await this.computer.capabilities();return this.log("capabilities",JSON.stringify(t)),t}async click(t,e,i,s,o=""){await this.computer.click(t,e,i,s,o),this.log("click",`(${t}, ${e}) button=${i} count=${s}${o?` display=${o}`:""}`)}async move(t,e,i=""){await this.computer.move(t,e,i),this.log("move",`(${t}, ${e})${i?` display=${i}`:""}`)}async drag(t,e,i,s,o=""){await this.computer.drag(t,e,i,s,o),this.log("drag",`(${t}, ${e}) \u2192 (${i}, ${s})${o?` display=${o}`:""}`)}async mouseDown(t,e,i=""){await this.computer.mouseDown(t,e,i),this.log("mouseDown",`(${t}, ${e})${i?` display=${i}`:""}`)}async mouseUp(t,e,i=""){await this.computer.mouseUp(t,e,i),this.log("mouseUp",`(${t}, ${e})${i?` display=${i}`:""}`)}async type(t){await this.computer.type(t),this.log("type",`"${t}"`)}async key(t){await this.computer.key(t),this.log("key",t)}async scroll(t,e,i,s,o=""){await this.computer.scroll(t,e,i,s,o),this.log("scroll",`(${t}, ${e}) dx=${i} dy=${s}${o?` display=${o}`:""}`)}async cursorPosition(t=""){let e=await this.computer.cursorPosition(t);return this.log("cursorPosition",`(${e.x}, ${e.y})${t?` display=${t}`:""}`),e}async display(t=""){let e=await this.computer.display(t);return this.log("display",`${e.width}x${e.height}${t?` display=${t}`:""}`),e}async listDisplays(){let t=await this.computer.listDisplays(),e=t.map(i=>`${i.isPrimary?"*":""}${i.displayId} "${i.label}" ${i.width}x${i.height}`).join("; ");return this.log("listDisplays",`${t.length} displays${e?`: ${e}`:""}`),t}async listWindows(){let t=await this.computer.listWindows(),e=new Map;for(let s of t){let o=s.displayId||"(unknown)",n=e.get(o)??{visible:0,minimized:0};s.isMinimized?n.minimized+=1:n.visible+=1,e.set(o,n)}let i=[...e.entries()].sort(([s],[o])=>s.localeCompare(o)).map(([s,o])=>`${s}: ${o.visible} visible, ${o.minimized} minimized`).join("; ");return this.log("listWindows",`${t.length} windows${i?`: ${i}`:""}`),t}async getActiveWindow(){let t=await this.computer.getActiveWindow();return this.log("getActiveWindow",t?`${t.windowId} title="${t.title}"`:"null"),t}async activateApplication(t){let e=await this.computer.activateApplication(t);return this.log("activateApplication",`${t} -> ${e}`),e}async concealApplication(t){let e=await this.computer.concealApplication(t);return this.log("concealApplication",`${t} -> ${e}`),e}async restoreApplication(t){let e=await this.computer.restoreApplication(t);return this.log("restoreApplication",`${t} -> ${e}`),e}async activateWindow(t){let e=await this.computer.activateWindow(t);return this.log("activateWindow",`${t} -> ${e}`),e}async concealWindow(t){let e=await this.computer.concealWindow(t);return this.log("concealWindow",`${t} -> ${e}`),e}async restoreWindow(t){let e=await this.computer.restoreWindow(t);return this.log("restoreWindow",`${t} -> ${e}`),e}async getClipboard(){let t=await this.computer.getClipboard(),e=t.slice(0,200)+(t.length>200?"\u2026":"");return this.log("getClipboard",`"${e}"`),t}async setClipboard(t){await this.computer.setClipboard(t);let e=t.slice(0,200)+(t.length>200?"\u2026":"");this.log("setClipboard",`"${e}" (${t.length} chars)`)}lock(t){return this.computer.lock(t)}unlock(){this.computer.unlock()}async prepareForInput(t,e,i){let s=await this.computer.prepareForInput(t,e,i),o=i?` @ (${i.x},${i.y})`:"";return this.log("prepareForInput",`${t} -> ${s} (${e.length} apps)${o}`),s}async screenshot(t,e,i,s,o,n,r){let c=await this.computer.screenshot(t,e,i,s,o,n,r),a=s&&o?` (${s}x${o})`:"",l=n?` crop=[${n.join(",")}]`:"",u=`display=${t||"(primary)"}`,f=r===void 0?"":` quality=${r}`,h=`${u}${a}${l}${f}`;return this.logger.logScreenshot("screenshot",h,c,r),c}};import{createRequire as _t}from"module";import{dirname as St,join as Pt}from"path";import{fileURLToPath as Wt}from"url";var Ct=_t(import.meta.url),Tt=St(Wt(import.meta.url)),z=class p{constructor(t,e){this.native=t;t.setLogger?.((i,s,o)=>{e?.log(this.nativeLogLevel(i),"Driver",s,o)})}native;static create(t){let e=Pt(Tt,"prebuilds",`${process.platform}-${process.arch}`,"computer.node"),i;try{i=Ct(e)}catch(s){let o=s instanceof Error?s.message:String(s);throw new Error(`Native computer bindings not available for ${process.platform}-${process.arch}: ${o} (path: ${e})`,{cause:s})}return new p(i,t)}async checkPermissions(t){return this.native.checkPermissions(t)}async requestPermissions(t){this.native.requestPermissions(t)}async click(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,c]=this.toNative(n,t,e);this.native.click(r,c,i,s)}nativeLogLevel(t){switch(t){case 0:return"trace";case 1:return"debug";case 2:return"info";case 3:return"warn";case 4:return"error";default:return"info"}}async move(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.move(o,n)}async drag(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e),this.assertPointInBounds(n,i,s);let[r,c]=this.toNative(n,t,e),[a,l]=this.toNative(n,i,s);this.native.drag(r,c,a,l)}async mouseDown(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseDown(o,n)}async mouseUp(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseUp(o,n)}async type(t){this.native.type(t)}async key(t){this.native.key(t)}async scroll(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,c]=this.toNative(n,t,e);this.native.scroll(r,c,i,s)}async cursorPosition(t=""){let e=this.native.cursorPosition(),i=this.resolveCoordinateSpace(t),[s,o]=this.fromNative(i,e.x,e.y);return{x:s,y:o}}async display(t=""){let e=this.resolveCoordinateSpace(t);return{width:e.targetWidth,height:e.targetHeight}}async screenshot(t,e,i,s,o,n,r){let c=this.resolveCoordinateSpace(t,s,o),a=r??-1;if(n){this.assertRegionEdgeInBounds(c,n[0],n[1]),this.assertRegionEdgeInBounds(c,n[2],n[3]);let[l,u,f,h]=this.toNativeCrop(c,n[0],n[1],n[2],n[3]);return this.native.screenshot(t,e,i,0,0,l,u,f,h,a)}return this.native.screenshot(t,e,i,c.targetWidth,c.targetHeight,0,0,0,0,a)}async capabilities(){return{...this.native.capabilities(),canGetActiveWindow:typeof this.native.getActiveWindow=="function"}}async listDisplays(){return this.native.listDisplays().map(e=>{let{targetWidth:i,targetHeight:s}=this.targetSizeForDisplay(e.bounds.width,e.bounds.height);return{displayId:e.displayId,label:e.label,width:i,height:s,isPrimary:e.isPrimary}})}async listWindows(){return this.native.listWindows()}async getActiveWindow(){return this.native.getActiveWindow?this.native.getActiveWindow():null}async activateApplication(t){return this.native.activateApplication(t)}async concealApplication(t){return this.native.concealApplication(t)}async restoreApplication(t){return this.native.restoreApplication(t)}async activateWindow(t){return this.native.activateWindow(t)}async concealWindow(t){return this.native.concealWindow(t)}async restoreWindow(t){return this.native.restoreWindow(t)}async getClipboard(){return this.native.getClipboard()}async setClipboard(t){this.native.setClipboard(t)}lock(t){return this.native.lock?.(t)??!1}unlock(){this.native.unlock?.()}async prepareForInput(t,e,i){let s=Number.NaN,o=Number.NaN;if(i){let n=this.resolveCoordinateSpace(t);this.assertPointInBounds(n,i.x,i.y),[s,o]=this.toNative(n,i.x,i.y)}return this.native.prepareForInput(t,e,i?.blockedWindowIds??[],s,o)}resolveDisplay(t){let e=this.native.listDisplays();return t!==""?e.find(i=>i.displayId===t):e.find(i=>i.isPrimary)??e[0]}targetSizeForDisplay(t,e,i=0,s=0){if(i>0&&s>0)return{targetWidth:i,targetHeight:s};let r=Math.min(1,1568/Math.max(t,e),Math.sqrt(115e4/(t*e)));return{targetWidth:Math.floor(t*r),targetHeight:Math.floor(e*r)}}resolveCoordinateSpace(t,e=0,i=0){let s=this.resolveDisplay(t),o=s?.bounds.x??0,n=s?.bounds.y??0,r=s?void 0:this.native.display(),c=s?.bounds.width??r?.width??1,a=s?.bounds.height??r?.height??1,{targetWidth:l,targetHeight:u}=this.targetSizeForDisplay(c,a,e,i);return{originX:o,originY:n,displayWidth:c,displayHeight:a,scaleX:l/c,scaleY:u/a,targetWidth:l,targetHeight:u}}toNative(t,e,i){return[t.originX+e/t.scaleX,t.originY+i/t.scaleY]}toNativeCrop(t,e,i,s,o){return[Math.floor(e/t.scaleX),Math.floor(i/t.scaleY),Math.ceil(s/t.scaleX),Math.ceil(o/t.scaleY)]}assertPointInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>=t.targetWidth||i>=t.targetHeight)throw new RangeError(`Coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}assertRegionEdgeInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>t.targetWidth||i>t.targetHeight)throw new RangeError(`Crop coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}fromNative(t,e,i){return[Math.round((e-t.originX)*t.scaleX),Math.round((i-t.originY)*t.scaleY)]}};var C=.8,Nt=250,S={readOnlyHint:!0,destructiveHint:!1,openWorldHint:!1},b={readOnlyHint:!1,destructiveHint:!0,openWorldHint:!0},V={readOnlyHint:!1,destructiveHint:!1,openWorldHint:!1},w=" Requires `request_access`.",at=" If the target app is missing, call `list_applications` then `request_access`.",T=d.coerce.number().transform(Math.round),M=d.object({x:T.describe("Horizontal pixel coordinate."),y:T.describe("Vertical pixel coordinate.")}),Rt=d.object({x1:T.describe("Left edge of the region in screenshot pixel coordinates."),y1:T.describe("Top edge of the region in screenshot pixel coordinates."),x2:T.describe("Right edge of the region in screenshot pixel coordinates."),y2:T.describe("Bottom edge of the region in screenshot pixel coordinates.")}).refine(({x1:p,x2:t})=>t>p,{message:"x2 must be greater than x1.",path:["x2"]}).refine(({y1:p,y2:t})=>t>p,{message:"y2 must be greater than y1.",path:["y2"]}),qt=d.string().min(1).describe("Application name or stable app id from `list_applications`. Examples: 'Google Chrome', 'Microsoft Outlook', 'app.windows.abc123'."),Mt=d.object({display_id:d.string().optional().describe("Display id to select. Omit or pass an empty string to use the default display.")}),Lt=d.object({display_id:d.string().optional().describe("Optional display id to filter by. When provided, only apps with at least one non-minimized window on that display are returned.")}),pt=M.optional().describe("{x, y} pixel coordinate. Clicks at current cursor position if omitted."),dt=M.describe("{x, y} pixel coordinate to move the cursor to."),ut=M.describe("{x, y} pixel coordinate to start the drag from."),ht=M.describe("{x, y} pixel coordinate to drag to."),mt=d.string().describe("The text to type."),gt=d.string().describe("Key combo to press (e.g. 'Return', 'ctrl+s', 'alt+Tab')."),wt=d.enum(["up","down","left","right"]).describe("Direction to scroll."),yt=M.optional().describe("{x, y} pixel coordinate to scroll at. Scrolls at current cursor position if omitted."),ft=d.number().int().nonnegative().max(100).optional().describe("Number of scroll clicks (default 3)."),jt=d.number().nonnegative().max(100).describe("Seconds to wait (max 100). Use after a screenshot shows the UI is not in the expected state yet, but an operation may still finish."),bt=d.string().describe("The text to copy to the clipboard."),Et=d.object({action:d.enum(["left_click","right_click","middle_click","double_click","triple_click"]),coordinate:pt}),Ht=d.object({action:d.literal("mouse_move"),coordinate:dt}),Ut=d.object({action:d.literal("left_click_drag"),start_coordinate:ut,coordinate:ht}),Ft=d.object({action:d.literal("left_mouse_down")}),Bt=d.object({action:d.literal("left_mouse_up")}),zt=d.object({action:d.literal("type"),text:mt}),Ot=d.object({action:d.literal("key"),text:gt}),Yt=d.object({action:d.literal("scroll"),scroll_direction:wt,coordinate:yt,scroll_amount:ft}),Gt=d.object({action:d.literal("set_clipboard"),text:bt}),Xt=d.discriminatedUnion("action",[Et,Ht,Ut,Ft,Bt,zt,Ot,Yt,Gt]),Kt=d.object({actions:d.array(Xt).min(1).describe("Ordered actions. Do not include waits; call the separate `wait` tool only after observing a screenshot that is not ready yet.")});function rt(p,t){let e=p.length===0?["","Computer Use runs on your actual desktop and can send mouse and keyboard input.","","Computer Use wants to control your desktop for this session.","","Apps that are not allowed may be hidden."]:["","Computer Use runs on your actual desktop and can send mouse and keyboard input to the apps you share.","","Computer Use wants to control these apps:","",...p.map(i=>`- ${i}`),"","Apps that are not allowed may be hidden."];return t&&e.push("","Reason:",t),e.join(`
|
|
33
|
-
`)}
|
|
32
|
+
`),i=[],s=[],o=()=>{s.length>0&&(i.push(`<ul style="margin:0;padding-left:1.2em">${s.map(n=>`<li>${this.escapeHtml(n)}</li>`).join("")}</ul>`),s=[])};for(let n of e)n.startsWith("- ")?s.push(n.slice(2)):(o(),n.length>0&&(i.length>0&&i.push("<br>"),i.push(this.escapeHtml(n))));return o(),i.join("")}log(t,e,i,s){this.insert(this.rowHtml(i,this.renderMarkdown(s),t,e))}logScreenshot(t,e,i,s){if(i){let o=s===void 0?"png":"jpg",n=`${this.timestamp()}.${o}`;A.mkdir(this.logDir,{recursive:!0}).then(()=>A.writeFile(H.join(this.logDir,n),i)),this.insert(this.rowHtml(t,`${this.escapeHtml(e)}<br><img src="${n}">`,"info","Computer"))}else this.insert(this.rowHtml(t,this.escapeHtml(e),"error","Computer"))}};var F=class{computer;logger;constructor(t,e){this.computer=t,this.logger=e}log(t,e){this.logger.log("info","Computer",t,e)}async checkPermissions(t){return this.computer.checkPermissions(t)}async requestPermissions(t){return this.computer.requestPermissions(t)}async capabilities(){let t=await this.computer.capabilities();return this.log("capabilities",JSON.stringify(t)),t}async click(t,e,i,s,o=""){await this.computer.click(t,e,i,s,o),this.log("click",`(${t}, ${e}) button=${i} count=${s}${o?` display=${o}`:""}`)}async move(t,e,i=""){await this.computer.move(t,e,i),this.log("move",`(${t}, ${e})${i?` display=${i}`:""}`)}async drag(t,e,i,s,o=""){await this.computer.drag(t,e,i,s,o),this.log("drag",`(${t}, ${e}) \u2192 (${i}, ${s})${o?` display=${o}`:""}`)}async mouseDown(t,e,i=""){await this.computer.mouseDown(t,e,i),this.log("mouseDown",`(${t}, ${e})${i?` display=${i}`:""}`)}async mouseUp(t,e,i=""){await this.computer.mouseUp(t,e,i),this.log("mouseUp",`(${t}, ${e})${i?` display=${i}`:""}`)}async type(t){await this.computer.type(t),this.log("type",`"${t}"`)}async key(t){await this.computer.key(t),this.log("key",t)}async scroll(t,e,i,s,o=""){await this.computer.scroll(t,e,i,s,o),this.log("scroll",`(${t}, ${e}) dx=${i} dy=${s}${o?` display=${o}`:""}`)}async cursorPosition(t=""){let e=await this.computer.cursorPosition(t);return this.log("cursorPosition",`(${e.x}, ${e.y})${t?` display=${t}`:""}`),e}async display(t=""){let e=await this.computer.display(t);return this.log("display",`${e.width}x${e.height}${t?` display=${t}`:""}`),e}async listDisplays(){let t=await this.computer.listDisplays(),e=t.map(i=>`${i.isPrimary?"*":""}${i.displayId} "${i.label}" ${i.width}x${i.height}`).join("; ");return this.log("listDisplays",`${t.length} displays${e?`: ${e}`:""}`),t}async listWindows(){let t=await this.computer.listWindows(),e=new Map;for(let s of t){let o=s.displayId||"(unknown)",n=e.get(o)??{visible:0,minimized:0};s.isMinimized?n.minimized+=1:n.visible+=1,e.set(o,n)}let i=[...e.entries()].sort(([s],[o])=>s.localeCompare(o)).map(([s,o])=>`${s}: ${o.visible} visible, ${o.minimized} minimized`).join("; ");return this.log("listWindows",`${t.length} windows${i?`: ${i}`:""}`),t}async windowAtPoint(t,e,i){let s=await this.computer.windowAtPoint(t,e,i);return this.log("windowAtPoint",`display=${t||"(primary)"} (${e}, ${i}) -> ${s?`${s.windowId} title="${s.title}"`:"null"}`),s}async getActiveWindow(){let t=await this.computer.getActiveWindow();return this.log("getActiveWindow",t?`${t.windowId} title="${t.title}"`:"null"),t}async activateApplication(t){let e=await this.computer.activateApplication(t);return this.log("activateApplication",`${t} -> ${e}`),e}async concealApplication(t){let e=await this.computer.concealApplication(t);return this.log("concealApplication",`${t} -> ${e}`),e}async restoreApplication(t){let e=await this.computer.restoreApplication(t);return this.log("restoreApplication",`${t} -> ${e}`),e}async activateWindow(t){let e=await this.computer.activateWindow(t);return this.log("activateWindow",`${t} -> ${e}`),e}async concealWindow(t){let e=await this.computer.concealWindow(t);return this.log("concealWindow",`${t} -> ${e}`),e}async restoreWindow(t){let e=await this.computer.restoreWindow(t);return this.log("restoreWindow",`${t} -> ${e}`),e}async getClipboard(){let t=await this.computer.getClipboard(),e=t.slice(0,200)+(t.length>200?"\u2026":"");return this.log("getClipboard",`"${e}"`),t}async setClipboard(t){await this.computer.setClipboard(t);let e=t.slice(0,200)+(t.length>200?"\u2026":"");this.log("setClipboard",`"${e}" (${t.length} chars)`)}lock(t){return this.computer.lock(t)}unlock(){this.computer.unlock()}async prepareForInput(t,e,i){let s=await this.computer.prepareForInput(t,e,i),o=i?` @ (${i.x},${i.y})`:"";return this.log("prepareForInput",`${t} -> ${s} (${e.length} apps)${o}`),s}async screenshot(t,e,i,s,o,n,r){let c=await this.computer.screenshot(t,e,i,s,o,n,r),a=s&&o?` (${s}x${o})`:"",l=n?` crop=[${n.join(",")}]`:"",u=`display=${t||"(primary)"}`,f=r===void 0?"":` quality=${r}`,b=`${u}${a}${l}${f}`;return this.logger.logScreenshot("screenshot",b,c,r),c}};import{createRequire as xt}from"module";import{dirname as Pt,join as Wt}from"path";import{fileURLToPath as St}from"url";var z=class d{constructor(t,e){this.native=t;t.setLogger?.((i,s,o)=>{e?.log(this.nativeLogLevel(i),"Driver",s,o)})}native;static create(t){let e=Wt(Pt(St(import.meta.url)),"prebuilds",`${process.platform}-${process.arch}`,"computer.node"),i;try{i=xt(import.meta.url)(e)}catch(s){let o=s instanceof Error?s.message:String(s);throw new Error(`Native computer bindings not available for ${process.platform}-${process.arch}: ${o} (path: ${e})`,{cause:s})}return new d(i,t)}async checkPermissions(t){return this.native.checkPermissions(t)}async requestPermissions(t){this.native.requestPermissions(t)}async click(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,c]=this.toNative(n,t,e);this.native.click(r,c,i,s)}nativeLogLevel(t){switch(t){case 0:return"trace";case 1:return"debug";case 2:return"info";case 3:return"warn";case 4:return"error";default:return"info"}}async move(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.move(o,n)}async drag(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e),this.assertPointInBounds(n,i,s);let[r,c]=this.toNative(n,t,e),[a,l]=this.toNative(n,i,s);this.native.drag(r,c,a,l)}async mouseDown(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseDown(o,n)}async mouseUp(t,e,i=""){let s=this.resolveCoordinateSpace(i);this.assertPointInBounds(s,t,e);let[o,n]=this.toNative(s,t,e);this.native.mouseUp(o,n)}async type(t){this.native.type(t)}async key(t){this.native.key(t)}async scroll(t,e,i,s,o=""){let n=this.resolveCoordinateSpace(o);this.assertPointInBounds(n,t,e);let[r,c]=this.toNative(n,t,e);this.native.scroll(r,c,i,s)}async cursorPosition(t=""){let e=this.native.cursorPosition(),i=this.resolveCoordinateSpace(t),[s,o]=this.fromNative(i,e.x,e.y);return{x:s,y:o}}async display(t=""){let e=this.resolveCoordinateSpace(t);return{width:e.targetWidth,height:e.targetHeight}}async screenshot(t,e,i,s,o,n,r){let c=this.resolveCoordinateSpace(t,s,o),a=r??-1;if(n){this.assertRegionEdgeInBounds(c,n[0],n[1]),this.assertRegionEdgeInBounds(c,n[2],n[3]);let[l,u,f,b]=this.toNativeCrop(c,n[0],n[1],n[2],n[3]);return this.native.screenshot(t,e,i,0,0,l,u,f,b,a)}return this.native.screenshot(t,e,i,c.targetWidth,c.targetHeight,0,0,0,0,a)}async capabilities(){let t=this.native.capabilities();return{...t,canGetActiveWindow:typeof this.native.getActiveWindow=="function",canLocateWindowAtPoint:typeof this.native.windowAtPoint=="function"&&!!t.canLocateWindowAtPoint}}async listDisplays(){return this.native.listDisplays().map(e=>{let{targetWidth:i,targetHeight:s}=this.targetSizeForDisplay(e.bounds.width,e.bounds.height);return{displayId:e.displayId,label:e.label,width:i,height:s,isPrimary:e.isPrimary}})}async listWindows(){return this.native.listWindows()}async windowAtPoint(t,e,i){if(!this.native.windowAtPoint)return null;let s=this.resolveCoordinateSpace(t);this.assertPointInBounds(s,e,i);let[o,n]=this.toNative(s,e,i);return this.native.windowAtPoint(t,o,n)}async getActiveWindow(){return this.native.getActiveWindow?this.native.getActiveWindow():null}async activateApplication(t){return this.native.activateApplication(t)}async concealApplication(t){return this.native.concealApplication(t)}async restoreApplication(t){return this.native.restoreApplication(t)}async activateWindow(t){return this.native.activateWindow(t)}async concealWindow(t){return this.native.concealWindow(t)}async restoreWindow(t){return this.native.restoreWindow(t)}async getClipboard(){return this.native.getClipboard()}async setClipboard(t){this.native.setClipboard(t)}lock(t){return this.native.lock?.(t)??!1}unlock(){this.native.unlock?.()}async prepareForInput(t,e,i){let s=Number.NaN,o=Number.NaN;if(i){let n=this.resolveCoordinateSpace(t);this.assertPointInBounds(n,i.x,i.y),[s,o]=this.toNative(n,i.x,i.y)}return this.native.prepareForInput(t,e,i?.blockedWindowIds??[],s,o)}resolveDisplay(t){let e=this.native.listDisplays();return t!==""?e.find(i=>i.displayId===t):e.find(i=>i.isPrimary)??e[0]}targetSizeForDisplay(t,e,i=0,s=0){if(i>0&&s>0)return{targetWidth:i,targetHeight:s};let r=Math.min(1,1568/Math.max(t,e),Math.sqrt(115e4/(t*e)));return{targetWidth:Math.floor(t*r),targetHeight:Math.floor(e*r)}}resolveCoordinateSpace(t,e=0,i=0){let s=this.resolveDisplay(t),o=s?.bounds.x??0,n=s?.bounds.y??0,r=s?void 0:this.native.display(),c=s?.bounds.width??r?.width??1,a=s?.bounds.height??r?.height??1,{targetWidth:l,targetHeight:u}=this.targetSizeForDisplay(c,a,e,i);return{originX:o,originY:n,displayWidth:c,displayHeight:a,scaleX:l/c,scaleY:u/a,targetWidth:l,targetHeight:u}}toNative(t,e,i){return[t.originX+e/t.scaleX,t.originY+i/t.scaleY]}toNativeCrop(t,e,i,s,o){return[Math.floor(e/t.scaleX),Math.floor(i/t.scaleY),Math.ceil(s/t.scaleX),Math.ceil(o/t.scaleY)]}assertPointInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>=t.targetWidth||i>=t.targetHeight)throw new RangeError(`Coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}assertRegionEdgeInBounds(t,e,i){if(!Number.isFinite(e)||!Number.isFinite(i))throw new RangeError(`Coordinates must be finite numbers: x=${e}, y=${i}`);if(e<0||i<0||e>t.targetWidth||i>t.targetHeight)throw new RangeError(`Crop coordinates out of bounds for target display: x=${e}, y=${i}, width=${t.targetWidth}, height=${t.targetHeight}`)}fromNative(t,e,i){return[Math.round((e-t.originX)*t.scaleX),Math.round((i-t.originY)*t.scaleY)]}};var _=.8,_t=250,W={readOnlyHint:!0,destructiveHint:!1,openWorldHint:!1},I={readOnlyHint:!1,destructiveHint:!0,openWorldHint:!0},Z={readOnlyHint:!1,destructiveHint:!1,openWorldHint:!1},w=" Requires `request_access`.",rt=" If the target app is missing, call `list_applications` then `request_access`.",T=p.coerce.number().transform(Math.round),q=p.object({x:T.describe("Horizontal pixel coordinate."),y:T.describe("Vertical pixel coordinate.")}),Tt=p.object({x1:T.describe("Left edge of the region in screenshot pixel coordinates."),y1:T.describe("Top edge of the region in screenshot pixel coordinates."),x2:T.describe("Right edge of the region in screenshot pixel coordinates."),y2:T.describe("Bottom edge of the region in screenshot pixel coordinates.")}).refine(({x1:d,x2:t})=>t>d,{message:"x2 must be greater than x1.",path:["x2"]}).refine(({y1:d,y2:t})=>t>d,{message:"y2 must be greater than y1.",path:["y2"]}),Nt=p.string().min(1).describe("Application name or stable app id from `list_applications`. Examples: 'Google Chrome', 'Microsoft Outlook', 'app.windows.abc123'."),Dt=p.object({display_id:p.string().optional().describe("Display id to select. Omit or pass an empty string to use the default display.")}),Mt=p.object({display_id:p.string().optional().describe("Optional display id to filter by. When provided, only apps with at least one non-minimized window on that display are returned.")}),ct=q.optional().describe("{x, y} pixel coordinate. Clicks at current cursor position if omitted."),lt=p.enum(["left","right","middle"]).optional().describe("Mouse button to click (default left)."),pt=p.coerce.number().int().min(1).max(3).optional().describe("Number of clicks: 1 single, 2 double, 3 triple (default 1)."),dt=q.describe("{x, y} pixel coordinate to move the cursor to."),ut=q.describe("{x, y} pixel coordinate to start the drag from."),ht=q.describe("{x, y} pixel coordinate to drag to."),mt=p.string().describe("The text to type."),gt=p.string().describe("Key combo to press (e.g. 'Return', 'ctrl+s', 'alt+Tab')."),wt=p.enum(["up","down","left","right"]).describe("Direction to scroll."),yt=q.optional().describe("{x, y} pixel coordinate to scroll at. Scrolls at current cursor position if omitted."),ft=p.number().int().nonnegative().max(100).optional().describe("Number of scroll clicks (default 3)."),Rt=p.number().nonnegative().max(100).describe("Seconds to wait (max 100). Use after a screenshot shows the UI is not in the expected state yet, but an operation may still finish."),bt=p.string().describe("The text to copy to the clipboard."),qt=p.object({action:p.literal("click"),coordinate:ct,button:lt,count:pt}),Lt=p.object({action:p.literal("mouse_move"),coordinate:dt}),Et=p.object({action:p.literal("left_click_drag"),start_coordinate:ut,coordinate:ht}),jt=p.object({action:p.literal("left_mouse_down")}),Bt=p.object({action:p.literal("left_mouse_up")}),Ht=p.object({action:p.literal("type"),text:mt}),Ut=p.object({action:p.literal("key"),text:gt}),Ft=p.object({action:p.literal("scroll"),scroll_direction:wt,coordinate:yt,scroll_amount:ft}),zt=p.object({action:p.literal("set_clipboard"),text:bt}),Ot=p.discriminatedUnion("action",[qt,Lt,Et,jt,Bt,Ht,Ut,Ft,zt]),Yt=p.object({actions:p.array(Ot).min(1).describe("Ordered actions. Do not include waits; call the separate `wait` tool only after observing a screenshot that is not ready yet.")}),O=class d{constructor(t={}){this.options=t;process.env.DEBUG&&(this.logger=new U)}options;_computer;_caps;_locked=null;_safetyTimer;static SAFETY_TIMEOUT_MS=180*1e3;server;logger;accessStore=new j;appNameResolver=new B;lastUnsharedWindowsNote=null;selectedDisplay="";toText(t,e){let i=[{type:"text",text:t}];return e?{content:i,isError:!0}:{content:i}}toJson(t){return this.toText(JSON.stringify(t,null,2))}toImage(t,e,i){return t?{content:[{type:"image",data:t.toString("base64"),mimeType:this.imageMimeType(i),_meta:{screenshot:!0}}]}:this.toText(e,!0)}buildAccessMessage(t,e){let i=t.length===0?["","Computer Use runs on your actual desktop and can send mouse and keyboard input.","","Computer Use wants to control your desktop for this session.","","Apps that are not allowed may be hidden."]:["","Computer Use runs on your actual desktop and can send mouse and keyboard input to the apps you share.","","Computer Use wants to control these apps:","",...t.map(s=>`- ${s}`),"","Apps that are not allowed may be hidden."];return e&&i.push("","Reason:",e),i.join(`
|
|
33
|
+
`)}imageMimeType(t){return t!==void 0?"image/jpeg":"image/png"}collectDisplayIds(t,e){let i=new Set;for(let s of t)!s.isMinimized&&s.displayId&&(!e||s.applicationId===e)&&i.add(s.displayId);return[...i]}buildAllowedAppsInfo(t,e){return t.map(i=>({appId:i.id,name:i.displayName,displayIds:this.collectDisplayIds(e,i.id)}))}connect(t){this.server=t;let e=p.object({method:p.literal("notifications/copilot"),params:p.object({type:p.string()}).passthrough()});t.server.setNotificationHandler(e,s=>{switch(s.params.type){case"assistant.turn_start":this._locked===null&&(this._locked=!1),this.unlock();break;case"assistant.turn_end":this.unlock();break}}),t.registerTool("list_displays",{description:"List available displays and the currently selected display used for screenshots, zoom, cursor position, and coordinate-based actions.",annotations:W},()=>this.list_displays()),t.registerTool("select_display",{description:"Select the active display used for screenshots, zoom, cursor position, and coordinate-based actions. Omit display_id or pass an empty string to use the default display.",inputSchema:Dt.shape,annotations:Z},s=>this.select_display(s)),t.registerTool("list_applications",{description:"List running apps and `selectedDisplay`. Use for discovery only; `request_access` can resolve named apps directly.",inputSchema:Mt.shape,annotations:W},s=>this.list_applications(s)),t.registerTool("request_access",{description:"Request app access by friendly name or appId; pass [] for all visible apps. For a named app, call this directly; it finds the app and selects its display. Returns JSON (`allowed`, `allowAll`, `message`, `allowedApps?`, `selectedDisplay?`) and, when allowed, a screenshot. `allowAll=true` covers future apps for this session.",inputSchema:{apps:p.array(Nt).describe('Applications to allow. Each entry can be a friendly name (e.g. "Outlook") or a stable appId, if already known. Pass an empty array to allow all apps on the selected display.'),reason:p.string().min(1).optional().describe("Optional reason text shown in the access dialog.")},annotations:Z},(s,o)=>this.request_access(s,o.signal)),t.registerTool("screenshot",{description:"Capture the current filtered screen. Use returned post-action screenshots when available. "+rt+w,annotations:W},()=>this.screenshot()),t.registerTool("cursor_position",{description:"Get the current cursor position in pixel coordinates. Returns {x, y}."+w,annotations:W},()=>this.cursor_position());let i={coordinate:ct,button:lt,count:pt};t.registerTool("click",{description:"Click a mouse button. Defaults to a single left click; use `button` for right/middle click and `count` 2 or 3 for double/triple-click. Optionally move to a coordinate first."+w,inputSchema:i,annotations:I},s=>this.click(s)),t.registerTool("mouse_move",{description:"Move the mouse cursor to the specified pixel coordinate."+w,inputSchema:{coordinate:dt},annotations:I},s=>this.mouse_move(s)),t.registerTool("left_click_drag",{description:"Click and drag from a start coordinate to an end coordinate."+w,inputSchema:{start_coordinate:ut,coordinate:ht},annotations:I},s=>this.left_click_drag(s)),t.registerTool("left_mouse_down",{description:"Press and hold the left mouse button at the current cursor position."+w,annotations:I},()=>this.left_mouse_down()),t.registerTool("left_mouse_up",{description:"Release the left mouse button at the current cursor position."+w,annotations:I},()=>this.left_mouse_up()),t.registerTool("type",{description:"Type a string for text input fields. Use `key` for physical-key controls."+w,inputSchema:{text:mt},annotations:I},s=>this.type(s)),t.registerTool("key",{description:"Press a key or combo, e.g. 'Return', 'ctrl+s', 'alt+Tab', '1'."+w,inputSchema:{text:gt},annotations:I},s=>this.key(s)),t.registerTool("scroll",{description:"Scroll the screen in a given direction at an optional coordinate."+w,inputSchema:{scroll_direction:wt,coordinate:yt,scroll_amount:ft},annotations:I},s=>this.scroll(s)),t.registerTool("wait",{description:"Pause, then return an updated screenshot if access is active. Use after a screenshot shows the UI is not ready yet.",inputSchema:{duration:Rt},annotations:{...W,idempotentHint:!0}},s=>this.wait(s)),t.registerTool("get_clipboard",{description:"Get the current text contents of the system clipboard."+w,annotations:W},()=>this.get_clipboard()),t.registerTool("set_clipboard",{description:"Set the system clipboard to the specified text."+w,inputSchema:{text:bt},annotations:Z},s=>this.set_clipboard(s)),t.registerTool("zoom",{description:"Capture a filtered screen region at full resolution."+rt+w,inputSchema:{region:Tt.describe("{x1, y1, x2, y2} coordinates defining top-left and bottom-right corners of the region to capture.")},annotations:W},s=>this.zoom(s)),t.registerTool("batch",{description:"Run ordered actions and return one final screenshot after a built-in settle delay. Use for predictable sequences; wait is intentionally not supported. Supported actions: click, mouse_move, left_click_drag, left_mouse_down, left_mouse_up, type, key, scroll, set_clipboard.",inputSchema:Yt.shape,annotations:I},s=>this.batch(s))}lock(){this._locked===null||!this._computer||(this._locked||(this._locked=this._computer.lock(()=>{this.unlock(),this.server.server.notification({method:"notifications/copilot",params:{type:"user.abort"}}).catch(()=>{});let t=this.accessStore.getState().hostWindowId;t&&this._computer&&this._computer.activateWindow(t).catch(()=>{})})),this._safetyTimer&&clearTimeout(this._safetyTimer),this._safetyTimer=setTimeout(()=>this.unlock(),d.SAFETY_TIMEOUT_MS))}unlock(){this._locked===null||!this._computer||(this._safetyTimer&&(clearTimeout(this._safetyTimer),this._safetyTimer=void 0),this._locked&&this._computer.unlock(),this._locked=!1)}async computer(){return this._computer||(this._computer=z.create(this.logger),this.logger&&(this._computer=new F(this._computer,this.logger))),this._computer}logEvent(t,e){this.logger?.log("info","Server",t,e)}async capabilities(){if(!this._caps){let t=await this.computer();this._caps=await t.capabilities()}return this._caps}async requestPermission(){switch(process.platform){case"linux":{await(await this.computer()).display();break}case"darwin":{let t=await this.computer(),e=[],i=await t.checkPermissions("accessibility");i||e.push("Accessibility");let s=await t.checkPermissions("screen");if(s||e.push("Screen Recording"),e.length>0){if(!this.server.server.getClientCapabilities()?.elicitation?.form)break;let n=["",`Computer Use needs the following macOS permission${e.length>1?"s":""} to control your desktop:`,"",...e.map(a=>`\u2022 ${a}`),"",'Clicking "Open System Settings" will open the relevant settings page(s).',`Enable the permission${e.length>1?"s":""} for your terminal app, then click "I've granted the permissions".`,"","Note: You may need to restart your terminal after granting permissions for them to take effect."];await this.showHostWindow(t);let r=await this.server.server.elicitInput({mode:"form",message:n.join(`
|
|
34
34
|
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:`Grant ${e.join(" and ")} permission${e.length>1?"s":""}`,oneOf:[{const:"open",title:"1. Open System Settings"},{const:"done",title:"2. I've granted the permissions"},{const:"skip",title:"3. Skip (things may not work)"}]}},required:["action"]}}),c=r.action==="accept"?r.content?.action:void 0;if(c==="open"){i||await t.requestPermissions("accessibility"),s||await t.requestPermissions("screen"),await this.showHostWindow(t);let a=await this.server.server.elicitInput({mode:"form",message:["","System Settings has been opened.","",`Enable ${e.join(" and ")} for your terminal app.`,e.length>1?"Both settings pages have been opened \u2014 check each one.":"","",`After granting permissions, click "I've granted the permissions" below.`,"If the toggle was already on, try removing and re-adding your terminal app."].filter(Boolean).join(`
|
|
35
|
-
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:"Confirm permissions",oneOf:[{const:"done",title:"1. I've granted the permissions"},{const:"skip",title:"2. Skip (things may not work)"}]}},required:["action"]}});if((a.action==="accept"?a.content?.action:void 0)!=="done")break}else if(c!=="done")break;if(i=await t.checkPermissions("accessibility"),s=await t.checkPermissions("screen"),!i||!s){let a=[];throw i||a.push("Accessibility"),s||a.push("Screen Recording"),new Error(`${a.join(" and ")} permission${a.length>1?"s are":" is"} still not detected. This usually means you need to restart your terminal after granting the permission. Please quit and reopen your terminal, then try again.`)}}break}}}async activeDisplayId(t){return this.selectedDisplay!==""?this.selectedDisplay:(await t.listDisplays()).find(i=>i.isPrimary)?.displayId??""}async requireAccess(){let t=this.accessStore.getState();if(!t.accessActive)throw new Error("No access session is active. Call request_access first to start a session.");let e=await this.computer();if(t.allowAll&&(await this.capabilities()).canListWindows){let{windows:s}=await this.filterHostWindow(e),o=W(s);this.accessStore.rememberApplications(o),this.appNameResolver.updateApplications(o),t=this.accessStore.getState()}return{computer:e,allowedAppIds:t.allowedAppIds,hostWindowId:t.hostWindowId}}async hostWindowId(t){let e=this.accessStore.getState().hostWindowId;if(e)return e;if(!(await this.capabilities()).canListWindows)return null;let s=await t.getActiveWindow();return s&&(await t.listWindows()).some(n=>n.windowId===s.windowId)?s.windowId:null}async showHostWindow(t){let e=await this.hostWindowId(t);if(e)try{await t.activateWindow(e)}catch{}}async filterHostWindow(t,e){let i=e??await t.listWindows(),s=await this.hostWindowId(t);return{windows:s?i.filter(o=>o.windowId!==s):i,hostWindowId:s}}async prepareForInput(t,e,i){let s=await this.capabilities();if(!s.canListWindows)return;let o=await this.activeDisplayId(t),n=i&&s.canHitTest;if(e.length===0&&!n)return;if(!await t.prepareForInput(o,e,n?i:void 0))throw n?new Error("Input blocked: a disallowed app at the target could not be hidden. Call `list_applications` to see what's there, then `request_access` to allow it."):new Error("No allowed visible windows are available on the selected display.")}async validateActiveWindow(t,e){if(!(await this.capabilities()).canGetActiveWindow)return;let s=await t.getActiveWindow();if(s&&!e.includes(s.applicationId))throw new Error(`Keyboard input blocked: the focused application ("${s.applicationName}") is not allowed. Click an allowed application first, or call \`request_access\` to allow it.`)}async list_displays(){let t=await this.computer(),e=await t.listDisplays(),i=await this.activeDisplayId(t);return this.toJson({displays:e,selectedDisplay:i})}async select_display({display_id:t}){let e=t??"",i=await this.computer(),s=await i.listDisplays();if(e!==""&&!s.some(r=>r.displayId===e))throw new Error(`Unknown display id '${e}'. Call list_displays first.`);this.selectedDisplay=e;let o=await this.activeDisplayId(i),n=s.find(r=>r.displayId===o)??null;return this.toJson({selectedDisplay:o,status:"selected",message:e===""?"Using the default display for screenshots, zoom, cursor position, and coordinate-based actions.":`Selected display '${n?.label??e}' for screenshots, zoom, cursor position, and coordinate-based actions.`})}async list_applications({display_id:t}={}){let e=await this.computer(),{windows:i}=await this.filterHostWindow(e),s=W(i);this.accessStore.rememberApplications(s),this.appNameResolver.updateApplications(s);let o=t??"";if(o!==""&&!(await e.listDisplays()).some(l=>l.displayId===o))throw new Error(`Unknown display id '${o}'. Call list_displays first.`);let n=o===""?i:i.filter(a=>!a.isMinimized&&a.displayId===o),r=W(n),c=await this.activeDisplayId(e);return this.toJson({selectedDisplay:c,applications:r})}async request_access({apps:t,reason:e}){this.logEvent("request_access",`Start: apps=[${t.join(", ")}]${e?` reason="${e}"`:""}`),await this.requestPermission();let i={const:"allow",title:"Allow"},s={const:"allow_all",title:"Allow all apps (don't ask again)"},o={const:"deny",title:"Deny (Esc)"},n,r=async(m,y,D)=>{this.logEvent("request_access",`End: allowed=${m} allowAll=${y.allowAll} selectedDisplay=${this.selectedDisplay||"(default)"} hostWindowId=${y.hostWindowId??"null"} message="${D}"`);let $=n??(y.allowAll?void 0:y.allowedAppIds.map(v=>{let x=this.accessStore.tryGetKnownApplication(v);return{appId:v,name:x?.displayName,displayIds:vt(x?.windows??[])}})),N=this.toJson({allowed:m,allowAll:y.allowAll,...$?{allowedApps:$}:{},selectedDisplay:this.selectedDisplay||void 0,message:D});if(!m)return N;let P=await this.computer();this.lock();let X=y.hostWindowId?[y.hostWindowId]:[];return this.addScreenshot(N,P,y.allowedAppIds,X)};if(this.options.yolo){let m=this.accessStore.allowApplications([],!0);return await r(!0,m,"YOLO mode is enabled. Auto-allowing full desktop access for this session.")}let c=await this.capabilities();if(!this.server.server.getClientCapabilities()?.elicitation?.form){let m=this.accessStore.allowApplications([],!0);return await r(!0,m,"Elicitation is not supported by this client. Auto-allowing full desktop access for this session.")}let l,u,f,h,Y;if(c.canListWindows){let m=await this.computer(),y=this.accessStore.getState().hostWindowId,D=await m.listWindows(),$=y?D.filter(g=>g.windowId!==y):D;Y=$;let N=W($);this.accessStore.rememberApplications(N),this.appNameResolver.updateApplications(N);let P=t&&t.length>0?"":await this.activeDisplayId(m);P&&(this.selectedDisplay=P);let X=new Set(y?[y]:[]),v;if(t&&t.length>0){let g=new Map;for(let k of t){let q=this.appNameResolver.resolve(k);if(q.length===0)throw new Error(`No matching application found for '${k}'. Call list_applications to all running apps and their IDs.`);for(let _ of q)if(!g.has(_.appId)){let j=this.accessStore.tryGetKnownApplication(_.appId);g.set(_.appId,j??{id:_.appId,displayName:_.displayName,windows:[]})}}v=[...g.values()];let It=new Set(g.keys()),L=new Map;for(let k of $)It.has(k.applicationId)&&!k.isMinimized&&k.displayId&&L.set(k.displayId,(L.get(k.displayId)??0)+1);if(L.size>0){let k="",q=0;for(let[_,j]of L)j>q&&(k=_,q=j);this.selectedDisplay=k}n=lt(v,$)}else v=W($.filter(g=>!g.isMinimized&&(P===""||g.displayId===P)&&!X.has(g.windowId)));let x=v.map(g=>g.id),K=this.accessStore.getState(),R=v.length;u=["allow","allow_all"],f=x.length===0?K.allowAll||K.accessActive:this.accessStore.areAllowedForAccess(x),h=x.length===0?K.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.":R===1?`'${v[0].displayName}' is already shared for this session.`:"The requested apps are already shared for this session.",l={choices:[i,s,o],message:rt(v.map(g=>g.displayName),e),denyMessage:R===0?"The user declined desktop access.":R===1?`The user declined to share '${v[0].displayName}'.`:"The user declined to share the requested apps.",allow:g=>this.accessStore.allowApplications(x,g==="allow_all"),allowedMessage:g=>g==="allow_all"?"All current and future apps are allowed for the rest of this session, so you do not need to call request_access again.":R===0?"Desktop access is active for this session.":R===1?`Access session started for '${v[0].displayName}'.`:"Access session started for the requested apps."}}else{let m=this.accessStore.getState();u=["allow_all"],f=m.allowAll||m.accessActive,h=m.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.",l={choices:[s,o],message:rt([],e),denyMessage:"The user declined desktop access.",allow:()=>this.accessStore.allowApplications([],!0),allowedMessage:()=>"Access session started for the desktop. All current and future apps are allowed for the rest of this session, so you do not need to call request_access again."}}if(f){let m=this.accessStore.getState();return await r(!0,m,h)}this.accessStore.getState().hostWindowId&&await this.showHostWindow(await this.computer());let G=await this.server.server.request({method:"elicitation/create",params:{mode:"form",message:l.message,requestedSchema:{type:"object",properties:{decision:{type:"string",title:"Allow access for this session?",oneOf:l.choices}},required:["decision"]}}},Dt),At=await this.computer(),Z=await this.hostWindowId(At);Z&&this.accessStore.setHostWindowId(Z);let I=G.action==="accept"?G.content?.decision:void 0;if(G.action!=="accept"||!I||typeof I!="string"||!u.includes(I)){let m=this.accessStore.getState(),y=typeof I=="string"&&I!==o.const?`The user did not approve this access request. User response: ${I}`:l.denyMessage;return await r(!1,m,y)}let tt=l.allow(I);return tt.allowAll&&Y&&(n=lt(this.accessStore.allKnownApplications(),Y)),await r(!0,tt,l.allowedMessage(I))}async screenshot(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[];return this.captureScreenshot(t,e,s)}logScreenshotFilter(t,e){let i=r=>{let c=this.accessStore.tryGetKnownApplication(r);return c?`${c.displayName} (${r})`:r},s=r=>{let c=r.includes("|")?r.split("|")[1]:r;return i(c)},o=(r,c)=>c.length===1?`${r}: ${c[0]}`:`${r}:
|
|
35
|
+
`),requestedSchema:{type:"object",properties:{action:{type:"string",title:"Confirm permissions",oneOf:[{const:"done",title:"1. I've granted the permissions"},{const:"skip",title:"2. Skip (things may not work)"}]}},required:["action"]}});if((a.action==="accept"?a.content?.action:void 0)!=="done")break}else if(c!=="done")break;if(i=await t.checkPermissions("accessibility"),s=await t.checkPermissions("screen"),!i||!s){let a=[];throw i||a.push("Accessibility"),s||a.push("Screen Recording"),new Error(`${a.join(" and ")} permission${a.length>1?"s are":" is"} still not detected. This usually means you need to restart your terminal after granting the permission. Please quit and reopen your terminal, then try again.`)}}break}}}async activeDisplayId(t){return this.selectedDisplay!==""?this.selectedDisplay:(await t.listDisplays()).find(i=>i.isPrimary)?.displayId??""}async requireAccess(){let t=this.accessStore.getState();if(!t.accessActive)throw new Error("No access session is active. Call request_access first to start a session.");let e=await this.computer();if(t.allowAll&&(await this.capabilities()).canListWindows){let{windows:s}=await this.filterHostWindow(e),o=C(s);this.accessStore.rememberApplications(o),this.appNameResolver.updateApplications(o),t=this.accessStore.getState()}return{computer:e,allowedAppIds:t.allowedAppIds,hostWindowId:t.hostWindowId}}async hostWindowId(t){let e=this.accessStore.getState().hostWindowId;if(e)return e;if(!(await this.capabilities()).canListWindows)return null;let s=await t.getActiveWindow();return s&&(await t.listWindows()).some(n=>n.windowId===s.windowId)?s.windowId:null}async showHostWindow(t){let e=await this.hostWindowId(t);if(e)try{await t.activateWindow(e)}catch{}}async filterHostWindow(t,e){let i=e??await t.listWindows(),s=await this.hostWindowId(t);return{windows:s?i.filter(o=>o.windowId!==s):i,hostWindowId:s}}async prepareForInput(t,e,i){let s=await this.capabilities();if(!s.canListWindows)return;let o=await this.activeDisplayId(t),n=i&&s.canHitTest;if(e.length===0&&!n)return;if(!await t.prepareForInput(o,e,n?i:void 0)){if(n)throw new Error("Input blocked: a disallowed app at the target could not be hidden. Call `list_applications` to see what's there, then `request_access` to allow it.");if(s.canGetActiveWindow){let c=await t.getActiveWindow();if(c&&!e.includes(c.applicationId))throw new Error(this.keyboardInputBlockedMessage(c))}throw new Error("Keyboard input blocked: an allowed application could not be focused. Click an allowed application first, or call `request_access` to allow it.")}}keyboardInputBlockedMessage(t){return`Keyboard input blocked: the focused application ("${t.applicationName}") is not allowed. Click an allowed application first, or call \`request_access\` to allow it.`}async validateActiveWindow(t,e){if(!(await this.capabilities()).canGetActiveWindow)return;let s=await t.getActiveWindow();if(s&&!e.includes(s.applicationId))throw new Error(this.keyboardInputBlockedMessage(s))}pointerInputBlockedMessage(t){return`Pointer input blocked: the focused application ("${t.applicationName}") is not allowed. Focus an allowed application first, or call \`request_access\` to allow it.`}async validatePointerInput(t,e,i){if(e.length===0)return;let s=await this.capabilities();if(s.canGetActiveWindow){let c=await t.getActiveWindow();if(c&&!e.includes(c.applicationId))throw new Error(this.pointerInputBlockedMessage(c))}if(!s.canLocateWindowAtPoint)return;let o=await this.activeDisplayId(t),n=await t.windowAtPoint(o,i.x,i.y);if(!n||e.includes(n.applicationId))return;let r=n.applicationName||n.title||n.applicationId;throw new Error(`Pointer input blocked: the target location is covered by disallowed application ("${r}"). Call \`list_applications\` to see what's there, then \`request_access\` to allow it.`)}async list_displays(){let t=await this.computer(),e=await t.listDisplays(),i=await this.activeDisplayId(t);return this.toJson({displays:e,selectedDisplay:i})}async select_display({display_id:t}){let e=t??"",i=await this.computer(),s=await i.listDisplays();if(e!==""&&!s.some(r=>r.displayId===e))throw new Error(`Unknown display id '${e}'. Call list_displays first.`);this.selectedDisplay=e;let o=await this.activeDisplayId(i),n=s.find(r=>r.displayId===o)??null;return this.toJson({selectedDisplay:o,status:"selected",message:e===""?"Using the default display for screenshots, zoom, cursor position, and coordinate-based actions.":`Selected display '${n?.label??e}' for screenshots, zoom, cursor position, and coordinate-based actions.`})}async list_applications({display_id:t}={}){let e=await this.computer(),{windows:i}=await this.filterHostWindow(e),s=C(i);this.accessStore.rememberApplications(s),this.appNameResolver.updateApplications(s);let o=t??"";if(o!==""&&!(await e.listDisplays()).some(l=>l.displayId===o))throw new Error(`Unknown display id '${o}'. Call list_displays first.`);let n=o===""?i:i.filter(a=>!a.isMinimized&&a.displayId===o),r=C(n),c=await this.activeDisplayId(e);return this.toJson({selectedDisplay:c,applications:r})}async request_access({apps:t,reason:e},i){this.logEvent("request_access",`Start: apps=[${t.join(", ")}]${e?` reason="${e}"`:""}`),await this.requestPermission();let s={const:"allow",title:"Allow"},o={const:"allow_all",title:"Allow all apps (don't ask again)"},n={const:"deny",title:"Deny (Esc)"},r,c=async(h,g,N)=>{this.logEvent("request_access",`End: allowed=${h} allowAll=${g.allowAll} selectedDisplay=${this.selectedDisplay||"(default)"} hostWindowId=${g.hostWindowId??"null"} message="${N}"`);let $=r??(g.allowAll?void 0:g.allowedAppIds.map(y=>{let x=this.accessStore.tryGetKnownApplication(y);return{appId:y,name:x?.displayName,displayIds:this.collectDisplayIds(x?.windows??[])}})),D=this.toJson({allowed:h,allowAll:g.allowAll,...$?{allowedApps:$}:{},selectedDisplay:this.selectedDisplay||void 0,message:N});if(!h)return D;let S=await this.computer();this.lock();let K=g.hostWindowId?[g.hostWindowId]:[];return this.addScreenshot(D,S,g.allowedAppIds,K)};if(this.options.yolo){let h=this.accessStore.allowApplications([],!0);return await c(!0,h,"YOLO mode is enabled. Auto-allowing full desktop access for this session.")}let a=await this.capabilities();if(!this.server.server.getClientCapabilities()?.elicitation?.form){let h=this.accessStore.allowApplications([],!0);return await c(!0,h,"Elicitation is not supported by this client. Auto-allowing full desktop access for this session.")}let u,f,b,Y,G;if(a.canListWindows){let h=await this.computer(),g=this.accessStore.getState().hostWindowId,N=await h.listWindows(),$=g?N.filter(m=>m.windowId!==g):N;G=$;let D=C($);this.accessStore.rememberApplications(D),this.appNameResolver.updateApplications(D);let S=t&&t.length>0?"":await this.activeDisplayId(h);S&&(this.selectedDisplay=S);let K=new Set(g?[g]:[]),y;if(t&&t.length>0){let m=new Map;for(let v of t){let R=this.appNameResolver.resolve(v);if(R.length===0)throw new Error(`No matching application found for '${v}'. Call list_applications to all running apps and their IDs.`);for(let P of R)if(!m.has(P.appId)){let E=this.accessStore.tryGetKnownApplication(P.appId);m.set(P.appId,E??{id:P.appId,displayName:P.displayName,windows:[]})}}y=[...m.values()];let It=new Set(m.keys()),L=new Map;for(let v of $)It.has(v.applicationId)&&!v.isMinimized&&v.displayId&&L.set(v.displayId,(L.get(v.displayId)??0)+1);if(L.size>0){let v="",R=0;for(let[P,E]of L)E>R&&(v=P,R=E);this.selectedDisplay=v}r=this.buildAllowedAppsInfo(y,$)}else y=C($.filter(m=>!m.isMinimized&&(S===""||m.displayId===S)&&!K.has(m.windowId)));let x=y.map(m=>m.id),J=this.accessStore.getState(),M=y.length;f=["allow","allow_all"],b=x.length===0?J.allowAll||J.accessActive:this.accessStore.areAllowedForAccess(x),Y=x.length===0?J.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.":M===1?`'${y[0].displayName}' is already shared for this session.`:"The requested apps are already shared for this session.",u={choices:[s,o,n],message:this.buildAccessMessage(y.map(m=>m.displayName),e),denyMessage:M===0?"The user declined desktop access.":M===1?`The user declined to share '${y[0].displayName}'.`:"The user declined to share the requested apps.",allow:m=>this.accessStore.allowApplications(x,m==="allow_all"),allowedMessage:m=>m==="allow_all"?"All current and future apps are allowed for the rest of this session, so you do not need to call request_access again.":M===0?"Desktop access is active for this session.":M===1?`Access session started for '${y[0].displayName}'.`:"Access session started for the requested apps."}}else{let h=this.accessStore.getState();f=["allow_all"],b=h.allowAll||h.accessActive,Y=h.allowAll?"Access is already configured to auto-allow future requests for this session, so you do not need to call request_access again.":"Desktop access is already active for this session.",u={choices:[o,n],message:this.buildAccessMessage([],e),denyMessage:"The user declined desktop access.",allow:()=>this.accessStore.allowApplications([],!0),allowedMessage:()=>"Access session started for the desktop. All current and future apps are allowed for the rest of this session, so you do not need to call request_access again."}}if(b){let h=this.accessStore.getState();return await c(!0,h,Y)}this.accessStore.getState().hostWindowId&&await this.showHostWindow(await this.computer());let X=await this.server.server.request({method:"elicitation/create",params:{mode:"form",message:u.message,requestedSchema:{type:"object",properties:{decision:{type:"string",title:"Allow access for this session?",oneOf:u.choices}},required:["decision"]}}},Ct,{signal:i,timeout:300*1e3}),At=await this.computer(),tt=await this.hostWindowId(At);tt&&this.accessStore.setHostWindowId(tt);let k=X.action==="accept"?X.content?.decision:void 0;if(X.action!=="accept"||!k||typeof k!="string"||!f.includes(k)){let h=this.accessStore.getState(),g=typeof k=="string"&&k!==n.const?`The user did not approve this access request. User response: ${k}`:u.denyMessage;return await c(!1,h,g)}let et=u.allow(k);return et.allowAll&&G&&(r=this.buildAllowedAppsInfo(this.accessStore.allKnownApplications(),G)),await c(!0,et,u.allowedMessage(k))}async screenshot(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[];return this.captureScreenshot(t,e,s)}logScreenshotFilter(t,e){let i=r=>{let c=this.accessStore.tryGetKnownApplication(r);return c?`${c.displayName} (${r})`:r},s=r=>{let c=r.includes("|")?r.split("|")[1]:r;return i(c)},o=(r,c)=>c.length===1?`${r}: ${c[0]}`:`${r}:
|
|
36
36
|
${c.map(a=>`- ${a}`).join(`
|
|
37
37
|
`)}`,n=[t.length?o("allowed",t.map(i)):"allowed: (all apps)"];e.length&&n.push(o("blocked",e.map(s))),this.logEvent("screenshot filter",n.join(`
|
|
38
|
-
`))}async captureScreenshot(t,e,i){this.logScreenshotFilter(e,i);let s=await t.screenshot(this.selectedDisplay,e,i,0,0,void 0,
|
|
38
|
+
`))}async captureScreenshot(t,e,i){this.logScreenshotFilter(e,i);let s=await t.screenshot(this.selectedDisplay,e,i,0,0,void 0,_),o=this.toImage(s,"Screenshot failed",_);if(s){let n=await this.unsharedWindowsNote(t,e,i);this.appendUnsharedWindowsNote(o,n,!0)}return o}async addScreenshot(t,e,i,s){await new Promise(r=>setTimeout(r,_t)),this.logScreenshotFilter(i,s);let o=null;try{o=await e.screenshot(this.selectedDisplay,i,s,0,0,void 0,_)}catch{}if(!o)return t.content.push({type:"text",text:"Screenshot capture failed"}),t;t.content.push({type:"image",data:o.toString("base64"),mimeType:this.imageMimeType(_),_meta:{screenshot:!0}});let n=await this.unsharedWindowsNote(e,i,s);return this.appendUnsharedWindowsNote(t,n,!1),t}appendUnsharedWindowsNote(t,e,i){if(!e){this.lastUnsharedWindowsNote=null;return}(i||e!==this.lastUnsharedWindowsNote)&&t.content.push({type:"text",text:e}),this.lastUnsharedWindowsNote=e}async unsharedWindowsNote(t,e,i){if(!(await this.capabilities()).canListWindows)return null;let o=await this.activeDisplayId(t),n=new Set(e),r=new Set(i),{windows:c}=await this.filterHostWindow(t),a=c.filter(u=>!u.isMinimized&&(o===""||u.displayId===o)&&!n.has(u.applicationId)&&!r.has(u.windowId));if(a.length===0)return null;let l=a.length===1?"window":"windows";return`${a.length} other ${l} on this display ${a.length===1?"is":"are"} from an unshared app. The screenshot may not show everything that's running. Use \`list_applications\` to see what else is there if you need to.`}async cursor_position(){let e=await(await this.computer()).cursorPosition(this.selectedDisplay);return this.toText(`${e.x},${e.y}`)}async _click(t,e,i,s,o,n,r){let c;switch(n){case 1:c="";break;case 2:c="double ";break;case 3:c="triple ";break;default:throw new Error(`Invalid click count '${n}'. Expected 1, 2, or 3.`)}let a=r??await t.cursorPosition(s);await this.prepareForInput(t,e,{x:a.x,y:a.y,blockedWindowIds:i}),await this.validatePointerInput(t,e,a),r&&await t.move(a.x,a.y,s),await t.click(a.x,a.y,o,n,s);let l=r?` at (${r.x},${r.y})`:"",u=o==="left"?"":`${o} `;return`${c}${u}click${l}`}async click({coordinate:t,button:e,count:i}){let{computer:s,allowedAppIds:o,hostWindowId:n}=await this.requireAccess();this.lock();let r=n?[n]:[],c=await this._click(s,o,r,this.selectedDisplay,e??"left",i??1,t);return this.addScreenshot(this.toText(c),s,o,r)}async mouse_move({coordinate:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];return await this.prepareForInput(e,i,{x:t.x,y:t.y,blockedWindowIds:o}),await this.validatePointerInput(e,i,t),await e.move(t.x,t.y,this.selectedDisplay),this.addScreenshot(this.toText(`Moved to (${t.x},${t.y})`),e,i,o)}async left_click_drag({start_coordinate:t,coordinate:e}){let{computer:i,allowedAppIds:s,hostWindowId:o}=await this.requireAccess();this.lock();let n=o?[o]:[];return await this.prepareForInput(i,s,{x:t.x,y:t.y,blockedWindowIds:n}),await this.prepareForInput(i,s,{x:e.x,y:e.y,blockedWindowIds:n}),await this.validatePointerInput(i,s,t),await this.validatePointerInput(i,s,e),await i.drag(t.x,t.y,e.x,e.y,this.selectedDisplay),this.addScreenshot(this.toText(`Dragged (${t.x},${t.y}) -> (${e.x},${e.y})`),i,s,n)}async left_mouse_down(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[],o=await t.cursorPosition(this.selectedDisplay);return await this.prepareForInput(t,e,{x:o.x,y:o.y,blockedWindowIds:s}),await this.validatePointerInput(t,e,o),await t.mouseDown(o.x,o.y,this.selectedDisplay),this.addScreenshot(this.toText(`Mouse down at (${o.x},${o.y})`),t,e,s)}async left_mouse_up(){let{computer:t,allowedAppIds:e,hostWindowId:i}=await this.requireAccess();this.lock();let s=i?[i]:[],o=await t.cursorPosition(this.selectedDisplay);return await this.prepareForInput(t,e,{x:o.x,y:o.y,blockedWindowIds:s}),await this.validatePointerInput(t,e,o),await t.mouseUp(o.x,o.y,this.selectedDisplay),this.addScreenshot(this.toText(`Mouse up at (${o.x},${o.y})`),t,e,s)}async type({text:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];return await this.prepareForInput(e,i),await this.validateActiveWindow(e,i),await e.type(t),this.addScreenshot(this.toText(`Typed ${t.length} chars`),e,i,o)}async key({text:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];await this.prepareForInput(e,i),await this.validateActiveWindow(e,i);let n=Q(t);return await e.key(n),this.addScreenshot(this.toText(`Pressed ${n}`),e,i,o)}async scroll({scroll_direction:t,coordinate:e,scroll_amount:i}){let{computer:s,allowedAppIds:o,hostWindowId:n}=await this.requireAccess();this.lock();let r=n?[n]:[],c=i??3,a=await s.cursorPosition(this.selectedDisplay),l=e?.x??a.x,u=e?.y??a.y;await this.prepareForInput(s,o,{x:l,y:u,blockedWindowIds:r}),await this.validatePointerInput(s,o,{x:l,y:u});let f=t==="left"?-c:t==="right"?c:0,b=t==="down"?c:t==="up"?-c:0;return await s.scroll(l,u,f,b,this.selectedDisplay),this.addScreenshot(this.toText(`Scrolled ${t} ${c} at (${l},${u})`),s,o,r)}async wait({duration:t}){this.logEvent("wait",`duration=${String(t)}s`),await new Promise(r=>setTimeout(r,t*1e3));let e=this.toText(`Waited ${t}s`);if(!this.accessStore.getState().accessActive)return e;let{computer:i,allowedAppIds:s,hostWindowId:o}=await this.requireAccess(),n=o?[o]:[];return this.addScreenshot(e,i,s,n)}async get_clipboard(){let{computer:t}=await this.requireAccess();this.lock();let e=await t.getClipboard();return this.toText(e)}async set_clipboard({text:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[];return await e.setClipboard(t),this.addScreenshot(this.toText(`Clipboard set (${t.length} chars)`),e,i,o)}async zoom({region:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock();let o=s?[s]:[],n=await e.screenshot(this.selectedDisplay,i,o,0,0,[t.x1,t.y1,t.x2,t.y2],_);return this.toImage(n,"Zoom screenshot failed",_)}async batch({actions:t}){let{computer:e,allowedAppIds:i,hostWindowId:s}=await this.requireAccess();this.lock(),this.logEvent("batch",`${t.length} actions: ${t.map(c=>c.action).join(", ")}`);let o=this.selectedDisplay,n=s?[s]:[],r=[];for(let c=0;c<t.length;c++){let a=t[c];try{switch(a.action){case"click":{let l=await this._click(e,i,n,o,a.button??"left",a.count??1,a.coordinate);r.push(`[${c}] ${l}`);break}case"mouse_move":await this.prepareForInput(e,i,{x:a.coordinate.x,y:a.coordinate.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,a.coordinate),await e.move(a.coordinate.x,a.coordinate.y,o),r.push(`[${c}] moved to (${a.coordinate.x},${a.coordinate.y})`);break;case"left_click_drag":await this.prepareForInput(e,i,{x:a.start_coordinate.x,y:a.start_coordinate.y,blockedWindowIds:n}),await this.prepareForInput(e,i,{x:a.coordinate.x,y:a.coordinate.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,a.start_coordinate),await this.validatePointerInput(e,i,a.coordinate),await e.drag(a.start_coordinate.x,a.start_coordinate.y,a.coordinate.x,a.coordinate.y,o),r.push(`[${c}] dragged (${a.start_coordinate.x},${a.start_coordinate.y}) -> (${a.coordinate.x},${a.coordinate.y})`);break;case"left_mouse_down":{let l=await e.cursorPosition(o);await this.prepareForInput(e,i,{x:l.x,y:l.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,l),await e.mouseDown(l.x,l.y,o),r.push(`[${c}] mouse down at (${l.x},${l.y})`);break}case"left_mouse_up":{let l=await e.cursorPosition(o);await this.prepareForInput(e,i,{x:l.x,y:l.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,l),await e.mouseUp(l.x,l.y,o),r.push(`[${c}] mouse up at (${l.x},${l.y})`);break}case"type":await this.prepareForInput(e,i),await this.validateActiveWindow(e,i),await e.type(a.text),r.push(`[${c}] typed ${a.text.length} chars`);break;case"key":{await this.prepareForInput(e,i),await this.validateActiveWindow(e,i);let l=Q(a.text);await e.key(l),r.push(`[${c}] key ${a.text}`);break}case"scroll":{let l=a.scroll_amount??3,u=a.coordinate??await e.cursorPosition(o);await this.prepareForInput(e,i,{x:u.x,y:u.y,blockedWindowIds:n}),await this.validatePointerInput(e,i,u);let f=a.scroll_direction==="left"?-l:a.scroll_direction==="right"?l:0,b=a.scroll_direction==="down"?l:a.scroll_direction==="up"?-l:0;await e.scroll(u.x,u.y,f,b,o),r.push(`[${c}] scroll ${a.scroll_direction}${a.coordinate?` at (${u.x},${u.y})`:""}`);break}case"set_clipboard":await e.setClipboard(a.text),r.push(`[${c}] clipboard set`);break}}catch(l){return r.push(`[${c}] ${a.action}: FAILED - ${l instanceof Error?l.message:String(l)}`),this.toText(r.join(`
|
|
39
39
|
`),!0)}}return this.addScreenshot(this.toText(r.join(`
|
|
40
|
-
`)),e,i,n)}};function
|
|
41
|
-
`)});return new O(
|
|
42
|
-
`)),process.exit(s==="-h"||s==="--help"?0:1));let t=
|
|
40
|
+
`)),e,i,n)}};function vt(d={}){let t={name:"computer-use",version:"1.0.0"},e=["This MCP server provides desktop automation tools (mouse, keyboard, screenshots, clipboard).",d.yolo?"YOLO mode is enabled. Call `request_access` once to auto-allow all current and future apps for this session.":"Before using access-gated tools (screenshot, click, type, clipboard, etc.), you MUST call `request_access` to start an access session.","For a named target app, call `request_access` directly; it can find the app, select its display, and return the first screenshot. Use `list_applications` only for discovery or ambiguity.","Calling `request_access` with an empty apps array allows all visible apps on the selected display. If none are visible, it falls back to desktop access and empty desktop screenshots until apps are allowed.","Screenshots and zoom captures are composited to show ONLY the windows of allowed applications (plus system UI like the Dock). Disallowed app windows are not visible.","For speed, use returned screenshots instead of post-action `screenshot`, and batch predictable actions. Returned screenshots already include a small settle delay; do not add waits just for screenshot timing.","Prefer keyboard shortcuts, `type`, and `key` over visual menu navigation when reliable.","If `request_access` returns `allowAll=true`, all current and future apps are allowed for the rest of this session and you do not need to call `request_access` again.","Otherwise, if you launch a new app during the session, call `request_access` again to add it. Use `list_applications` first if you need to discover the app or its display."],i=new Gt(t,{instructions:e.join(`
|
|
41
|
+
`)});return new O(d).connect(i),i}async function Kt(){let d=!1;for(let s of process.argv.slice(2))s==="--yolo"?d=!0:(console.log(["Usage: computer-use-mcp [--yolo]","","Options:"," --yolo Auto-allow full desktop access when request_access is called."," -h, --help Show this help message."].join(`
|
|
42
|
+
`)),process.exit(s==="-h"||s==="--help"?0:1));let t=vt({yolo:d});async function e(){await t.close(),process.exit(0)}process.on("SIGINT",()=>{e()}),process.on("SIGTERM",()=>{e()});let i=new Xt;await t.connect(i)}Kt().catch(d=>{console.error(d),process.exit(1)});
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|