@octavus/agent 1.0.20 → 1.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-extension/assets/index.ts-ofuJnrPV.js +8 -0
- package/dist/browser-extension/service-worker-loader.js +1 -1
- package/dist/computer-mcp/linux/linux-driver.py +274 -10
- package/dist/index.js +40 -40
- package/package.json +1 -1
- package/dist/browser-extension/assets/index.ts-Bis5r5UE.js +0 -8
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
var $=Object.defineProperty;var B=(r,e,t)=>e in r?$(r,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):r[e]=t;var g=(r,e,t)=>B(r,typeof e!="symbol"?e+"":e,t);import{S as U,O as F}from"./messages-akAS58tQ.js";async function N(r){try{await chrome.tabs.setZoomSettings(r,{scope:"per-tab"})}catch{}}const x=25e3,W=2e4,O=8e3,V=5e3,j="render_wedged";class H extends Error{constructor(t){super(t);g(this,"code",j);this.name="RenderWedgedError"}}async function K(r,e,t){let a;try{return await Promise.race([r,new Promise((n,o)=>{a=setTimeout(()=>o(new Error(`${t} timed out after ${e}ms`)),e)})])}finally{a&&clearTimeout(a)}}function X(r){return new Promise(e=>{setTimeout(e,r)})}const G=33,J=50,z=45;class Z{constructor(){g(this,"attached",new Set);g(this,"queues",new Map);g(this,"pendingDialogs",new Map);g(this,"fileChooserResolvers",new Map)}async enqueue(e,t){const a=this.queues.get(e)??Promise.resolve(),n=()=>{let i;return Promise.race([t().finally(()=>clearTimeout(i)),new Promise((s,c)=>{i=setTimeout(()=>c(new Error(`CDP operation timed out after ${x}ms`)),x)})])},o=a.then(n,n);return this.queues.set(e,o.then(()=>{},()=>{})),await o}async attach(e){if(!this.attached.has(e)){await chrome.debugger.attach({tabId:e},"1.3");try{await Promise.all([chrome.debugger.sendCommand({tabId:e},"Page.enable"),chrome.debugger.sendCommand({tabId:e},"DOM.enable"),chrome.debugger.sendCommand({tabId:e},"Accessibility.enable")])}catch(t){throw await chrome.debugger.detach({tabId:e}).catch(()=>{}),t}this.attached.add(e),await N(e)}}async detach(e){if(this.attached.has(e)){try{await chrome.debugger.detach({tabId:e})}catch{}this.handleTabDetached(e)}}async captureScreenshot(e){return await this.enqueue(e,async()=>{await this.attach(e);const t=await this.readDeviceScaleFactor(e).catch(()=>1),a=[{label:"surface",budgetMs:O,run:()=>this.captureFromSurface(e)},{label:"renderer",budgetMs:O,run:()=>this.captureFromRenderer(e)},{label:"visible-tab",budgetMs:V,run:()=>this.captureFromVisibleTab(e)}];let n;for(const o of a)try{const i=await K(o.run(),o.budgetMs,`screenshot(${o.label})`);return await te(i,t)}catch(i){n=i}throw new H(`Screenshot failed after all capture paths: the browser is not producing frames (render pipeline wedged). Last error: ${n instanceof Error?n.message:String(n)}`)})}async captureFromSurface(e){return(await chrome.debugger.sendCommand({tabId:e},"Page.captureScreenshot",{format:"jpeg",quality:75})).data}async captureFromRenderer(e){return(await chrome.debugger.sendCommand({tabId:e},"Page.captureScreenshot",{format:"jpeg",quality:75,fromSurface:!1,captureBeyondViewport:!1})).data}async captureFromVisibleTab(e){const t=await chrome.tabs.get(e),[a]=await chrome.tabs.query({active:!0,windowId:t.windowId});if((a==null?void 0:a.id)!==e)throw new Error("target tab is not the active tab for captureVisibleTab");const n=await chrome.tabs.captureVisibleTab(t.windowId,{format:"jpeg",quality:75}),[,o]=n.split(",",2);if(!o)throw new Error("captureVisibleTab returned an invalid image payload");return o}async readDeviceScaleFactor(e){const[t,a]=await Promise.all([this.getDevicePixelRatio(e),chrome.tabs.getZoom(e).catch(()=>{})]);if(t===void 0||a===void 0||a<=0)return 1;const n=t/a;return Number.isFinite(n)&&n>0?n:1}async getDevicePixelRatio(e){var n;const a=(n=(await chrome.debugger.sendCommand({tabId:e},"Runtime.evaluate",{expression:"window.devicePixelRatio",returnByValue:!0})).result)==null?void 0:n.value;return typeof a=="number"&&a>0?a:void 0}async click(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:1})})}async doubleClick(e,t,a){await this.enqueue(e,async()=>{await this.attach(e);for(let n=1;n<=2;n++)await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:n}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:n})})}async rightClick(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"right",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"right",clickCount:1})})}async type(e,t){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.insertText",{text:t})})}async fill(e,t,a,n){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Runtime.evaluate",{expression:"(() => { const el = document.activeElement; if (el && typeof el.select === 'function') el.select(); })()",returnByValue:!0,awaitPromise:!0}),await chrome.debugger.sendCommand({tabId:e},"Input.insertText",{text:n})})}async keyPress(e,t){await this.enqueue(e,async()=>{await this.attach(e);const a=t.split("+").map(c=>c.trim()).filter(Boolean),n=a.pop(),o=a.map(c=>c.toLowerCase());let i=0;(o.includes("alt")||o.includes("option"))&&(i|=1),(o.includes("control")||o.includes("ctrl"))&&(i|=2),(o.includes("meta")||o.includes("command")||o.includes("cmd"))&&(i|=4),o.includes("shift")&&(i|=8);const s=le(n,i);await chrome.debugger.sendCommand({tabId:e},"Input.dispatchKeyEvent",{type:"rawKeyDown",key:s.key,code:s.code,windowsVirtualKeyCode:s.keyCode,modifiers:i}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchKeyEvent",{type:"keyUp",key:s.key,code:s.code,windowsVirtualKeyCode:s.keyCode,modifiers:i})})}async navigate(e,t){await this.enqueue(e,async()=>{await chrome.tabs.update(e,{url:t})})}async scrollPage(e,t,a){await this.enqueue(e,async()=>{await this.attach(e);const n=t==="down"?1:-1,o=c=>chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseWheel",x:100,y:300,deltaX:0,deltaY:c}),i=Math.min(Math.max(Math.ceil(Math.abs(a)/J),1),z),s=a/i;for(let c=0;c<i;c++)await o(n*s),c<i-1&&await X(G)})}async hover(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseMoved",x:t,y:a})})}async drag(e,t,a,n,o){await this.enqueue(e,async()=>{await this.attach(e);const i=(l,d,u,h,m)=>chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:l,x:d,y:u,...h!==void 0&&{button:h},...m!==void 0&&{clickCount:m}});await i("mouseMoved",t,a),await i("mousePressed",t,a,"left",1);const s=Math.round((t+n)/2),c=Math.round((a+o)/2);await i("mouseMoved",s,c),await i("mouseMoved",n,o),await i("mouseReleased",n,o,"left",1)})}async evaluateScript(e,t){return await this.enqueue(e,async()=>{await this.attach(e);const a=await chrome.debugger.sendCommand({tabId:e},"Runtime.evaluate",{expression:t,returnByValue:!0,awaitPromise:!0});return{value:a.result.value,exceptionDetails:a.exceptionDetails}})}async getAccessibilityTree(e,t,a,n){return await this.enqueue(e,async()=>{await this.attach(e);const o=await chrome.debugger.sendCommand({tabId:e},"Page.getFrameTree"),i=L(o.frameTree),s=[];let c=n;for(const l of i){if(c<=0)break;let d;try{d=await chrome.debugger.sendCommand({tabId:e},"Accessibility.getFullAXTree",{frameId:l.id})}catch{continue}const h=l.id!==o.frameTree.frame.id?`--- iframe${l.url?` (${l.url})`:""} ---`:null,m=h?h.length+1:0,f=ae(d.nodes,t,a,c-m);f&&(h&&s.push(h),s.push(f),c-=m+f.length+1)}return s.join(`
|
|
2
|
+
`)})}async resolveBackendNodeCoordinates(e,t){return await this.enqueue(e,async()=>{await this.attach(e);let a;try{a=await chrome.debugger.sendCommand({tabId:e},"DOM.getBoxModel",{backendNodeId:t})}catch{throw new Error(`Element ax_${t} no longer in the DOM. Re-run get_accessibility_tree to get fresh refs.`)}const n=a.model.border.length>0?a.model.border:a.model.content;return ie(n)})}async getFrameElementOffset(e,t){return await this.enqueue(e,async()=>{await this.attach(e);const a=await chrome.debugger.sendCommand({tabId:e},"DOM.getFrameOwner",{frameId:t}),n=await chrome.debugger.sendCommand({tabId:e},"DOM.getBoxModel",{backendNodeId:a.backendNodeId}),o=n.model.border.length>0?n.model.border:n.model.content;if(o.length<8)throw new Error("Could not resolve iframe coordinates");return{x:Math.min(o[0],o[2],o[4],o[6]),y:Math.min(o[1],o[3],o[5],o[7])}})}async getFrameTree(e){return await this.enqueue(e,async()=>(await this.attach(e),(await chrome.debugger.sendCommand({tabId:e},"Page.getFrameTree")).frameTree))}async uploadFile(e,t,a,n){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Page.setInterceptFileChooserDialog",{enabled:!0});try{const o=new Promise(s=>{const c=setTimeout(()=>{this.fileChooserResolvers.delete(e),s(null)},W);this.fileChooserResolvers.set(e,l=>{clearTimeout(c),s(l)})});await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:1});const i=await o;if(i){await chrome.debugger.sendCommand({tabId:e},"DOM.setFileInputFiles",{files:n,backendNodeId:i.backendNodeId});return}await this.setFilesOnPageInput(e,n)}finally{await chrome.debugger.sendCommand({tabId:e},"Page.setInterceptFileChooserDialog",{enabled:!1}).catch(()=>{})}})}async setFilesOnPageInput(e,t){const a=await chrome.debugger.sendCommand({tabId:e},"DOM.getDocument",{depth:-1,pierce:!0}),n=se(a.root);if(n.length===0)throw new Error('File chooser did not open and no file input was found on the page - the clicked element may not trigger a file upload. Open the upload control first (e.g. click the "Files" option in an upload menu), then call upload_file on it.');const o=n.filter(s=>!s.disabled&&!s.directory);let i=o.length===1?o[0]:void 0;if(!i&&o.length>1){const s=de(t[0]),c=s?o.filter(l=>ue(l.accept,s)):[];c.length===1&&(i=c[0])}if(!i)throw new Error(`File chooser did not open and the page has ${o.length} possible file inputs, so the upload target is ambiguous. Click the specific upload control and try again, or use the service's connector to upload or replace the file.`);await chrome.debugger.sendCommand({tabId:e},"DOM.setFileInputFiles",{files:t,backendNodeId:i.backendNodeId})}onFileChooserOpened(e,t){const a=this.fileChooserResolvers.get(e);a&&(this.fileChooserResolvers.delete(e),a({backendNodeId:t.backendNodeId}))}onDialogOpening(e,t){this.pendingDialogs.set(e,{type:t.type,message:t.message,defaultPrompt:t.defaultPrompt})}getPendingDialog(e){return this.pendingDialogs.get(e)}async handleDialog(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Page.handleJavaScriptDialog",{accept:t,...a!==void 0&&{promptText:a}}),this.pendingDialogs.delete(e)})}handleTabDetached(e){this.attached.delete(e),this.queues.delete(e),this.pendingDialogs.delete(e),this.fileChooserResolvers.delete(e)}}const Q=new Set(["button","checkbox","combobox","link","menuitem","menuitemcheckbox","menuitemradio","option","radio","searchbox","slider","spinbutton","switch","tab","textbox","treeitem"]),Y=new Set(["alertdialog","dialog","form","heading","main","navigation","region","search","tabpanel"]),ee=.01;async function te(r,e){return Math.abs(e-1)<ee?r:await re(r,1/e)??r}async function re(r,e){try{const t=await(await fetch(`data:image/jpeg;base64,${r}`)).blob(),a=await createImageBitmap(t),n=Math.max(1,Math.round(a.width*e)),o=Math.max(1,Math.round(a.height*e)),i=new OffscreenCanvas(n,o),s=i.getContext("2d");if(!s)return a.close(),null;s.drawImage(a,0,0,n,o),a.close();const c=await i.convertToBlob({type:"image/jpeg",quality:.75}),l=new Uint8Array(await c.arrayBuffer()),d=32768;let u="";for(let h=0;h<l.length;h+=d)u+=String.fromCharCode(...l.subarray(h,h+d));return btoa(u)}catch{return null}}function L(r){return[r.frame,...(r.childFrames??[]).flatMap(L)]}function ae(r,e,t,a){const n=new Map(r.map(d=>[d.nodeId,d])),o=new Set(r.flatMap(d=>d.childIds??[])),i=r.filter(d=>!o.has(d.nodeId)),s=[];let c=0;function l(d,u){if(c>=a)return;if(d.ignored===!0){for(const T of d.childIds??[]){const b=n.get(T);b&&l(b,u)}return}const h=v(d.role,!1),m=v(d.name)||v(d.description)||v(d.value),f=Q.has(h),p=Y.has(h),P=d.backendDOMNodeId!==void 0&&h!==""&&(e==="interactive"?f||p:f||p||m!=="");if(P){const T=" ".repeat(u),b=`ax_${d.backendDOMNodeId}`,A=m.replace(/"/g,'\\"');let C=`${T}${h}`;A&&(C+=` "${A}"`),C+=` [${b}]`;const _=v(d.value);_&&_!==m&&(C+=` value="${_.replace(/"/g,'\\"')}"`),ne(d,"disabled")&&(C+=" disabled"),s.push(C),c+=C.length+1}const R=P?u+1:u;if(!(R>=t))for(const T of d.childIds??[]){const b=n.get(T);b&&l(b,R)}}for(const d of i)l(d,0);return s.length>0?s.join(`
|
|
3
|
+
`):""}function v(r,e=!0){const t=r==null?void 0:r.value;if(typeof t=="string"){const a=t.replace(/\s+/g," ").trim();return e?oe(a):a}return typeof t=="number"||typeof t=="boolean"?String(t):""}function ne(r,e){var t;return((t=r.properties)==null?void 0:t.some(a=>{var n;return a.name===e&&((n=a.value)==null?void 0:n.value)===!0}))??!1}function oe(r,e=100){return r.length<=e?r:`${r.slice(0,e)}...`}function ie(r){if(r.length<8)throw new Error("Could not resolve element coordinates from backend node");const e=[r[0],r[2],r[4],r[6]],t=[r[1],r[3],r[5],r[7]];return{x:Math.round((Math.min(...e)+Math.max(...e))/2),y:Math.round((Math.min(...t)+Math.max(...t))/2)}}function se(r){var a;const e=[],t=[r];for(;t.length>0;){const n=t.pop();if(((a=n.nodeName)==null?void 0:a.toUpperCase())==="INPUT"&&n.backendNodeId!==void 0){const o=ce(n.attributes);(o.type??"").toLowerCase()==="file"&&e.push({backendNodeId:n.backendNodeId,disabled:"disabled"in o,directory:"webkitdirectory"in o||"directory"in o,accept:o.accept})}for(const o of n.children??[])t.push(o);for(const o of n.shadowRoots??[])t.push(o);n.contentDocument&&t.push(n.contentDocument)}return e}function ce(r){const e={};if(!r)return e;for(let t=0;t+1<r.length;t+=2)e[r[t].toLowerCase()]=r[t+1];return e}function de(r){if(!r)return"";const e=r.split(/[\\/]/).pop()??"",t=e.lastIndexOf(".");return t>0?e.slice(t).toLowerCase():""}function ue(r,e){if(!r)return!0;const t=r.toLowerCase().split(",").map(a=>a.trim()).filter(Boolean);return t.length===0||t.includes("*/*")||t.includes("*")?!0:t.includes(e)}function le(r,e=0){const t=(e&8)!==0,a={Enter:{key:"Enter",code:"Enter",keyCode:13},Tab:{key:"Tab",code:"Tab",keyCode:9},Escape:{key:"Escape",code:"Escape",keyCode:27},Backspace:{key:"Backspace",code:"Backspace",keyCode:8},Delete:{key:"Delete",code:"Delete",keyCode:46},ArrowUp:{key:"ArrowUp",code:"ArrowUp",keyCode:38},ArrowDown:{key:"ArrowDown",code:"ArrowDown",keyCode:40},ArrowLeft:{key:"ArrowLeft",code:"ArrowLeft",keyCode:37},ArrowRight:{key:"ArrowRight",code:"ArrowRight",keyCode:39},Home:{key:"Home",code:"Home",keyCode:36},End:{key:"End",code:"End",keyCode:35},PageUp:{key:"PageUp",code:"PageUp",keyCode:33},PageDown:{key:"PageDown",code:"PageDown",keyCode:34},Space:{key:" ",code:"Space",keyCode:32}},n={"/":{code:"Slash",keyCode:191,shifted:"?"},".":{code:"Period",keyCode:190,shifted:">"},",":{code:"Comma",keyCode:188,shifted:"<"},";":{code:"Semicolon",keyCode:186,shifted:":"},"'":{code:"Quote",keyCode:222,shifted:'"'},"[":{code:"BracketLeft",keyCode:219,shifted:"{"},"]":{code:"BracketRight",keyCode:221,shifted:"}"},"\\":{code:"Backslash",keyCode:220,shifted:"|"},"-":{code:"Minus",keyCode:189,shifted:"_"},"=":{code:"Equal",keyCode:187,shifted:"+"},"`":{code:"Backquote",keyCode:192,shifted:"~"}},o={"?":"/",">":".","<":",",":":";",'"':"'","{":"[","}":"]","|":"\\",_:"-","+":"=","~":"`"},i=r.charAt(0).toUpperCase()+r.slice(1);if(i in a)return a[i];if(r.length===1){if(/[a-zA-Z]/.test(r)){const c=r.toUpperCase();return{key:t?c:r.toLowerCase(),code:`Key${c}`,keyCode:c.charCodeAt(0)}}if(/[0-9]/.test(r))return{key:r,code:`Digit${r}`,keyCode:r.charCodeAt(0)};if(r in n){const c=n[r];return{key:t?c.shifted:r,code:c.code,keyCode:c.keyCode}}if(r in o){const c=o[r],l=n[c];return{key:r,code:l.code,keyCode:l.keyCode}}const s=r.toUpperCase();return{key:r,code:`Key${s}`,keyCode:s.charCodeAt(0)}}return{key:r,code:r,keyCode:0}}class he{constructor(){g(this,"tabId",null);g(this,"tabs",[])}async getCurrentTabId(){if(this.tabId!==null)try{return await chrome.tabs.get(this.tabId),this.tabId}catch{this.tabId=null}const[e]=await chrome.tabs.query({active:!0,lastFocusedWindow:!0});if((e==null?void 0:e.id)!==void 0)return this.tabId=e.id,this.tabs.includes(e.id)||this.tabs.push(e.id),e.id;throw new Error("No active tab found")}setCurrentTab(e){this.tabId=e,this.tabs.includes(e)||this.tabs.push(e)}removeTab(e){this.tabs=this.tabs.filter(t=>t!==e),this.tabId===e&&(this.tabId=this.tabs[this.tabs.length-1]??null)}getTrackedTabs(){return[...this.tabs]}}const me="src/offscreen/recording.html";function fe(r){return new Promise((e,t)=>{chrome.tabCapture.getMediaStreamId({targetTabId:r},a=>{const n=chrome.runtime.lastError;n||!a?t(new Error((n==null?void 0:n.message)??"Failed to acquire tab media stream")):e(a)})})}async function we(r){try{const[e]=await chrome.scripting.executeScript({target:{tabId:r},func:()=>({width:Math.round(window.innerWidth*window.devicePixelRatio),height:Math.round(window.innerHeight*window.devicePixelRatio)})}),t=e==null?void 0:e.result;if(t&&t.width>0&&t.height>0)return t}catch{}return null}class ge{constructor(e,t){g(this,"activeRecordingId",null);this.tabTracker=e,this.forward=t,chrome.runtime.onMessage.addListener(a=>{const n=a;(n==null?void 0:n.target)===U&&(this.forward(n),(n.type==="stopped"||n.type==="error")&&this.activeRecordingId===n.recordingId&&(this.activeRecordingId=null))})}async start(e){if(this.activeRecordingId)throw new Error("A recording is already in progress");const t=await this.tabTracker.getCurrentTabId(),a=await fe(t),n=await we(t);await this.ensureOffscreenDocument();const o={target:F,type:"start",recordingId:e.recordingId,streamId:a,videoBitsPerSecond:e.videoBitsPerSecond,audio:e.audio,maxDurationMs:e.maxDurationMs,captureWidth:n==null?void 0:n.width,captureHeight:n==null?void 0:n.height};return await chrome.runtime.sendMessage(o),this.activeRecordingId=e.recordingId,{recordingId:e.recordingId,started:!0}}async stop(e){const t={target:F,type:"stop",recordingId:e};return await chrome.runtime.sendMessage(t),{recordingId:e,stopping:!0}}async ensureOffscreenDocument(){if(!await chrome.offscreen.hasDocument())try{await chrome.offscreen.createDocument({url:me,reasons:[chrome.offscreen.Reason.USER_MEDIA],justification:"Record the agent browser tab to a video for demos and walkthroughs."})}catch(e){if(!await chrome.offscreen.hasDocument())throw e}}}const pe=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return{image:await e.cdp.captureScreenshot(t),format:"jpeg"}},ye=12e6,be=480*6e4,Ce=async(r,e)=>{const t=typeof r.recordingId=="string"?r.recordingId:crypto.randomUUID(),a=typeof r.videoBitsPerSecond=="number"?r.videoBitsPerSecond:ye,n=typeof r.maxDurationMs=="number"?r.maxDurationMs:be,o=r.audio===!0;return await e.recording.start({recordingId:t,videoBitsPerSecond:a,audio:o,maxDurationMs:n})},Te=async(r,e)=>{if(typeof r.recordingId!="string"||r.recordingId.length===0)throw new Error("recordingId is required to stop a recording");return await e.recording.stop(r.recordingId)},ve=3e4;function S(r,e=ve){return new Promise((t,a)=>{let n=null;const o=i=>{i.tabId===r&&i.frameId===0&&(n!==null&&clearTimeout(n),chrome.webNavigation.onCompleted.removeListener(o),t())};n=setTimeout(()=>{chrome.webNavigation.onCompleted.removeListener(o),a(new Error("Navigation timed out"))},e),chrome.webNavigation.onCompleted.addListener(o)})}const Ee=async(r,e)=>{const t=r.url;if(!t)throw new Error("navigate requires a url parameter");const a=await e.tabTracker.getCurrentTabId();return await e.cdp.navigate(a,t),await S(a),{success:!0,url:t}},ke=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return await chrome.tabs.goBack(t),{success:!0}},_e=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return await chrome.tabs.goForward(t),{success:!0}},Me=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return await chrome.tabs.reload(t),await S(t),{success:!0}};async function y(r,e,t){if(e.startsWith("ax_")){const u=Number(e.slice(3));if(!Number.isInteger(u))throw new Error(`Element ${e} is not a valid accessibility ref`);return await t.resolveBackendNodeCoordinates(r,u)}const a=await chrome.webNavigation.getAllFrames({tabId:r});if(!a||a.length===0)throw new Error(`Element ${e} not found. Re-run get_accessibility_tree to get fresh refs.`);const n=await Ie(r,e,a);if(!n)throw new Error(`Element ${e} not found or garbage collected. Re-run get_accessibility_tree to get fresh refs.`);const{x:o,y:i,frameId:s}=n;if(s===0)return{x:o,y:i};const c=await t.getFrameTree(r),d=De(a,c).get(s);if(!d)return{x:o,y:i};try{const u=await t.getFrameElementOffset(r,d);return{x:Math.round(o+u.x),y:Math.round(i+u.y)}}catch{return{x:o,y:i}}}async function Ie(r,e,t){const a=await Promise.allSettled(t.map(async n=>{const[o]=await chrome.scripting.executeScript({target:{tabId:r,frameIds:[n.frameId]},func:Se,args:[e]});return{frameId:n.frameId,point:(o==null?void 0:o.result)??null}}));for(const n of a)if(n.status==="fulfilled"&&n.value.point!==null)return{x:n.value.point.x,y:n.value.point.y,frameId:n.value.frameId};return null}function Se(r){const e=window.__octavusElementMap;if(e===void 0)return null;const t=e[r];if(t===void 0)return null;const a=t.deref();if(!a)return null;const n=a.getBoundingClientRect();return{x:Math.round(n.left+n.width/2),y:Math.round(n.top+n.height/2)}}function De(r,e){const t=new Map,a=new Map;for(const o of r){if(o.frameId===0)continue;const i=a.get(o.parentFrameId)??[];i.push(o),a.set(o.parentFrameId,i)}function n(o,i){t.set(o,i.frame.id);const s=[...i.childFrames??[]];for(const c of a.get(o)??[]){const l=s.findIndex(u=>u.frame.url===c.url);if(l===-1)continue;const[d]=s.splice(l,1);n(c.frameId,d)}}return n(0,e),t}const Pe=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("click requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.click(a,n,o),{success:!0,ref_id:t,x:n,y:o}},Re=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("double_click requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.doubleClick(a,n,o),{success:!0,ref_id:t,x:n,y:o}},Ae=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("right_click requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.rightClick(a,n,o),{success:!0,ref_id:t,x:n,y:o}},Fe=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("hover requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.hover(a,n,o),{success:!0,ref_id:t,x:n,y:o}},xe=async(r,e)=>{const t=r.from_ref_id,a=r.to_ref_id;if(!t)throw new Error("drag requires a from_ref_id parameter");if(!a)throw new Error("drag requires a to_ref_id parameter");const n=await e.tabTracker.getCurrentTabId(),o=await y(n,t,e.cdp),i=await y(n,a,e.cdp);return await e.cdp.drag(n,o.x,o.y,i.x,i.y),{success:!0,from_ref_id:t,to_ref_id:a}},Oe=async(r,e)=>{const t=r.text;if(!t)throw new Error("type requires a text parameter");const a=await e.tabTracker.getCurrentTabId();return await e.cdp.type(a,t),{success:!0,text:t}},Ne=async(r,e)=>{const t=r.key;if(!t)throw new Error("key requires a key parameter");const a=await e.tabTracker.getCurrentTabId();return await e.cdp.keyPress(a,t),{success:!0,key:t}},Le=500,qe=async(r,e)=>{const t=r.direction;if(!t||t!=="up"&&t!=="down")throw new Error('scroll requires direction "up" or "down"');const a=r.amount??Le,n=await e.tabTracker.getCurrentTabId();return await e.cdp.scrollPage(n,t,a),{success:!0,direction:t,amount:a}},$e=async(r,e)=>{const t=r.filter??"all",a=r.maxDepth??15,n=r.maxLength??5e4,o=await e.tabTracker.getCurrentTabId();if(t==="interactive"){const d=await e.cdp.getAccessibilityTree(o,t,a,n);return d?{tree:d}:{tree:"Error: Could not generate accessibility tree"}}const i=await chrome.webNavigation.getAllFrames({tabId:o});if(!i||i.length===0)return{tree:"Error: Could not enumerate frames"};const s=await Promise.allSettled(i.map(async d=>{const[u]=await chrome.scripting.executeScript({target:{tabId:o,frameIds:[d.frameId]},func:(h,m,f)=>{const p=window.__octavusGenerateAccessibilityTree;return p===void 0?"":p(h,m,f)},args:[t,a,n]});return{frame:d,tree:(u==null?void 0:u.result)??""}})),c=[];for(const d of s){if(d.status!=="fulfilled")continue;const{frame:u,tree:h}=d.value;if(h){if(u.frameId!==0){const m=u.url?` (${u.url})`:"";c.push(`--- iframe${m} ---`)}c.push(h)}}const l=c.join(`
|
|
4
|
+
`);return l?{tree:l}:{tree:"Error: Could not generate accessibility tree"}},Be=async(r,e)=>({tabs:(await chrome.tabs.query({})).filter(a=>a.id!==void 0).map(a=>({tabId:a.id,title:a.title??"",url:a.url??"",active:a.active}))}),Ue=async(r,e)=>{const t=r.url,a=await chrome.tabs.create({url:t??"about:blank",active:!0});if(a.id===void 0)throw new Error("Failed to create tab");return e.tabTracker.setCurrentTab(a.id),{success:!0,tabId:a.id,url:t??"about:blank"}},We=async(r,e)=>{const t=r.tabId;if(t===void 0)throw new Error("switch_tab requires a tabId parameter");return await chrome.tabs.update(t,{active:!0}),e.tabTracker.setCurrentTab(t),{success:!0,tabId:t}},Ve=async(r,e)=>{const t=r.tabId??await e.tabTracker.getCurrentTabId(),a=e.tabTracker.getTrackedTabs();if(a.length<=1&&a.includes(t))throw new Error("Cannot close the last tab.");return await e.cdp.detach(t),await chrome.tabs.remove(t),e.tabTracker.removeTab(t),{success:!0,closedTabId:t}},je=1e3,He=3e4,Ke=500,Xe=async(r,e)=>{const t=r.ms??je;if(r.for_navigation===!0){const a=await e.tabTracker.getCurrentTabId();return await S(a),{success:!0,waited_for:"navigation"}}return await new Promise(a=>{setTimeout(()=>a(),t)}),{success:!0,waited_ms:t}},Ge=async(r,e)=>{var d;const t=r.text;if(!Array.isArray(t)||t.length===0)throw new Error("wait_for requires a non-empty text array");const a=r.timeout??He,n=await e.tabTracker.getCurrentTabId(),i=`(() => {
|
|
5
|
+
const targets = ${JSON.stringify(t)};
|
|
6
|
+
const body = document.body?.innerText ?? '';
|
|
7
|
+
return targets.find(t => body.includes(t)) ?? null;
|
|
8
|
+
})()`;let s;const c=Date.now()+a;for(;Date.now()<c;){const{value:u,exceptionDetails:h}=await e.cdp.evaluateScript(n,i);if(h!==void 0){const m=h;s=((d=m.exception)==null?void 0:d.description)??m.text??"Script evaluation failed"}else{if(u!=null)return{success:!0,matchedText:u};s=void 0}await new Promise(m=>{setTimeout(()=>m(),Ke)})}const l=s?`Script error: ${s}`:`text not found: ${t.join(", ")}`;throw new Error(`Timed out after ${a}ms - ${l}`)},Je=async(r,e)=>{const t=r.ref_id,a=r.value;if(!t)throw new Error("select_option requires a ref_id parameter");if(!a)throw new Error("select_option requires a value parameter");const n=await e.tabTracker.getCurrentTabId(),[o]=await chrome.scripting.executeScript({target:{tabId:n},func:(s,c)=>{var m;const l=window.__octavusElementMap;if(l===void 0)return{error:"Element map not available"};const d=l[s];if(d===void 0)return{error:`Element ${s} not found`};const u=d.deref();if(!u)return{error:`Element ${s} has been garbage collected`};if(!(u instanceof HTMLSelectElement))return{error:`Element ${s} is not a <select> element`};let h=!1;for(const f of u.options)if(f.value===c||((m=f.textContent)==null?void 0:m.trim())===c){f.selected=!0,h=!0;break}return h?(u.dispatchEvent(new Event("change",{bubbles:!0})),{success:!0,selected:c}):{error:`Option "${c}" not found in ${s}`,available:Array.from(u.options).map(f=>{var p;return{value:f.value,text:(p=f.textContent)==null?void 0:p.trim()}})}},args:[t,a]}),i=o==null?void 0:o.result;if(typeof(i==null?void 0:i.error)=="string")throw new Error(i.error);return i??{success:!0}},ze=async(r,e)=>{const t=r.ref_id,a=r.value;if(!t)throw new Error("fill requires a ref_id parameter");if(a==null)throw new Error("fill requires a value parameter");const n=await e.tabTracker.getCurrentTabId(),{x:o,y:i}=await y(n,t,e.cdp);return await e.cdp.fill(n,o,i,a),{success:!0,ref_id:t}},Ze=async(r,e)=>{const t=r.action;if(t!=="accept"&&t!=="dismiss")throw new Error('handle_dialog requires action to be "accept" or "dismiss"');const a=r.promptText,n=await e.tabTracker.getCurrentTabId(),o=e.cdp.getPendingDialog(n);if(!o)throw new Error("No dialog is currently open on this tab");return await e.cdp.handleDialog(n,t==="accept",a),{success:!0,action:t,dialogType:o.type,dialogMessage:o.message}},Qe=async(r,e)=>{var s;const t=r.function;if(!t)throw new Error("evaluate_script requires a function parameter");const a=`(${t})()`,n=await e.tabTracker.getCurrentTabId(),{value:o,exceptionDetails:i}=await e.cdp.evaluateScript(n,a);if(i!==void 0){const c=i,l=((s=c.exception)==null?void 0:s.description)??c.text??"Script evaluation failed";throw new Error(l)}return{success:!0,result:o}},Ye=async(r,e)=>{const t=r.ref_id,a=r.filePath;if(!t)throw new Error("upload_file requires a ref_id parameter");if(!a)throw new Error("upload_file requires a filePath parameter");const n=await e.tabTracker.getCurrentTabId(),{x:o,y:i}=await y(n,t,e.cdp);return await e.cdp.uploadFile(n,o,i,[a]),{success:!0,ref_id:t,filePath:a}},et={screenshot:pe,start_recording:Ce,stop_recording:Te,get_accessibility_tree:$e,click:Pe,right_click:Ae,double_click:Re,hover:Fe,drag:xe,type:Oe,key:Ne,fill:ze,navigate:Ee,go_back:ke,go_forward:_e,reload:Me,scroll:qe,select_option:Je,wait:Xe,wait_for:Ge,handle_dialog:Ze,evaluate_script:Qe,upload_file:Ye,list_tabs:Be,new_tab:Ue,switch_tab:We,close_tab:Ve};async function tt(r,e,t){const a=et[r];if(a===void 0)throw new Error(`Unknown command: ${r}`);return await a(e,t)}const rt=2e4,at=1e3,nt=3e4,E=new Z,D=new he,ot=new ge(D,r=>I(r)),it={cdp:E,tabTracker:D,recording:ot};let w=null,k=null,M=0;async function st(){const r=chrome.runtime.getURL("config.json"),e=await fetch(r);if(!e.ok)throw new Error(`Failed to load config.json: ${e.status}`);return await e.json()}function q(r){w=new WebSocket(`ws://127.0.0.1:${r}`),w.onopen=()=>{M=0,w.send(JSON.stringify({type:"ready"})),k=setInterval(()=>{(w==null?void 0:w.readyState)===WebSocket.OPEN&&w.send(JSON.stringify({type:"heartbeat"}))},rt)},w.onmessage=e=>{ct(e.data)},w.onclose=()=>{dt(),ut(r)},w.onerror=()=>{}}async function ct(r){let e;try{e=JSON.parse(r)}catch{return}const{id:t,cmd:a,args:n}=e;if(!(!t||!a))try{const o=await tt(a,n??{},it);I({id:t,result:o})}catch(o){const i=o==null?void 0:o.code,s=typeof i=="string"?i:void 0;I({id:t,error:o instanceof Error?o.message:String(o),...s!==void 0?{code:s}:{}})}}function I(r){(w==null?void 0:w.readyState)===WebSocket.OPEN&&w.send(JSON.stringify(r))}function dt(){k&&(clearInterval(k),k=null)}function ut(r){const e=Math.min(at*2**M,nt);M+=1,setTimeout(()=>q(r),e)}chrome.tabs.onRemoved.addListener(r=>{E.handleTabDetached(r),D.removeTab(r)});chrome.webNavigation.onCommitted.addListener(r=>{r.frameId===0&&N(r.tabId)});chrome.debugger.onDetach.addListener(r=>{r.tabId!==void 0&&E.handleTabDetached(r.tabId)});chrome.debugger.onEvent.addListener((r,e,t)=>{if(r.tabId===void 0)return;const a=t??{};e==="Page.javascriptDialogOpening"?E.onDialogOpening(r.tabId,a):e==="Page.fileChooserOpened"&&E.onFileChooserOpened(r.tabId,a)});st().then(r=>q(r.wsPort)).catch(r=>console.error("[browser-extension] Failed to start:",r));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import './assets/index.ts-
|
|
1
|
+
import './assets/index.ts-ofuJnrPV.js';
|
|
@@ -133,8 +133,22 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
|
|
|
133
133
|
|
|
134
134
|
|
|
135
135
|
def _walk_atspi(
|
|
136
|
-
node,
|
|
136
|
+
node,
|
|
137
|
+
elements,
|
|
138
|
+
Atspi,
|
|
139
|
+
screen_w,
|
|
140
|
+
screen_h,
|
|
141
|
+
depth=0,
|
|
142
|
+
max_depth=20,
|
|
143
|
+
trust_geometry=False,
|
|
144
|
+
window_origin=None,
|
|
137
145
|
):
|
|
146
|
+
"""Collect the on-screen actionable elements under ``node``.
|
|
147
|
+
|
|
148
|
+
``window_origin`` is set for a toolkit whose extents are window-relative
|
|
149
|
+
(see ``_uses_window_relative_extents``): extents are then read in WINDOW
|
|
150
|
+
coordinates and offset by the window's on-screen client-area origin.
|
|
151
|
+
"""
|
|
138
152
|
if depth > max_depth:
|
|
139
153
|
return
|
|
140
154
|
|
|
@@ -151,8 +165,17 @@ def _walk_atspi(
|
|
|
151
165
|
try:
|
|
152
166
|
comp = node.get_component_iface()
|
|
153
167
|
if comp is not None:
|
|
154
|
-
|
|
155
|
-
|
|
168
|
+
if window_origin is None:
|
|
169
|
+
extents = comp.get_extents(Atspi.CoordType.SCREEN)
|
|
170
|
+
rect = (extents.x, extents.y, extents.width, extents.height)
|
|
171
|
+
else:
|
|
172
|
+
extents = comp.get_extents(Atspi.CoordType.WINDOW)
|
|
173
|
+
rect = (
|
|
174
|
+
extents.x + window_origin[0],
|
|
175
|
+
extents.y + window_origin[1],
|
|
176
|
+
extents.width,
|
|
177
|
+
extents.height,
|
|
178
|
+
)
|
|
156
179
|
except Exception:
|
|
157
180
|
rect = None
|
|
158
181
|
|
|
@@ -232,7 +255,15 @@ def _walk_atspi(
|
|
|
232
255
|
child = node.get_child_at_index(i)
|
|
233
256
|
if child is not None:
|
|
234
257
|
_walk_atspi(
|
|
235
|
-
child,
|
|
258
|
+
child,
|
|
259
|
+
elements,
|
|
260
|
+
Atspi,
|
|
261
|
+
screen_w,
|
|
262
|
+
screen_h,
|
|
263
|
+
depth + 1,
|
|
264
|
+
max_depth,
|
|
265
|
+
trust_geometry,
|
|
266
|
+
window_origin,
|
|
236
267
|
)
|
|
237
268
|
except Exception:
|
|
238
269
|
pass
|
|
@@ -311,10 +342,11 @@ def _get_active_window_pid():
|
|
|
311
342
|
|
|
312
343
|
|
|
313
344
|
def _get_active_window_geometry():
|
|
314
|
-
"""Screen rect of the focused window as {x, y, width, height}, or None.
|
|
345
|
+
"""Screen rect of the focused window as {id, x, y, width, height}, or None.
|
|
315
346
|
|
|
316
|
-
|
|
317
|
-
|
|
347
|
+
The focused window is the one the agent is actually looking at (one desktop,
|
|
348
|
+
what-you-see-is-what-you-get), so it is the reference for anything that must
|
|
349
|
+
be clipped to, or placed within, the visible app.
|
|
318
350
|
"""
|
|
319
351
|
try:
|
|
320
352
|
out = subprocess.check_output(
|
|
@@ -332,6 +364,7 @@ def _get_active_window_geometry():
|
|
|
332
364
|
vals[key.strip()] = value.strip()
|
|
333
365
|
try:
|
|
334
366
|
return {
|
|
367
|
+
"id": int(vals["WINDOW"]),
|
|
335
368
|
"x": int(vals["X"]),
|
|
336
369
|
"y": int(vals["Y"]),
|
|
337
370
|
"width": int(vals["WIDTH"]),
|
|
@@ -382,10 +415,181 @@ def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h, trust_geom
|
|
|
382
415
|
continue
|
|
383
416
|
except Exception:
|
|
384
417
|
continue
|
|
385
|
-
|
|
418
|
+
if _uses_window_relative_extents(app):
|
|
419
|
+
_walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry)
|
|
420
|
+
else:
|
|
421
|
+
_walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
|
|
386
422
|
return elements
|
|
387
423
|
|
|
388
424
|
|
|
425
|
+
def _uses_window_relative_extents(app):
|
|
426
|
+
"""Whether the app's AT-SPI extents are relative to its window, not the screen.
|
|
427
|
+
|
|
428
|
+
GTK 4 has no global coordinates: 4.6 answers a SCREEN extents query with
|
|
429
|
+
window-relative values ("Screen coordinates not supported") and later releases
|
|
430
|
+
report every SCREEN origin as 0,0. Taken as screen coordinates, every label
|
|
431
|
+
lands near the screen's top-left corner (GNOME's Activities button and dock)
|
|
432
|
+
instead of on its control. GTK 3 (through the ATK bridge), Chromium, and Qt
|
|
433
|
+
report true screen coordinates.
|
|
434
|
+
"""
|
|
435
|
+
try:
|
|
436
|
+
toolkit = (app.get_toolkit_name() or "").lower()
|
|
437
|
+
major = int((app.get_toolkit_version() or "0").split(".")[0])
|
|
438
|
+
except Exception:
|
|
439
|
+
return False
|
|
440
|
+
return toolkit == "gtk" and major >= 4
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry):
|
|
444
|
+
"""Walk the app's focused window, placing its window-relative extents on screen.
|
|
445
|
+
|
|
446
|
+
Only the focused window is walked: the app's other windows have their own
|
|
447
|
+
origins, which the agent cannot see past the window in front anyway.
|
|
448
|
+
"""
|
|
449
|
+
window = _get_active_window_geometry()
|
|
450
|
+
if window is None:
|
|
451
|
+
return
|
|
452
|
+
frame = _focused_frame(app, window["id"], Atspi)
|
|
453
|
+
if frame is None:
|
|
454
|
+
return
|
|
455
|
+
left, top = _client_side_decoration_offset(window["id"])
|
|
456
|
+
_walk_atspi(
|
|
457
|
+
frame,
|
|
458
|
+
elements,
|
|
459
|
+
Atspi,
|
|
460
|
+
screen_w,
|
|
461
|
+
screen_h,
|
|
462
|
+
trust_geometry=trust_geometry,
|
|
463
|
+
window_origin=(window["x"] + left, window["y"] + top),
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _focused_frame(app, window_id, Atspi):
|
|
468
|
+
"""The app's toplevel accessible for the focused window: the one AT-SPI marks
|
|
469
|
+
ACTIVE, else the one titled like the focused X11 window."""
|
|
470
|
+
frames = []
|
|
471
|
+
try:
|
|
472
|
+
for i in range(app.get_child_count()):
|
|
473
|
+
frame = app.get_child_at_index(i)
|
|
474
|
+
if frame is not None:
|
|
475
|
+
frames.append(frame)
|
|
476
|
+
except Exception:
|
|
477
|
+
return None
|
|
478
|
+
|
|
479
|
+
for frame in frames:
|
|
480
|
+
try:
|
|
481
|
+
if frame.get_state_set().contains(Atspi.StateType.ACTIVE):
|
|
482
|
+
return frame
|
|
483
|
+
except Exception:
|
|
484
|
+
continue
|
|
485
|
+
|
|
486
|
+
try:
|
|
487
|
+
title = subprocess.check_output(
|
|
488
|
+
["xdotool", "getwindowname", str(window_id)], text=True, timeout=5
|
|
489
|
+
).strip()
|
|
490
|
+
except Exception:
|
|
491
|
+
return None
|
|
492
|
+
for frame in frames:
|
|
493
|
+
try:
|
|
494
|
+
if title and frame.get_name() == title:
|
|
495
|
+
return frame
|
|
496
|
+
except Exception:
|
|
497
|
+
continue
|
|
498
|
+
return None
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
_XA_CARDINAL = 6
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _client_side_decoration_offset(window_id):
|
|
505
|
+
"""(left, top) inset of a window's visible frame within its X11 window.
|
|
506
|
+
|
|
507
|
+
GTK draws client-side decorations - its drop shadow - inside the X11 window and
|
|
508
|
+
advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
|
|
509
|
+
measures window-relative extents from the visible frame, not the X11 window's
|
|
510
|
+
corner. Read through libX11 directly because xprop is not installed on the
|
|
511
|
+
cloud image. (0, 0) for a window without client-side decorations.
|
|
512
|
+
"""
|
|
513
|
+
try:
|
|
514
|
+
import ctypes
|
|
515
|
+
import ctypes.util
|
|
516
|
+
|
|
517
|
+
xlib = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
|
|
518
|
+
except Exception:
|
|
519
|
+
return 0, 0
|
|
520
|
+
xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
|
|
521
|
+
xlib.XOpenDisplay.restype = ctypes.c_void_p
|
|
522
|
+
xlib.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
|
|
523
|
+
xlib.XInternAtom.restype = ctypes.c_ulong
|
|
524
|
+
xlib.XGetWindowProperty.argtypes = [
|
|
525
|
+
ctypes.c_void_p,
|
|
526
|
+
ctypes.c_ulong,
|
|
527
|
+
ctypes.c_ulong,
|
|
528
|
+
ctypes.c_long,
|
|
529
|
+
ctypes.c_long,
|
|
530
|
+
ctypes.c_int,
|
|
531
|
+
ctypes.c_ulong,
|
|
532
|
+
ctypes.POINTER(ctypes.c_ulong),
|
|
533
|
+
ctypes.POINTER(ctypes.c_int),
|
|
534
|
+
ctypes.POINTER(ctypes.c_ulong),
|
|
535
|
+
ctypes.POINTER(ctypes.c_ulong),
|
|
536
|
+
ctypes.POINTER(ctypes.c_void_p),
|
|
537
|
+
]
|
|
538
|
+
xlib.XFree.argtypes = [ctypes.c_void_p]
|
|
539
|
+
xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
|
|
540
|
+
xlib.XSetErrorHandler.restype = ctypes.c_void_p
|
|
541
|
+
|
|
542
|
+
# Xlib's default error handler exits the process. The focused window can be
|
|
543
|
+
# destroyed between the xdotool lookup and this read (BadWindow), which must
|
|
544
|
+
# degrade to no offset rather than take the driver down mid-observe. The
|
|
545
|
+
# callback has to stay referenced while the display is open.
|
|
546
|
+
ignore_x_errors = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
|
|
547
|
+
lambda _display, _error: 0
|
|
548
|
+
)
|
|
549
|
+
xlib.XSetErrorHandler(ignore_x_errors)
|
|
550
|
+
|
|
551
|
+
display = xlib.XOpenDisplay(None)
|
|
552
|
+
if not display:
|
|
553
|
+
return 0, 0
|
|
554
|
+
try:
|
|
555
|
+
atom = xlib.XInternAtom(display, b"_GTK_FRAME_EXTENTS", True)
|
|
556
|
+
if not atom:
|
|
557
|
+
return 0, 0
|
|
558
|
+
actual_type = ctypes.c_ulong()
|
|
559
|
+
actual_format = ctypes.c_int()
|
|
560
|
+
n_items = ctypes.c_ulong()
|
|
561
|
+
bytes_after = ctypes.c_ulong()
|
|
562
|
+
data = ctypes.c_void_p()
|
|
563
|
+
status = xlib.XGetWindowProperty(
|
|
564
|
+
display,
|
|
565
|
+
window_id,
|
|
566
|
+
atom,
|
|
567
|
+
0,
|
|
568
|
+
4,
|
|
569
|
+
False,
|
|
570
|
+
_XA_CARDINAL,
|
|
571
|
+
ctypes.byref(actual_type),
|
|
572
|
+
ctypes.byref(actual_format),
|
|
573
|
+
ctypes.byref(n_items),
|
|
574
|
+
ctypes.byref(bytes_after),
|
|
575
|
+
ctypes.byref(data),
|
|
576
|
+
)
|
|
577
|
+
if status != 0 or not data.value:
|
|
578
|
+
return 0, 0
|
|
579
|
+
try:
|
|
580
|
+
if actual_format.value != 32 or n_items.value != 4:
|
|
581
|
+
return 0, 0
|
|
582
|
+
# Xlib hands format-32 property items back as C longs.
|
|
583
|
+
left, _right, top, _bottom = ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:4]
|
|
584
|
+
return int(left), int(top)
|
|
585
|
+
finally:
|
|
586
|
+
xlib.XFree(data)
|
|
587
|
+
except Exception:
|
|
588
|
+
return 0, 0
|
|
589
|
+
finally:
|
|
590
|
+
xlib.XCloseDisplay(display)
|
|
591
|
+
|
|
592
|
+
|
|
389
593
|
def _collect_chromium(desktop, Atspi, screen_w, screen_h):
|
|
390
594
|
"""Walk only the Chrome/Chromium AT-SPI app node(s), matched by name.
|
|
391
595
|
|
|
@@ -713,6 +917,35 @@ def annotate_screenshot(screenshot_path, elements):
|
|
|
713
917
|
# slightly-less-synchronous move instead of a hard failure.
|
|
714
918
|
_MOUSEMOVE_SYNC_TIMEOUT = 3
|
|
715
919
|
|
|
920
|
+
# Every keysym a modifier can be held under, left and right variants.
|
|
921
|
+
_MODIFIER_KEYSYMS = [
|
|
922
|
+
"Control_L", "Control_R", "Alt_L", "Alt_R", "Shift_L", "Shift_R", "Super_L", "Super_R",
|
|
923
|
+
]
|
|
924
|
+
_MODIFIER_KEYUP_TIMEOUT = 3
|
|
925
|
+
|
|
926
|
+
def release_modifiers(keysyms=_MODIFIER_KEYSYMS):
|
|
927
|
+
"""Release modifier keys the X server still counts as held.
|
|
928
|
+
|
|
929
|
+
The driver runs one process per action and has no hold-key primitive, so a
|
|
930
|
+
modifier that is logically down when an action starts is always stale - most
|
|
931
|
+
often a combo whose release never landed because the ``xdotool key`` that
|
|
932
|
+
pressed it was killed by the ``_run`` timeout under display lag. Left in
|
|
933
|
+
place, a stale Ctrl turns the next wheel scroll into browser zoom and the
|
|
934
|
+
next click into a modified click. ``--clearmodifiers`` does not cure it: it
|
|
935
|
+
restores whatever it found held once its own keystroke is done. Best-effort
|
|
936
|
+
and never fatal: a failed release must not fail the action itself.
|
|
937
|
+
"""
|
|
938
|
+
try:
|
|
939
|
+
subprocess.run(
|
|
940
|
+
["xdotool", "keyup", *keysyms],
|
|
941
|
+
capture_output=True,
|
|
942
|
+
text=True,
|
|
943
|
+
timeout=_MODIFIER_KEYUP_TIMEOUT,
|
|
944
|
+
check=False,
|
|
945
|
+
)
|
|
946
|
+
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
947
|
+
pass
|
|
948
|
+
|
|
716
949
|
def mouse_move(x, y):
|
|
717
950
|
ix, iy = str(int(x)), str(int(y))
|
|
718
951
|
try:
|
|
@@ -768,6 +1001,14 @@ _MOD_MAP = {
|
|
|
768
1001
|
"ctrl": "ctrl", "control": "ctrl",
|
|
769
1002
|
}
|
|
770
1003
|
|
|
1004
|
+
# The keysyms a combo's modifier is pressed under, for the release guarantee.
|
|
1005
|
+
_MOD_KEYSYMS = {
|
|
1006
|
+
"super": ["Super_L", "Super_R"],
|
|
1007
|
+
"shift": ["Shift_L", "Shift_R"],
|
|
1008
|
+
"alt": ["Alt_L", "Alt_R"],
|
|
1009
|
+
"ctrl": ["Control_L", "Control_R"],
|
|
1010
|
+
}
|
|
1011
|
+
|
|
771
1012
|
def press_key(key_spec):
|
|
772
1013
|
parts = key_spec.lower().split("+")
|
|
773
1014
|
mods = []
|
|
@@ -782,7 +1023,18 @@ def press_key(key_spec):
|
|
|
782
1023
|
xdot_key = _KEY_MAP.get(key_name, key_name)
|
|
783
1024
|
|
|
784
1025
|
combo = "+".join(mods + [xdot_key]) if mods else xdot_key
|
|
785
|
-
|
|
1026
|
+
if not mods:
|
|
1027
|
+
_run(["xdotool", "key", "--clearmodifiers", combo])
|
|
1028
|
+
return
|
|
1029
|
+
# ``xdotool key`` presses the combo's modifiers, taps the key, then releases
|
|
1030
|
+
# them in order. Killed part-way (the ``_run`` timeout under display lag),
|
|
1031
|
+
# it leaves the modifiers logically held in the X server, and every later
|
|
1032
|
+
# wheel scroll and click arrives modified. The release runs on every exit,
|
|
1033
|
+
# including the SystemExit the timeout raises, so no combo leaves one behind.
|
|
1034
|
+
try:
|
|
1035
|
+
_run(["xdotool", "key", "--clearmodifiers", combo])
|
|
1036
|
+
finally:
|
|
1037
|
+
release_modifiers([keysym for mod in mods for keysym in _MOD_KEYSYMS[mod]])
|
|
786
1038
|
|
|
787
1039
|
def _interpolate_points(from_x, from_y, to_x, to_y, steps):
|
|
788
1040
|
"""Generate linearly interpolated points between two coordinates."""
|
|
@@ -834,8 +1086,10 @@ def do_scroll(direction, steps=3, at_x=None, at_y=None):
|
|
|
834
1086
|
time.sleep(0.05)
|
|
835
1087
|
|
|
836
1088
|
button = "5" if direction == "down" else "4"
|
|
1089
|
+
# Chrome turns a wheel event with Ctrl held into zoom, and nine notches
|
|
1090
|
+
# reach its 25% floor; a scroll must never carry a modifier, held or stale.
|
|
837
1091
|
for _ in range(steps):
|
|
838
|
-
_run(["xdotool", "click", "--repeat", "3", button])
|
|
1092
|
+
_run(["xdotool", "click", "--clearmodifiers", "--repeat", "3", button])
|
|
839
1093
|
time.sleep(0.016)
|
|
840
1094
|
|
|
841
1095
|
|
|
@@ -1011,12 +1265,22 @@ def activate_app(name):
|
|
|
1011
1265
|
# Main CLI
|
|
1012
1266
|
# ---------------------------------------------------------------------------
|
|
1013
1267
|
|
|
1268
|
+
# Commands that synthesize keyboard or mouse input. Each starts from a neutral
|
|
1269
|
+
# modifier state: the driver is one process per action with no hold-key
|
|
1270
|
+
# primitive, so any modifier still held when an action begins is stale by
|
|
1271
|
+
# construction (see ``release_modifiers``), and this is a headless machine with
|
|
1272
|
+
# no person at the keyboard whose keys could be released by mistake.
|
|
1273
|
+
_INPUT_COMMANDS = {"click", "double-click", "right-click", "type", "key", "scroll", "drag"}
|
|
1274
|
+
|
|
1275
|
+
|
|
1014
1276
|
def main():
|
|
1015
1277
|
args = sys.argv[1:]
|
|
1016
1278
|
if not args:
|
|
1017
1279
|
exit_with_error("Usage: linux-driver.py <command> [args...]")
|
|
1018
1280
|
|
|
1019
1281
|
command = args[0]
|
|
1282
|
+
if command in _INPUT_COMMANDS:
|
|
1283
|
+
release_modifiers()
|
|
1020
1284
|
|
|
1021
1285
|
if command == "observe":
|
|
1022
1286
|
pid = None
|