@octavus/agent 1.0.18 → 1.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -45,6 +45,7 @@ The working directory you launch in is the agent's workspace (override with `--w
45
45
  - `--chrome-path <path>` - path to Chrome for Testing (else resolved from `PATH`).
46
46
  - `--model <provider/model-id>` - model for this run only (else the agent's default).
47
47
  - `--backup-model <provider/model-id>` - backup model for this run only.
48
+ - `--thinking <off|low|medium|high|max|auto>` - thinking effort for this run only; `auto` lets the model decide.
48
49
  - `--capability <slug>=<on|off>` - toggle one capability for this run (repeatable).
49
50
  - `--config <file>` - JSON run config for scripted sweeps (see below).
50
51
  - `--benchmark-tools` - enable the opt-in benchmark tools (e.g. `ask_human`) for this run (see below).
@@ -67,7 +68,7 @@ Configure how the agent runs for a single invocation, without changing its dashb
67
68
 
68
69
  ```bash
69
70
  # Choose the model + backup model for this run only.
70
- octoagent run --model openrouter/moonshotai/kimi-k2 --backup-model anthropic/claude-sonnet-5 "..."
71
+ octoagent run --model openrouter/moonshotai/kimi-k2.6 --backup-model anthropic/claude-sonnet-5 "..."
71
72
 
72
73
  # Toggle capabilities for this run (repeatable). Unlisted capabilities inherit the agent default.
73
74
  octoagent run --capability memory=off --capability handbook=on "..."
@@ -79,10 +80,10 @@ octoagent run --config run.json "..."
79
80
  `run.json`:
80
81
 
81
82
  ```json
82
- { "model": "openrouter/moonshotai/kimi-k2", "backupModel": "anthropic/claude-sonnet-5", "capabilities": { "memory": false } }
83
+ { "model": "openrouter/moonshotai/kimi-k2.6", "backupModel": "anthropic/claude-sonnet-5", "capabilities": { "memory": false } }
83
84
  ```
84
85
 
85
- Precedence: explicit flags > `--config` file > the agent's dashboard defaults. Any model from the [model catalog](https://octavus.ai/pricing/models) is allowed, as long as a key resolves for its provider (your project/org BYOK or the platform default); any other model, or an unrunnable or malformed one, is rejected up front, before the run starts. Capability toggles are bounded to the capabilities the agent's protocol declares. A run with no run config behaves exactly as before.
86
+ Precedence: explicit flags > `--config` file > the agent's dashboard defaults. Any model from the [model catalog](https://octavus.ai/pricing/models) is allowed, as long as a key resolves for its provider (your project/org BYOK or the platform default), and so is any `openrouter/...` model when your project or org has its own OpenRouter key; any other model, or an unrunnable or malformed one, is rejected up front, before the run starts. Capability toggles are bounded to the capabilities the agent's protocol declares. A run with no run config behaves exactly as before.
86
87
 
87
88
  ### Machine-readable output
88
89
 
@@ -0,0 +1,8 @@
1
+ var $=Object.defineProperty;var B=(r,e,t)=>e in r?$(r,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):r[e]=t;var g=(r,e,t)=>B(r,typeof e!="symbol"?e+"":e,t);import{S as U,O as F}from"./messages-akAS58tQ.js";async function N(r){try{await chrome.tabs.setZoomSettings(r,{scope:"per-tab"})}catch{}}const x=25e3,W=2e4,O=8e3,V=5e3,j="render_wedged";class H extends Error{constructor(t){super(t);g(this,"code",j);this.name="RenderWedgedError"}}async function K(r,e,t){let a;try{return await Promise.race([r,new Promise((n,o)=>{a=setTimeout(()=>o(new Error(`${t} timed out after ${e}ms`)),e)})])}finally{a&&clearTimeout(a)}}function X(r){return new Promise(e=>{setTimeout(e,r)})}const G=33,J=50,z=45;class Z{constructor(){g(this,"attached",new Set);g(this,"queues",new Map);g(this,"pendingDialogs",new Map);g(this,"fileChooserResolvers",new Map)}async enqueue(e,t){const a=this.queues.get(e)??Promise.resolve(),n=()=>{let i;return Promise.race([t().finally(()=>clearTimeout(i)),new Promise((s,c)=>{i=setTimeout(()=>c(new Error(`CDP operation timed out after ${x}ms`)),x)})])},o=a.then(n,n);return this.queues.set(e,o.then(()=>{},()=>{})),await o}async attach(e){if(!this.attached.has(e)){await chrome.debugger.attach({tabId:e},"1.3");try{await Promise.all([chrome.debugger.sendCommand({tabId:e},"Page.enable"),chrome.debugger.sendCommand({tabId:e},"DOM.enable"),chrome.debugger.sendCommand({tabId:e},"Accessibility.enable")])}catch(t){throw await chrome.debugger.detach({tabId:e}).catch(()=>{}),t}this.attached.add(e),await N(e)}}async detach(e){if(this.attached.has(e)){try{await chrome.debugger.detach({tabId:e})}catch{}this.handleTabDetached(e)}}async captureScreenshot(e){return await this.enqueue(e,async()=>{await this.attach(e);const t=await this.readDeviceScaleFactor(e).catch(()=>1),a=[{label:"surface",budgetMs:O,run:()=>this.captureFromSurface(e)},{label:"renderer",budgetMs:O,run:()=>this.captureFromRenderer(e)},{label:"visible-tab",budgetMs:V,run:()=>this.captureFromVisibleTab(e)}];let n;for(const o of a)try{const i=await K(o.run(),o.budgetMs,`screenshot(${o.label})`);return await te(i,t)}catch(i){n=i}throw new H(`Screenshot failed after all capture paths: the browser is not producing frames (render pipeline wedged). Last error: ${n instanceof Error?n.message:String(n)}`)})}async captureFromSurface(e){return(await chrome.debugger.sendCommand({tabId:e},"Page.captureScreenshot",{format:"jpeg",quality:75})).data}async captureFromRenderer(e){return(await chrome.debugger.sendCommand({tabId:e},"Page.captureScreenshot",{format:"jpeg",quality:75,fromSurface:!1,captureBeyondViewport:!1})).data}async captureFromVisibleTab(e){const t=await chrome.tabs.get(e),[a]=await chrome.tabs.query({active:!0,windowId:t.windowId});if((a==null?void 0:a.id)!==e)throw new Error("target tab is not the active tab for captureVisibleTab");const n=await chrome.tabs.captureVisibleTab(t.windowId,{format:"jpeg",quality:75}),[,o]=n.split(",",2);if(!o)throw new Error("captureVisibleTab returned an invalid image payload");return o}async readDeviceScaleFactor(e){const[t,a]=await Promise.all([this.getDevicePixelRatio(e),chrome.tabs.getZoom(e).catch(()=>{})]);if(t===void 0||a===void 0||a<=0)return 1;const n=t/a;return Number.isFinite(n)&&n>0?n:1}async getDevicePixelRatio(e){var n;const a=(n=(await chrome.debugger.sendCommand({tabId:e},"Runtime.evaluate",{expression:"window.devicePixelRatio",returnByValue:!0})).result)==null?void 0:n.value;return typeof a=="number"&&a>0?a:void 0}async click(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:1})})}async doubleClick(e,t,a){await this.enqueue(e,async()=>{await this.attach(e);for(let n=1;n<=2;n++)await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:n}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:n})})}async rightClick(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"right",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"right",clickCount:1})})}async type(e,t){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.insertText",{text:t})})}async fill(e,t,a,n){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Runtime.evaluate",{expression:"(() => { const el = document.activeElement; if (el && typeof el.select === 'function') el.select(); })()",returnByValue:!0,awaitPromise:!0}),await chrome.debugger.sendCommand({tabId:e},"Input.insertText",{text:n})})}async keyPress(e,t){await this.enqueue(e,async()=>{await this.attach(e);const a=t.split("+").map(c=>c.trim()).filter(Boolean),n=a.pop(),o=a.map(c=>c.toLowerCase());let i=0;(o.includes("alt")||o.includes("option"))&&(i|=1),(o.includes("control")||o.includes("ctrl"))&&(i|=2),(o.includes("meta")||o.includes("command")||o.includes("cmd"))&&(i|=4),o.includes("shift")&&(i|=8);const s=le(n,i);await chrome.debugger.sendCommand({tabId:e},"Input.dispatchKeyEvent",{type:"rawKeyDown",key:s.key,code:s.code,windowsVirtualKeyCode:s.keyCode,modifiers:i}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchKeyEvent",{type:"keyUp",key:s.key,code:s.code,windowsVirtualKeyCode:s.keyCode,modifiers:i})})}async navigate(e,t){await this.enqueue(e,async()=>{await chrome.tabs.update(e,{url:t})})}async scrollPage(e,t,a){await this.enqueue(e,async()=>{await this.attach(e);const n=t==="down"?1:-1,o=c=>chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseWheel",x:100,y:300,deltaX:0,deltaY:c}),i=Math.min(Math.max(Math.ceil(Math.abs(a)/J),1),z),s=a/i;for(let c=0;c<i;c++)await o(n*s),c<i-1&&await X(G)})}async hover(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseMoved",x:t,y:a})})}async drag(e,t,a,n,o){await this.enqueue(e,async()=>{await this.attach(e);const i=(l,d,u,h,m)=>chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:l,x:d,y:u,...h!==void 0&&{button:h},...m!==void 0&&{clickCount:m}});await i("mouseMoved",t,a),await i("mousePressed",t,a,"left",1);const s=Math.round((t+n)/2),c=Math.round((a+o)/2);await i("mouseMoved",s,c),await i("mouseMoved",n,o),await i("mouseReleased",n,o,"left",1)})}async evaluateScript(e,t){return await this.enqueue(e,async()=>{await this.attach(e);const a=await chrome.debugger.sendCommand({tabId:e},"Runtime.evaluate",{expression:t,returnByValue:!0,awaitPromise:!0});return{value:a.result.value,exceptionDetails:a.exceptionDetails}})}async getAccessibilityTree(e,t,a,n){return await this.enqueue(e,async()=>{await this.attach(e);const o=await chrome.debugger.sendCommand({tabId:e},"Page.getFrameTree"),i=L(o.frameTree),s=[];let c=n;for(const l of i){if(c<=0)break;let d;try{d=await chrome.debugger.sendCommand({tabId:e},"Accessibility.getFullAXTree",{frameId:l.id})}catch{continue}const h=l.id!==o.frameTree.frame.id?`--- iframe${l.url?` (${l.url})`:""} ---`:null,m=h?h.length+1:0,f=ae(d.nodes,t,a,c-m);f&&(h&&s.push(h),s.push(f),c-=m+f.length+1)}return s.join(`
2
+ `)})}async resolveBackendNodeCoordinates(e,t){return await this.enqueue(e,async()=>{await this.attach(e);let a;try{a=await chrome.debugger.sendCommand({tabId:e},"DOM.getBoxModel",{backendNodeId:t})}catch{throw new Error(`Element ax_${t} no longer in the DOM. Re-run get_accessibility_tree to get fresh refs.`)}const n=a.model.border.length>0?a.model.border:a.model.content;return ie(n)})}async getFrameElementOffset(e,t){return await this.enqueue(e,async()=>{await this.attach(e);const a=await chrome.debugger.sendCommand({tabId:e},"DOM.getFrameOwner",{frameId:t}),n=await chrome.debugger.sendCommand({tabId:e},"DOM.getBoxModel",{backendNodeId:a.backendNodeId}),o=n.model.border.length>0?n.model.border:n.model.content;if(o.length<8)throw new Error("Could not resolve iframe coordinates");return{x:Math.min(o[0],o[2],o[4],o[6]),y:Math.min(o[1],o[3],o[5],o[7])}})}async getFrameTree(e){return await this.enqueue(e,async()=>(await this.attach(e),(await chrome.debugger.sendCommand({tabId:e},"Page.getFrameTree")).frameTree))}async uploadFile(e,t,a,n){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Page.setInterceptFileChooserDialog",{enabled:!0});try{const o=new Promise(s=>{const c=setTimeout(()=>{this.fileChooserResolvers.delete(e),s(null)},W);this.fileChooserResolvers.set(e,l=>{clearTimeout(c),s(l)})});await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mousePressed",x:t,y:a,button:"left",clickCount:1}),await chrome.debugger.sendCommand({tabId:e},"Input.dispatchMouseEvent",{type:"mouseReleased",x:t,y:a,button:"left",clickCount:1});const i=await o;if(i){await chrome.debugger.sendCommand({tabId:e},"DOM.setFileInputFiles",{files:n,backendNodeId:i.backendNodeId});return}await this.setFilesOnPageInput(e,n)}finally{await chrome.debugger.sendCommand({tabId:e},"Page.setInterceptFileChooserDialog",{enabled:!1}).catch(()=>{})}})}async setFilesOnPageInput(e,t){const a=await chrome.debugger.sendCommand({tabId:e},"DOM.getDocument",{depth:-1,pierce:!0}),n=se(a.root);if(n.length===0)throw new Error('File chooser did not open and no file input was found on the page - the clicked element may not trigger a file upload. Open the upload control first (e.g. click the "Files" option in an upload menu), then call upload_file on it.');const o=n.filter(s=>!s.disabled&&!s.directory);let i=o.length===1?o[0]:void 0;if(!i&&o.length>1){const s=de(t[0]),c=s?o.filter(l=>ue(l.accept,s)):[];c.length===1&&(i=c[0])}if(!i)throw new Error(`File chooser did not open and the page has ${o.length} possible file inputs, so the upload target is ambiguous. Click the specific upload control and try again, or use the service's connector to upload or replace the file.`);await chrome.debugger.sendCommand({tabId:e},"DOM.setFileInputFiles",{files:t,backendNodeId:i.backendNodeId})}onFileChooserOpened(e,t){const a=this.fileChooserResolvers.get(e);a&&(this.fileChooserResolvers.delete(e),a({backendNodeId:t.backendNodeId}))}onDialogOpening(e,t){this.pendingDialogs.set(e,{type:t.type,message:t.message,defaultPrompt:t.defaultPrompt})}getPendingDialog(e){return this.pendingDialogs.get(e)}async handleDialog(e,t,a){await this.enqueue(e,async()=>{await this.attach(e),await chrome.debugger.sendCommand({tabId:e},"Page.handleJavaScriptDialog",{accept:t,...a!==void 0&&{promptText:a}}),this.pendingDialogs.delete(e)})}handleTabDetached(e){this.attached.delete(e),this.queues.delete(e),this.pendingDialogs.delete(e),this.fileChooserResolvers.delete(e)}}const Q=new Set(["button","checkbox","combobox","link","menuitem","menuitemcheckbox","menuitemradio","option","radio","searchbox","slider","spinbutton","switch","tab","textbox","treeitem"]),Y=new Set(["alertdialog","dialog","form","heading","main","navigation","region","search","tabpanel"]),ee=.01;async function te(r,e){return Math.abs(e-1)<ee?r:await re(r,1/e)??r}async function re(r,e){try{const t=await(await fetch(`data:image/jpeg;base64,${r}`)).blob(),a=await createImageBitmap(t),n=Math.max(1,Math.round(a.width*e)),o=Math.max(1,Math.round(a.height*e)),i=new OffscreenCanvas(n,o),s=i.getContext("2d");if(!s)return a.close(),null;s.drawImage(a,0,0,n,o),a.close();const c=await i.convertToBlob({type:"image/jpeg",quality:.75}),l=new Uint8Array(await c.arrayBuffer()),d=32768;let u="";for(let h=0;h<l.length;h+=d)u+=String.fromCharCode(...l.subarray(h,h+d));return btoa(u)}catch{return null}}function L(r){return[r.frame,...(r.childFrames??[]).flatMap(L)]}function ae(r,e,t,a){const n=new Map(r.map(d=>[d.nodeId,d])),o=new Set(r.flatMap(d=>d.childIds??[])),i=r.filter(d=>!o.has(d.nodeId)),s=[];let c=0;function l(d,u){if(c>=a)return;if(d.ignored===!0){for(const T of d.childIds??[]){const b=n.get(T);b&&l(b,u)}return}const h=v(d.role,!1),m=v(d.name)||v(d.description)||v(d.value),f=Q.has(h),p=Y.has(h),P=d.backendDOMNodeId!==void 0&&h!==""&&(e==="interactive"?f||p:f||p||m!=="");if(P){const T=" ".repeat(u),b=`ax_${d.backendDOMNodeId}`,A=m.replace(/"/g,'\\"');let C=`${T}${h}`;A&&(C+=` "${A}"`),C+=` [${b}]`;const _=v(d.value);_&&_!==m&&(C+=` value="${_.replace(/"/g,'\\"')}"`),ne(d,"disabled")&&(C+=" disabled"),s.push(C),c+=C.length+1}const R=P?u+1:u;if(!(R>=t))for(const T of d.childIds??[]){const b=n.get(T);b&&l(b,R)}}for(const d of i)l(d,0);return s.length>0?s.join(`
3
+ `):""}function v(r,e=!0){const t=r==null?void 0:r.value;if(typeof t=="string"){const a=t.replace(/\s+/g," ").trim();return e?oe(a):a}return typeof t=="number"||typeof t=="boolean"?String(t):""}function ne(r,e){var t;return((t=r.properties)==null?void 0:t.some(a=>{var n;return a.name===e&&((n=a.value)==null?void 0:n.value)===!0}))??!1}function oe(r,e=100){return r.length<=e?r:`${r.slice(0,e)}...`}function ie(r){if(r.length<8)throw new Error("Could not resolve element coordinates from backend node");const e=[r[0],r[2],r[4],r[6]],t=[r[1],r[3],r[5],r[7]];return{x:Math.round((Math.min(...e)+Math.max(...e))/2),y:Math.round((Math.min(...t)+Math.max(...t))/2)}}function se(r){var a;const e=[],t=[r];for(;t.length>0;){const n=t.pop();if(((a=n.nodeName)==null?void 0:a.toUpperCase())==="INPUT"&&n.backendNodeId!==void 0){const o=ce(n.attributes);(o.type??"").toLowerCase()==="file"&&e.push({backendNodeId:n.backendNodeId,disabled:"disabled"in o,directory:"webkitdirectory"in o||"directory"in o,accept:o.accept})}for(const o of n.children??[])t.push(o);for(const o of n.shadowRoots??[])t.push(o);n.contentDocument&&t.push(n.contentDocument)}return e}function ce(r){const e={};if(!r)return e;for(let t=0;t+1<r.length;t+=2)e[r[t].toLowerCase()]=r[t+1];return e}function de(r){if(!r)return"";const e=r.split(/[\\/]/).pop()??"",t=e.lastIndexOf(".");return t>0?e.slice(t).toLowerCase():""}function ue(r,e){if(!r)return!0;const t=r.toLowerCase().split(",").map(a=>a.trim()).filter(Boolean);return t.length===0||t.includes("*/*")||t.includes("*")?!0:t.includes(e)}function le(r,e=0){const t=(e&8)!==0,a={Enter:{key:"Enter",code:"Enter",keyCode:13},Tab:{key:"Tab",code:"Tab",keyCode:9},Escape:{key:"Escape",code:"Escape",keyCode:27},Backspace:{key:"Backspace",code:"Backspace",keyCode:8},Delete:{key:"Delete",code:"Delete",keyCode:46},ArrowUp:{key:"ArrowUp",code:"ArrowUp",keyCode:38},ArrowDown:{key:"ArrowDown",code:"ArrowDown",keyCode:40},ArrowLeft:{key:"ArrowLeft",code:"ArrowLeft",keyCode:37},ArrowRight:{key:"ArrowRight",code:"ArrowRight",keyCode:39},Home:{key:"Home",code:"Home",keyCode:36},End:{key:"End",code:"End",keyCode:35},PageUp:{key:"PageUp",code:"PageUp",keyCode:33},PageDown:{key:"PageDown",code:"PageDown",keyCode:34},Space:{key:" ",code:"Space",keyCode:32}},n={"/":{code:"Slash",keyCode:191,shifted:"?"},".":{code:"Period",keyCode:190,shifted:">"},",":{code:"Comma",keyCode:188,shifted:"<"},";":{code:"Semicolon",keyCode:186,shifted:":"},"'":{code:"Quote",keyCode:222,shifted:'"'},"[":{code:"BracketLeft",keyCode:219,shifted:"{"},"]":{code:"BracketRight",keyCode:221,shifted:"}"},"\\":{code:"Backslash",keyCode:220,shifted:"|"},"-":{code:"Minus",keyCode:189,shifted:"_"},"=":{code:"Equal",keyCode:187,shifted:"+"},"`":{code:"Backquote",keyCode:192,shifted:"~"}},o={"?":"/",">":".","<":",",":":";",'"':"'","{":"[","}":"]","|":"\\",_:"-","+":"=","~":"`"},i=r.charAt(0).toUpperCase()+r.slice(1);if(i in a)return a[i];if(r.length===1){if(/[a-zA-Z]/.test(r)){const c=r.toUpperCase();return{key:t?c:r.toLowerCase(),code:`Key${c}`,keyCode:c.charCodeAt(0)}}if(/[0-9]/.test(r))return{key:r,code:`Digit${r}`,keyCode:r.charCodeAt(0)};if(r in n){const c=n[r];return{key:t?c.shifted:r,code:c.code,keyCode:c.keyCode}}if(r in o){const c=o[r],l=n[c];return{key:r,code:l.code,keyCode:l.keyCode}}const s=r.toUpperCase();return{key:r,code:`Key${s}`,keyCode:s.charCodeAt(0)}}return{key:r,code:r,keyCode:0}}class he{constructor(){g(this,"tabId",null);g(this,"tabs",[])}async getCurrentTabId(){if(this.tabId!==null)try{return await chrome.tabs.get(this.tabId),this.tabId}catch{this.tabId=null}const[e]=await chrome.tabs.query({active:!0,lastFocusedWindow:!0});if((e==null?void 0:e.id)!==void 0)return this.tabId=e.id,this.tabs.includes(e.id)||this.tabs.push(e.id),e.id;throw new Error("No active tab found")}setCurrentTab(e){this.tabId=e,this.tabs.includes(e)||this.tabs.push(e)}removeTab(e){this.tabs=this.tabs.filter(t=>t!==e),this.tabId===e&&(this.tabId=this.tabs[this.tabs.length-1]??null)}getTrackedTabs(){return[...this.tabs]}}const me="src/offscreen/recording.html";function fe(r){return new Promise((e,t)=>{chrome.tabCapture.getMediaStreamId({targetTabId:r},a=>{const n=chrome.runtime.lastError;n||!a?t(new Error((n==null?void 0:n.message)??"Failed to acquire tab media stream")):e(a)})})}async function we(r){try{const[e]=await chrome.scripting.executeScript({target:{tabId:r},func:()=>({width:Math.round(window.innerWidth*window.devicePixelRatio),height:Math.round(window.innerHeight*window.devicePixelRatio)})}),t=e==null?void 0:e.result;if(t&&t.width>0&&t.height>0)return t}catch{}return null}class ge{constructor(e,t){g(this,"activeRecordingId",null);this.tabTracker=e,this.forward=t,chrome.runtime.onMessage.addListener(a=>{const n=a;(n==null?void 0:n.target)===U&&(this.forward(n),(n.type==="stopped"||n.type==="error")&&this.activeRecordingId===n.recordingId&&(this.activeRecordingId=null))})}async start(e){if(this.activeRecordingId)throw new Error("A recording is already in progress");const t=await this.tabTracker.getCurrentTabId(),a=await fe(t),n=await we(t);await this.ensureOffscreenDocument();const o={target:F,type:"start",recordingId:e.recordingId,streamId:a,videoBitsPerSecond:e.videoBitsPerSecond,audio:e.audio,maxDurationMs:e.maxDurationMs,captureWidth:n==null?void 0:n.width,captureHeight:n==null?void 0:n.height};return await chrome.runtime.sendMessage(o),this.activeRecordingId=e.recordingId,{recordingId:e.recordingId,started:!0}}async stop(e){const t={target:F,type:"stop",recordingId:e};return await chrome.runtime.sendMessage(t),{recordingId:e,stopping:!0}}async ensureOffscreenDocument(){if(!await chrome.offscreen.hasDocument())try{await chrome.offscreen.createDocument({url:me,reasons:[chrome.offscreen.Reason.USER_MEDIA],justification:"Record the agent browser tab to a video for demos and walkthroughs."})}catch(e){if(!await chrome.offscreen.hasDocument())throw e}}}const pe=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return{image:await e.cdp.captureScreenshot(t),format:"jpeg"}},ye=12e6,be=480*6e4,Ce=async(r,e)=>{const t=typeof r.recordingId=="string"?r.recordingId:crypto.randomUUID(),a=typeof r.videoBitsPerSecond=="number"?r.videoBitsPerSecond:ye,n=typeof r.maxDurationMs=="number"?r.maxDurationMs:be,o=r.audio===!0;return await e.recording.start({recordingId:t,videoBitsPerSecond:a,audio:o,maxDurationMs:n})},Te=async(r,e)=>{if(typeof r.recordingId!="string"||r.recordingId.length===0)throw new Error("recordingId is required to stop a recording");return await e.recording.stop(r.recordingId)},ve=3e4;function S(r,e=ve){return new Promise((t,a)=>{let n=null;const o=i=>{i.tabId===r&&i.frameId===0&&(n!==null&&clearTimeout(n),chrome.webNavigation.onCompleted.removeListener(o),t())};n=setTimeout(()=>{chrome.webNavigation.onCompleted.removeListener(o),a(new Error("Navigation timed out"))},e),chrome.webNavigation.onCompleted.addListener(o)})}const Ee=async(r,e)=>{const t=r.url;if(!t)throw new Error("navigate requires a url parameter");const a=await e.tabTracker.getCurrentTabId();return await e.cdp.navigate(a,t),await S(a),{success:!0,url:t}},ke=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return await chrome.tabs.goBack(t),{success:!0}},_e=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return await chrome.tabs.goForward(t),{success:!0}},Me=async(r,e)=>{const t=await e.tabTracker.getCurrentTabId();return await chrome.tabs.reload(t),await S(t),{success:!0}};async function y(r,e,t){if(e.startsWith("ax_")){const u=Number(e.slice(3));if(!Number.isInteger(u))throw new Error(`Element ${e} is not a valid accessibility ref`);return await t.resolveBackendNodeCoordinates(r,u)}const a=await chrome.webNavigation.getAllFrames({tabId:r});if(!a||a.length===0)throw new Error(`Element ${e} not found. Re-run get_accessibility_tree to get fresh refs.`);const n=await Ie(r,e,a);if(!n)throw new Error(`Element ${e} not found or garbage collected. Re-run get_accessibility_tree to get fresh refs.`);const{x:o,y:i,frameId:s}=n;if(s===0)return{x:o,y:i};const c=await t.getFrameTree(r),d=De(a,c).get(s);if(!d)return{x:o,y:i};try{const u=await t.getFrameElementOffset(r,d);return{x:Math.round(o+u.x),y:Math.round(i+u.y)}}catch{return{x:o,y:i}}}async function Ie(r,e,t){const a=await Promise.allSettled(t.map(async n=>{const[o]=await chrome.scripting.executeScript({target:{tabId:r,frameIds:[n.frameId]},func:Se,args:[e]});return{frameId:n.frameId,point:(o==null?void 0:o.result)??null}}));for(const n of a)if(n.status==="fulfilled"&&n.value.point!==null)return{x:n.value.point.x,y:n.value.point.y,frameId:n.value.frameId};return null}function Se(r){const e=window.__octavusElementMap;if(e===void 0)return null;const t=e[r];if(t===void 0)return null;const a=t.deref();if(!a)return null;const n=a.getBoundingClientRect();return{x:Math.round(n.left+n.width/2),y:Math.round(n.top+n.height/2)}}function De(r,e){const t=new Map,a=new Map;for(const o of r){if(o.frameId===0)continue;const i=a.get(o.parentFrameId)??[];i.push(o),a.set(o.parentFrameId,i)}function n(o,i){t.set(o,i.frame.id);const s=[...i.childFrames??[]];for(const c of a.get(o)??[]){const l=s.findIndex(u=>u.frame.url===c.url);if(l===-1)continue;const[d]=s.splice(l,1);n(c.frameId,d)}}return n(0,e),t}const Pe=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("click requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.click(a,n,o),{success:!0,ref_id:t,x:n,y:o}},Re=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("double_click requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.doubleClick(a,n,o),{success:!0,ref_id:t,x:n,y:o}},Ae=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("right_click requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.rightClick(a,n,o),{success:!0,ref_id:t,x:n,y:o}},Fe=async(r,e)=>{const t=r.ref_id;if(!t)throw new Error("hover requires a ref_id parameter");const a=await e.tabTracker.getCurrentTabId(),{x:n,y:o}=await y(a,t,e.cdp);return await e.cdp.hover(a,n,o),{success:!0,ref_id:t,x:n,y:o}},xe=async(r,e)=>{const t=r.from_ref_id,a=r.to_ref_id;if(!t)throw new Error("drag requires a from_ref_id parameter");if(!a)throw new Error("drag requires a to_ref_id parameter");const n=await e.tabTracker.getCurrentTabId(),o=await y(n,t,e.cdp),i=await y(n,a,e.cdp);return await e.cdp.drag(n,o.x,o.y,i.x,i.y),{success:!0,from_ref_id:t,to_ref_id:a}},Oe=async(r,e)=>{const t=r.text;if(!t)throw new Error("type requires a text parameter");const a=await e.tabTracker.getCurrentTabId();return await e.cdp.type(a,t),{success:!0,text:t}},Ne=async(r,e)=>{const t=r.key;if(!t)throw new Error("key requires a key parameter");const a=await e.tabTracker.getCurrentTabId();return await e.cdp.keyPress(a,t),{success:!0,key:t}},Le=500,qe=async(r,e)=>{const t=r.direction;if(!t||t!=="up"&&t!=="down")throw new Error('scroll requires direction "up" or "down"');const a=r.amount??Le,n=await e.tabTracker.getCurrentTabId();return await e.cdp.scrollPage(n,t,a),{success:!0,direction:t,amount:a}},$e=async(r,e)=>{const t=r.filter??"all",a=r.maxDepth??15,n=r.maxLength??5e4,o=await e.tabTracker.getCurrentTabId();if(t==="interactive"){const d=await e.cdp.getAccessibilityTree(o,t,a,n);return d?{tree:d}:{tree:"Error: Could not generate accessibility tree"}}const i=await chrome.webNavigation.getAllFrames({tabId:o});if(!i||i.length===0)return{tree:"Error: Could not enumerate frames"};const s=await Promise.allSettled(i.map(async d=>{const[u]=await chrome.scripting.executeScript({target:{tabId:o,frameIds:[d.frameId]},func:(h,m,f)=>{const p=window.__octavusGenerateAccessibilityTree;return p===void 0?"":p(h,m,f)},args:[t,a,n]});return{frame:d,tree:(u==null?void 0:u.result)??""}})),c=[];for(const d of s){if(d.status!=="fulfilled")continue;const{frame:u,tree:h}=d.value;if(h){if(u.frameId!==0){const m=u.url?` (${u.url})`:"";c.push(`--- iframe${m} ---`)}c.push(h)}}const l=c.join(`
4
+ `);return l?{tree:l}:{tree:"Error: Could not generate accessibility tree"}},Be=async(r,e)=>({tabs:(await chrome.tabs.query({})).filter(a=>a.id!==void 0).map(a=>({tabId:a.id,title:a.title??"",url:a.url??"",active:a.active}))}),Ue=async(r,e)=>{const t=r.url,a=await chrome.tabs.create({url:t??"about:blank",active:!0});if(a.id===void 0)throw new Error("Failed to create tab");return e.tabTracker.setCurrentTab(a.id),{success:!0,tabId:a.id,url:t??"about:blank"}},We=async(r,e)=>{const t=r.tabId;if(t===void 0)throw new Error("switch_tab requires a tabId parameter");return await chrome.tabs.update(t,{active:!0}),e.tabTracker.setCurrentTab(t),{success:!0,tabId:t}},Ve=async(r,e)=>{const t=r.tabId??await e.tabTracker.getCurrentTabId(),a=e.tabTracker.getTrackedTabs();if(a.length<=1&&a.includes(t))throw new Error("Cannot close the last tab.");return await e.cdp.detach(t),await chrome.tabs.remove(t),e.tabTracker.removeTab(t),{success:!0,closedTabId:t}},je=1e3,He=3e4,Ke=500,Xe=async(r,e)=>{const t=r.ms??je;if(r.for_navigation===!0){const a=await e.tabTracker.getCurrentTabId();return await S(a),{success:!0,waited_for:"navigation"}}return await new Promise(a=>{setTimeout(()=>a(),t)}),{success:!0,waited_ms:t}},Ge=async(r,e)=>{var d;const t=r.text;if(!Array.isArray(t)||t.length===0)throw new Error("wait_for requires a non-empty text array");const a=r.timeout??He,n=await e.tabTracker.getCurrentTabId(),i=`(() => {
5
+ const targets = ${JSON.stringify(t)};
6
+ const body = document.body?.innerText ?? '';
7
+ return targets.find(t => body.includes(t)) ?? null;
8
+ })()`;let s;const c=Date.now()+a;for(;Date.now()<c;){const{value:u,exceptionDetails:h}=await e.cdp.evaluateScript(n,i);if(h!==void 0){const m=h;s=((d=m.exception)==null?void 0:d.description)??m.text??"Script evaluation failed"}else{if(u!=null)return{success:!0,matchedText:u};s=void 0}await new Promise(m=>{setTimeout(()=>m(),Ke)})}const l=s?`Script error: ${s}`:`text not found: ${t.join(", ")}`;throw new Error(`Timed out after ${a}ms - ${l}`)},Je=async(r,e)=>{const t=r.ref_id,a=r.value;if(!t)throw new Error("select_option requires a ref_id parameter");if(!a)throw new Error("select_option requires a value parameter");const n=await e.tabTracker.getCurrentTabId(),[o]=await chrome.scripting.executeScript({target:{tabId:n},func:(s,c)=>{var m;const l=window.__octavusElementMap;if(l===void 0)return{error:"Element map not available"};const d=l[s];if(d===void 0)return{error:`Element ${s} not found`};const u=d.deref();if(!u)return{error:`Element ${s} has been garbage collected`};if(!(u instanceof HTMLSelectElement))return{error:`Element ${s} is not a <select> element`};let h=!1;for(const f of u.options)if(f.value===c||((m=f.textContent)==null?void 0:m.trim())===c){f.selected=!0,h=!0;break}return h?(u.dispatchEvent(new Event("change",{bubbles:!0})),{success:!0,selected:c}):{error:`Option "${c}" not found in ${s}`,available:Array.from(u.options).map(f=>{var p;return{value:f.value,text:(p=f.textContent)==null?void 0:p.trim()}})}},args:[t,a]}),i=o==null?void 0:o.result;if(typeof(i==null?void 0:i.error)=="string")throw new Error(i.error);return i??{success:!0}},ze=async(r,e)=>{const t=r.ref_id,a=r.value;if(!t)throw new Error("fill requires a ref_id parameter");if(a==null)throw new Error("fill requires a value parameter");const n=await e.tabTracker.getCurrentTabId(),{x:o,y:i}=await y(n,t,e.cdp);return await e.cdp.fill(n,o,i,a),{success:!0,ref_id:t}},Ze=async(r,e)=>{const t=r.action;if(t!=="accept"&&t!=="dismiss")throw new Error('handle_dialog requires action to be "accept" or "dismiss"');const a=r.promptText,n=await e.tabTracker.getCurrentTabId(),o=e.cdp.getPendingDialog(n);if(!o)throw new Error("No dialog is currently open on this tab");return await e.cdp.handleDialog(n,t==="accept",a),{success:!0,action:t,dialogType:o.type,dialogMessage:o.message}},Qe=async(r,e)=>{var s;const t=r.function;if(!t)throw new Error("evaluate_script requires a function parameter");const a=`(${t})()`,n=await e.tabTracker.getCurrentTabId(),{value:o,exceptionDetails:i}=await e.cdp.evaluateScript(n,a);if(i!==void 0){const c=i,l=((s=c.exception)==null?void 0:s.description)??c.text??"Script evaluation failed";throw new Error(l)}return{success:!0,result:o}},Ye=async(r,e)=>{const t=r.ref_id,a=r.filePath;if(!t)throw new Error("upload_file requires a ref_id parameter");if(!a)throw new Error("upload_file requires a filePath parameter");const n=await e.tabTracker.getCurrentTabId(),{x:o,y:i}=await y(n,t,e.cdp);return await e.cdp.uploadFile(n,o,i,[a]),{success:!0,ref_id:t,filePath:a}},et={screenshot:pe,start_recording:Ce,stop_recording:Te,get_accessibility_tree:$e,click:Pe,right_click:Ae,double_click:Re,hover:Fe,drag:xe,type:Oe,key:Ne,fill:ze,navigate:Ee,go_back:ke,go_forward:_e,reload:Me,scroll:qe,select_option:Je,wait:Xe,wait_for:Ge,handle_dialog:Ze,evaluate_script:Qe,upload_file:Ye,list_tabs:Be,new_tab:Ue,switch_tab:We,close_tab:Ve};async function tt(r,e,t){const a=et[r];if(a===void 0)throw new Error(`Unknown command: ${r}`);return await a(e,t)}const rt=2e4,at=1e3,nt=3e4,E=new Z,D=new he,ot=new ge(D,r=>I(r)),it={cdp:E,tabTracker:D,recording:ot};let w=null,k=null,M=0;async function st(){const r=chrome.runtime.getURL("config.json"),e=await fetch(r);if(!e.ok)throw new Error(`Failed to load config.json: ${e.status}`);return await e.json()}function q(r){w=new WebSocket(`ws://127.0.0.1:${r}`),w.onopen=()=>{M=0,w.send(JSON.stringify({type:"ready"})),k=setInterval(()=>{(w==null?void 0:w.readyState)===WebSocket.OPEN&&w.send(JSON.stringify({type:"heartbeat"}))},rt)},w.onmessage=e=>{ct(e.data)},w.onclose=()=>{dt(),ut(r)},w.onerror=()=>{}}async function ct(r){let e;try{e=JSON.parse(r)}catch{return}const{id:t,cmd:a,args:n}=e;if(!(!t||!a))try{const o=await tt(a,n??{},it);I({id:t,result:o})}catch(o){const i=o==null?void 0:o.code,s=typeof i=="string"?i:void 0;I({id:t,error:o instanceof Error?o.message:String(o),...s!==void 0?{code:s}:{}})}}function I(r){(w==null?void 0:w.readyState)===WebSocket.OPEN&&w.send(JSON.stringify(r))}function dt(){k&&(clearInterval(k),k=null)}function ut(r){const e=Math.min(at*2**M,nt);M+=1,setTimeout(()=>q(r),e)}chrome.tabs.onRemoved.addListener(r=>{E.handleTabDetached(r),D.removeTab(r)});chrome.webNavigation.onCommitted.addListener(r=>{r.frameId===0&&N(r.tabId)});chrome.debugger.onDetach.addListener(r=>{r.tabId!==void 0&&E.handleTabDetached(r.tabId)});chrome.debugger.onEvent.addListener((r,e,t)=>{if(r.tabId===void 0)return;const a=t??{};e==="Page.javascriptDialogOpening"?E.onDialogOpening(r.tabId,a):e==="Page.fileChooserOpened"&&E.onFileChooserOpened(r.tabId,a)});st().then(r=>q(r.wsPort)).catch(r=>console.error("[browser-extension] Failed to start:",r));
@@ -1 +1 @@
1
- import './assets/index.ts-Bis5r5UE.js';
1
+ import './assets/index.ts-ofuJnrPV.js';
@@ -133,8 +133,22 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
133
133
 
134
134
 
135
135
  def _walk_atspi(
136
- node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20, trust_geometry=False
136
+ node,
137
+ elements,
138
+ Atspi,
139
+ screen_w,
140
+ screen_h,
141
+ depth=0,
142
+ max_depth=20,
143
+ trust_geometry=False,
144
+ window_origin=None,
137
145
  ):
146
+ """Collect the on-screen actionable elements under ``node``.
147
+
148
+ ``window_origin`` is set for a toolkit whose extents are window-relative
149
+ (see ``_uses_window_relative_extents``): extents are then read in WINDOW
150
+ coordinates and offset by the window's on-screen client-area origin.
151
+ """
138
152
  if depth > max_depth:
139
153
  return
140
154
 
@@ -151,8 +165,17 @@ def _walk_atspi(
151
165
  try:
152
166
  comp = node.get_component_iface()
153
167
  if comp is not None:
154
- extents = comp.get_extents(Atspi.CoordType.SCREEN)
155
- rect = (extents.x, extents.y, extents.width, extents.height)
168
+ if window_origin is None:
169
+ extents = comp.get_extents(Atspi.CoordType.SCREEN)
170
+ rect = (extents.x, extents.y, extents.width, extents.height)
171
+ else:
172
+ extents = comp.get_extents(Atspi.CoordType.WINDOW)
173
+ rect = (
174
+ extents.x + window_origin[0],
175
+ extents.y + window_origin[1],
176
+ extents.width,
177
+ extents.height,
178
+ )
156
179
  except Exception:
157
180
  rect = None
158
181
 
@@ -232,7 +255,15 @@ def _walk_atspi(
232
255
  child = node.get_child_at_index(i)
233
256
  if child is not None:
234
257
  _walk_atspi(
235
- child, elements, Atspi, screen_w, screen_h, depth + 1, max_depth, trust_geometry
258
+ child,
259
+ elements,
260
+ Atspi,
261
+ screen_w,
262
+ screen_h,
263
+ depth + 1,
264
+ max_depth,
265
+ trust_geometry,
266
+ window_origin,
236
267
  )
237
268
  except Exception:
238
269
  pass
@@ -311,10 +342,11 @@ def _get_active_window_pid():
311
342
 
312
343
 
313
344
  def _get_active_window_geometry():
314
- """Screen rect of the focused window as {x, y, width, height}, or None.
345
+ """Screen rect of the focused window as {id, x, y, width, height}, or None.
315
346
 
316
- Used to clip a recovered element set to the window the agent is actually
317
- looking at (one desktop, what-you-see-is-what-you-get).
347
+ The focused window is the one the agent is actually looking at (one desktop,
348
+ what-you-see-is-what-you-get), so it is the reference for anything that must
349
+ be clipped to, or placed within, the visible app.
318
350
  """
319
351
  try:
320
352
  out = subprocess.check_output(
@@ -332,6 +364,7 @@ def _get_active_window_geometry():
332
364
  vals[key.strip()] = value.strip()
333
365
  try:
334
366
  return {
367
+ "id": int(vals["WINDOW"]),
335
368
  "x": int(vals["X"]),
336
369
  "y": int(vals["Y"]),
337
370
  "width": int(vals["WIDTH"]),
@@ -382,10 +415,181 @@ def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h, trust_geom
382
415
  continue
383
416
  except Exception:
384
417
  continue
385
- _walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
418
+ if _uses_window_relative_extents(app):
419
+ _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry)
420
+ else:
421
+ _walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
386
422
  return elements
387
423
 
388
424
 
425
+ def _uses_window_relative_extents(app):
426
+ """Whether the app's AT-SPI extents are relative to its window, not the screen.
427
+
428
+ GTK 4 has no global coordinates: 4.6 answers a SCREEN extents query with
429
+ window-relative values ("Screen coordinates not supported") and later releases
430
+ report every SCREEN origin as 0,0. Taken as screen coordinates, every label
431
+ lands near the screen's top-left corner (GNOME's Activities button and dock)
432
+ instead of on its control. GTK 3 (through the ATK bridge), Chromium, and Qt
433
+ report true screen coordinates.
434
+ """
435
+ try:
436
+ toolkit = (app.get_toolkit_name() or "").lower()
437
+ major = int((app.get_toolkit_version() or "0").split(".")[0])
438
+ except Exception:
439
+ return False
440
+ return toolkit == "gtk" and major >= 4
441
+
442
+
443
+ def _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry):
444
+ """Walk the app's focused window, placing its window-relative extents on screen.
445
+
446
+ Only the focused window is walked: the app's other windows have their own
447
+ origins, which the agent cannot see past the window in front anyway.
448
+ """
449
+ window = _get_active_window_geometry()
450
+ if window is None:
451
+ return
452
+ frame = _focused_frame(app, window["id"], Atspi)
453
+ if frame is None:
454
+ return
455
+ left, top = _client_side_decoration_offset(window["id"])
456
+ _walk_atspi(
457
+ frame,
458
+ elements,
459
+ Atspi,
460
+ screen_w,
461
+ screen_h,
462
+ trust_geometry=trust_geometry,
463
+ window_origin=(window["x"] + left, window["y"] + top),
464
+ )
465
+
466
+
467
+ def _focused_frame(app, window_id, Atspi):
468
+ """The app's toplevel accessible for the focused window: the one AT-SPI marks
469
+ ACTIVE, else the one titled like the focused X11 window."""
470
+ frames = []
471
+ try:
472
+ for i in range(app.get_child_count()):
473
+ frame = app.get_child_at_index(i)
474
+ if frame is not None:
475
+ frames.append(frame)
476
+ except Exception:
477
+ return None
478
+
479
+ for frame in frames:
480
+ try:
481
+ if frame.get_state_set().contains(Atspi.StateType.ACTIVE):
482
+ return frame
483
+ except Exception:
484
+ continue
485
+
486
+ try:
487
+ title = subprocess.check_output(
488
+ ["xdotool", "getwindowname", str(window_id)], text=True, timeout=5
489
+ ).strip()
490
+ except Exception:
491
+ return None
492
+ for frame in frames:
493
+ try:
494
+ if title and frame.get_name() == title:
495
+ return frame
496
+ except Exception:
497
+ continue
498
+ return None
499
+
500
+
501
+ _XA_CARDINAL = 6
502
+
503
+
504
+ def _client_side_decoration_offset(window_id):
505
+ """(left, top) inset of a window's visible frame within its X11 window.
506
+
507
+ GTK draws client-side decorations - its drop shadow - inside the X11 window and
508
+ advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
509
+ measures window-relative extents from the visible frame, not the X11 window's
510
+ corner. Read through libX11 directly because xprop is not installed on the
511
+ cloud image. (0, 0) for a window without client-side decorations.
512
+ """
513
+ try:
514
+ import ctypes
515
+ import ctypes.util
516
+
517
+ xlib = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
518
+ except Exception:
519
+ return 0, 0
520
+ xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
521
+ xlib.XOpenDisplay.restype = ctypes.c_void_p
522
+ xlib.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
523
+ xlib.XInternAtom.restype = ctypes.c_ulong
524
+ xlib.XGetWindowProperty.argtypes = [
525
+ ctypes.c_void_p,
526
+ ctypes.c_ulong,
527
+ ctypes.c_ulong,
528
+ ctypes.c_long,
529
+ ctypes.c_long,
530
+ ctypes.c_int,
531
+ ctypes.c_ulong,
532
+ ctypes.POINTER(ctypes.c_ulong),
533
+ ctypes.POINTER(ctypes.c_int),
534
+ ctypes.POINTER(ctypes.c_ulong),
535
+ ctypes.POINTER(ctypes.c_ulong),
536
+ ctypes.POINTER(ctypes.c_void_p),
537
+ ]
538
+ xlib.XFree.argtypes = [ctypes.c_void_p]
539
+ xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
540
+ xlib.XSetErrorHandler.restype = ctypes.c_void_p
541
+
542
+ # Xlib's default error handler exits the process. The focused window can be
543
+ # destroyed between the xdotool lookup and this read (BadWindow), which must
544
+ # degrade to no offset rather than take the driver down mid-observe. The
545
+ # callback has to stay referenced while the display is open.
546
+ ignore_x_errors = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
547
+ lambda _display, _error: 0
548
+ )
549
+ xlib.XSetErrorHandler(ignore_x_errors)
550
+
551
+ display = xlib.XOpenDisplay(None)
552
+ if not display:
553
+ return 0, 0
554
+ try:
555
+ atom = xlib.XInternAtom(display, b"_GTK_FRAME_EXTENTS", True)
556
+ if not atom:
557
+ return 0, 0
558
+ actual_type = ctypes.c_ulong()
559
+ actual_format = ctypes.c_int()
560
+ n_items = ctypes.c_ulong()
561
+ bytes_after = ctypes.c_ulong()
562
+ data = ctypes.c_void_p()
563
+ status = xlib.XGetWindowProperty(
564
+ display,
565
+ window_id,
566
+ atom,
567
+ 0,
568
+ 4,
569
+ False,
570
+ _XA_CARDINAL,
571
+ ctypes.byref(actual_type),
572
+ ctypes.byref(actual_format),
573
+ ctypes.byref(n_items),
574
+ ctypes.byref(bytes_after),
575
+ ctypes.byref(data),
576
+ )
577
+ if status != 0 or not data.value:
578
+ return 0, 0
579
+ try:
580
+ if actual_format.value != 32 or n_items.value != 4:
581
+ return 0, 0
582
+ # Xlib hands format-32 property items back as C longs.
583
+ left, _right, top, _bottom = ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:4]
584
+ return int(left), int(top)
585
+ finally:
586
+ xlib.XFree(data)
587
+ except Exception:
588
+ return 0, 0
589
+ finally:
590
+ xlib.XCloseDisplay(display)
591
+
592
+
389
593
  def _collect_chromium(desktop, Atspi, screen_w, screen_h):
390
594
  """Walk only the Chrome/Chromium AT-SPI app node(s), matched by name.
391
595
 
@@ -713,6 +917,35 @@ def annotate_screenshot(screenshot_path, elements):
713
917
  # slightly-less-synchronous move instead of a hard failure.
714
918
  _MOUSEMOVE_SYNC_TIMEOUT = 3
715
919
 
920
+ # Every keysym a modifier can be held under, left and right variants.
921
+ _MODIFIER_KEYSYMS = [
922
+ "Control_L", "Control_R", "Alt_L", "Alt_R", "Shift_L", "Shift_R", "Super_L", "Super_R",
923
+ ]
924
+ _MODIFIER_KEYUP_TIMEOUT = 3
925
+
926
+ def release_modifiers(keysyms=_MODIFIER_KEYSYMS):
927
+ """Release modifier keys the X server still counts as held.
928
+
929
+ The driver runs one process per action and has no hold-key primitive, so a
930
+ modifier that is logically down when an action starts is always stale - most
931
+ often a combo whose release never landed because the ``xdotool key`` that
932
+ pressed it was killed by the ``_run`` timeout under display lag. Left in
933
+ place, a stale Ctrl turns the next wheel scroll into browser zoom and the
934
+ next click into a modified click. ``--clearmodifiers`` does not cure it: it
935
+ restores whatever it found held once its own keystroke is done. Best-effort
936
+ and never fatal: a failed release must not fail the action itself.
937
+ """
938
+ try:
939
+ subprocess.run(
940
+ ["xdotool", "keyup", *keysyms],
941
+ capture_output=True,
942
+ text=True,
943
+ timeout=_MODIFIER_KEYUP_TIMEOUT,
944
+ check=False,
945
+ )
946
+ except (FileNotFoundError, subprocess.TimeoutExpired):
947
+ pass
948
+
716
949
  def mouse_move(x, y):
717
950
  ix, iy = str(int(x)), str(int(y))
718
951
  try:
@@ -768,6 +1001,14 @@ _MOD_MAP = {
768
1001
  "ctrl": "ctrl", "control": "ctrl",
769
1002
  }
770
1003
 
1004
+ # The keysyms a combo's modifier is pressed under, for the release guarantee.
1005
+ _MOD_KEYSYMS = {
1006
+ "super": ["Super_L", "Super_R"],
1007
+ "shift": ["Shift_L", "Shift_R"],
1008
+ "alt": ["Alt_L", "Alt_R"],
1009
+ "ctrl": ["Control_L", "Control_R"],
1010
+ }
1011
+
771
1012
  def press_key(key_spec):
772
1013
  parts = key_spec.lower().split("+")
773
1014
  mods = []
@@ -782,7 +1023,18 @@ def press_key(key_spec):
782
1023
  xdot_key = _KEY_MAP.get(key_name, key_name)
783
1024
 
784
1025
  combo = "+".join(mods + [xdot_key]) if mods else xdot_key
785
- _run(["xdotool", "key", "--clearmodifiers", combo])
1026
+ if not mods:
1027
+ _run(["xdotool", "key", "--clearmodifiers", combo])
1028
+ return
1029
+ # ``xdotool key`` presses the combo's modifiers, taps the key, then releases
1030
+ # them in order. Killed part-way (the ``_run`` timeout under display lag),
1031
+ # it leaves the modifiers logically held in the X server, and every later
1032
+ # wheel scroll and click arrives modified. The release runs on every exit,
1033
+ # including the SystemExit the timeout raises, so no combo leaves one behind.
1034
+ try:
1035
+ _run(["xdotool", "key", "--clearmodifiers", combo])
1036
+ finally:
1037
+ release_modifiers([keysym for mod in mods for keysym in _MOD_KEYSYMS[mod]])
786
1038
 
787
1039
  def _interpolate_points(from_x, from_y, to_x, to_y, steps):
788
1040
  """Generate linearly interpolated points between two coordinates."""
@@ -834,8 +1086,10 @@ def do_scroll(direction, steps=3, at_x=None, at_y=None):
834
1086
  time.sleep(0.05)
835
1087
 
836
1088
  button = "5" if direction == "down" else "4"
1089
+ # Chrome turns a wheel event with Ctrl held into zoom, and nine notches
1090
+ # reach its 25% floor; a scroll must never carry a modifier, held or stale.
837
1091
  for _ in range(steps):
838
- _run(["xdotool", "click", "--repeat", "3", button])
1092
+ _run(["xdotool", "click", "--clearmodifiers", "--repeat", "3", button])
839
1093
  time.sleep(0.016)
840
1094
 
841
1095
 
@@ -1011,12 +1265,22 @@ def activate_app(name):
1011
1265
  # Main CLI
1012
1266
  # ---------------------------------------------------------------------------
1013
1267
 
1268
+ # Commands that synthesize keyboard or mouse input. Each starts from a neutral
1269
+ # modifier state: the driver is one process per action with no hold-key
1270
+ # primitive, so any modifier still held when an action begins is stale by
1271
+ # construction (see ``release_modifiers``), and this is a headless machine with
1272
+ # no person at the keyboard whose keys could be released by mistake.
1273
+ _INPUT_COMMANDS = {"click", "double-click", "right-click", "type", "key", "scroll", "drag"}
1274
+
1275
+
1014
1276
  def main():
1015
1277
  args = sys.argv[1:]
1016
1278
  if not args:
1017
1279
  exit_with_error("Usage: linux-driver.py <command> [args...]")
1018
1280
 
1019
1281
  command = args[0]
1282
+ if command in _INPUT_COMMANDS:
1283
+ release_modifiers()
1020
1284
 
1021
1285
  if command == "observe":
1022
1286
  pid = None