agent-device 0.16.7 → 0.16.8

package/dist/src/snapshot.js

@@ -1,2 +1,2 @@
-import{__webpack_require__ as e}from"./rslib-runtime.js";import{promises as t}from"node:fs";import n from"node:os";import a from"node:path";import{centerOfRect as s}from"./4057.js";import{normalizeError as o,AppError as r}from"./9152.js";import{withDiagnosticTimer as i,emitDiagnostic as l}from"./7599.js";import{pressAndroid as d,backAndroid as p}from"./input-actions.js";import{createDaemonRuntimeSessionStore as u,captureSnapshot as c,dispatchCommand as f,resolveSnapshotScope as m,runMacOsAlertAction as h,errorResponse as w,runIosRunnerCommand as _,isCommandSupportedOnDevice as g,snapshotAndroid as y,annotateScreenshotWithRefs as v,context_contextFromFlags as S}from"./2415.js";import{successText as b}from"./1998.js";import{sleep as I}from"./4829.js";import{buildSnapshotSession as A,dispatchWaitViaRuntime as C,createDaemonRuntimePolicy as R,recordIfSession as N,resolveSessionDevice as M,withSessionlessRunnerCleanup as P}from"./selector-runtime.js";import{parseTimeout as D}from"./6085.js";import{SETTINGS_INVALID_ARGS_MESSAGE as k,isMacOsSettingSupported as E,getUnsupportedMacOsSettingMessage as K}from"./1352.js";import{createAgentDevice as O}from"./9533.js";import{DAEMON_COMMAND_GROUPS as U}from"./5792.js";var x={};e.r(x),e.d(x,{SNAPSHOT_COMMAND_HANDLERS:()=>eh,handleSnapshotCommands:()=>ew});let H=new Set(["com.android.permissioncontroller","com.google.android.permissioncontroller","com.google.android.packageinstaller","com.android.packageinstaller"]),L=new Set(["android","com.android.systemui"]),$=/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i,T=/^android:id\/button[123]$/i,B=/(?:^|:)id\/permission_/i,q=/\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i,F=/^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i,j=/^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i;function z(e,t){let n=t[0];if(!n)return;let a=W(e,n.index);for(let n of t.slice(1)){let t=new Set(W(e,n.index));for(let e=a.length-1;e>=0;e-=1)t.has(a[e])||a.splice(e,1)}return a[a.length-1]}function W(e,t){let n=new Map(e.map(e=>[e.index,e])),a=[],s=n.get(t);for(;s;)a.push(s.index),s=void 0===s.parentIndex?void 0:n.get(s.parentIndex);return a.reverse()}function G(e,t){let n=new Map;for(let t of e){if(void 0===t.parentIndex)continue;let e=n.get(t.parentIndex)??[];e.push(t),n.set(t.parentIndex,e)}let a=new Set([t]),s=[t];for(let e of s)for(let t of n.get(e)??[])a.has(t.index)||(a.add(t.index),s.push(t.index));return e.filter(e=>a.has(e.index))}function V(e){let t=e.type??"",n=e.identifier??"";return!!(e.hittable||/\bbutton\b/i.test(t)||T.test(n)||/(?:^|:)id\/permission_(?:allow|deny)/i.test(n))}function J(e){if(!e)return"";let t=[e.label,e.value].filter(e=>"string"==typeof e&&e.trim().length>0);return t[0]?.trim()??""}async function Q(e,t,n={}){if("wait"===t)return await X(e,n.timeoutMs??1e4);if("get"===t){var a;let t=await ee(e);return{kind:"alertStatus",platform:"android",action:"get",alert:a=t?.alert??null,...a?b("Alert visible"):b("No alert visible")}}return await Y(e,t)}async function X(e,t){let n=Date.now(),a=await Z(e,t);if(!a)throw new r("COMMAND_FAILED","alert wait timed out");return{kind:"alertWait",platform:"android",action:"wait",alert:a.alert,waitedMs:Date.now()-n,...b("Alert visible")}}async function Y(e,t){var n;let a,s,o=await Z(e,2e3);if(!o)throw new r("COMMAND_FAILED","alert not found",{hint:"If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref."});let i=(n=o.buttons,a="accept"===t?"accept":"dismiss",(s=n.find(e=>e.role===a))||("dismiss"===t?n.find(e=>"neutral"===e.role)??null:1===n.length?n[0]??null:null));if(i)return await d(e,i.x,i.y),et(t,o.alert,i.label);if("dismiss"===t)return await p(e),et(t,o.alert,"Back");throw new r("COMMAND_FAILED","alert accept found an alert but no accept button",{alert:o.alert,hint:"Inspect alert get --json for visible buttons, then use press by visible label/ref if needed."})}async function Z(e,t){let n=Date.now();for(;Date.now()-n<t;){let t=await ee(e);if(t)return t;await I(300)}return null}async function ee(e){return function(e){var t,n,a,o;let r,i=function(e){let t=e.filter(e=>{var t;let n;return n=(t=e).bundleId??"",H.has(n)||B.test(t.identifier??"")});if(t.length)return{nodes:t,source:"permission"};let n=function(e){let t=e.filter(e=>{var t;let n;return n=(t=e).bundleId??"",L.has(n)&&q.test(J(t))});if(0===t.length)return[];let n=z(e,t);return void 0===n?t:G(e,n).filter(e=>e.bundleId&&L.has(e.bundleId))}(e);return n.length?{nodes:n,source:"system-dialog"}:{nodes:function(e){var t;let n,a,s=e.filter(e=>{var t;return t=e.type??"",/(?:^|[.$])[^.]*Dialog$/i.test(t)}),o=e.filter(e=>$.test(e.identifier??"")),r=s.length?[...s,...o]:(n=(t=o).some(e=>T.test(e.identifier??"")),a=t.some(e=>!T.test(e.identifier??"")),n&&a?t:[]);if(0===r.length)return[];let i=z(e,r);return void 0===i?r:G(e,i)}(e),source:"native-dialog"}}(e),l=i.nodes;if(0===l.length)return null;let d=function(e){let t=new Set,n=[];for(let a of e){let e=J(a);if(!e||!a.rect||!V(a))continue;let o=e.trim().toLowerCase();if(!o||t.has(o))continue;t.add(o);let r=s(a.rect);n.push({label:e,x:r.x,y:r.y,role:function(e,t){var n;let a=(n=e.identifier??"",/(?:^|:)id\/button1$/i.test(n)?"accept":/(?:^|:)id\/button2$/i.test(n)?"dismiss":/(?:^|:)id\/button3$/i.test(n)?"neutral":/(?:^|:)id\/permission_allow/i.test(n)?"accept":/(?:^|:)id\/permission_deny/i.test(n)?"dismiss":null);return a||(F.test(t.trim())?"accept":j.test(t.trim())?"dismiss":"neutral")}(a,e)})}return n}(l),p=l.filter(e=>J(e)&&!V(e)),u=J((t=p).find(e=>/(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(e.identifier??"")))||J(t[0]),c=(n=p,a=u,(r=n.map(e=>J(e)).filter(e=>e&&e!==a)).length?[...new Set(r)].join("\n"):void 0),f=(o=l,o.find(e=>e.bundleId)?.bundleId);return{alert:{...u?{title:u}:{},...c?{message:c}:{},buttons:d.map(e=>e.label),platform:"android",source:i.source,...f?{packageName:f}:{}},buttons:d}}((await i("snapshot_capture",async()=>await y(e,{helperWaitForIdleTimeoutMs:0,includeHiddenContentHints:!1}),{backend:"android",purpose:"alert"})).nodes)}function et(e,t,n){return{kind:"alertHandled",platform:"android",action:e,handled:!0,alert:t,button:n,...b(`Alert ${e}ed`)}}async function en(e){var t;let{req:n,logPath:a,session:s,device:o}=e,r="accept"===(t=n.positionals?.[0])||"dismiss"===t||"wait"===t?t:"get",i=s?"frontmost-app"===s.surface?{surface:"frontmost-app"}:{bundleId:s.appBundleId,surface:s.surface}:{};if(!g("alert",o))return w("UNSUPPORTED_OPERATION","alert is not supported on this device");if("android"===o.platform){let t=D(n.positionals?.[1])??1e4;return er(e,await Q(o,r,{timeoutMs:t}))}if("macos"===o.platform){let t=async e=>await h(e,i);return await ea(e,r,t)}let l={verbose:n.flags?.verbose,logPath:a,traceLogPath:s?.trace?.outPath,requestId:n.meta?.requestId},d=async e=>await _(o,{command:"alert",action:e,appBundleId:s?.appBundleId},l);return await ea(e,r,d)}async function ea(e,t,n){if("wait"===t)return await es(e,n);let a="accept"===t||"dismiss"===t?t:"get";return"accept"===a||"dismiss"===a?await eo(e,a,n):er(e,await n("get"))}async function es(e,t){let n=D(e.req.positionals?.[1])??1e4,a=Date.now();for(;Date.now()-a<n;){try{return er(e,await t("get"))}catch{}await I(300)}return w("COMMAND_FAILED","alert wait timed out")}async function eo(e,t,n){var a,s;let o,i,l=Date.now();for(;Date.now()-l<2e3;){try{return er(e,await n(t))}catch(t){i=t;let e=String(t?.message??"").toLowerCase();if(!e.includes("alert not found")&&!e.includes("no alert"))break}await I(300)}throw(a=i)instanceof r&&(s=a,(o=String(s?.message??"").toLowerCase()).includes("alert not found")||o.includes("no alert"))?new r(a.code,a.message,{...a.details??{},hint:"If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref."}):a}function er(e,t){return N(e.sessionStore,e.session,e.req,t),{ok:!0,data:t}}async function ei(e){let{req:t,logPath:n,sessionStore:a,session:s,device:o,parsed:r}=e,{setting:i,state:l,permissionTarget:d,latitude:p,longitude:u}=r;if(!g("settings",o))return w("UNSUPPORTED_OPERATION","settings is not supported on this device");if("macos"===o.platform&&!E(i))return w("INVALID_ARGS",K(i));let c=s?.appBundleId,m="permission"===i?[i,l,d??"",t.positionals?.[3]??"",c??""]:"location"===i&&"set"===l?[i,l,p??"",u??"",c??""]:[i,l,c??""],h=await f(o,"settings",m,t.flags?.out,{...S(n,t.flags,c,s?.trace?.outPath)});return N(a,s,t,h??{setting:i,state:l}),{ok:!0,data:h??{setting:i,state:l}}}async function el(e){var t,n;let a;if("snapshot"!==e.command||"android"!==e.device.platform)return;let s=o(e.error);if("COMMAND_FAILED"===(t=s).code&&(n=t,a=`${n.message}
-${n.hint??""}`,/Android UI hierarchy dump timed out/i.test(a)||/Stock UIAutomator fallback was skipped/i.test(a)||/Android accessibility snapshots can be blocked/i.test(a)||function(e){if(!e||"object"!=typeof e)return!1;let t=String(e.errorType??""),n=String(e.message??"");return/TimeoutException/i.test(t)||/timed out/i.test(n)}(t.details?.helper)||function(e){var t;if(!e)return!1;let n=e?.timeoutMs,a=e?.cmd,s=Array.isArray(t=e?.args)?t.map(String):"string"==typeof t?t.split(/\s+/):[];return"number"==typeof n&&"adb"===a&&s.includes("uiautomator")&&s.includes("dump")}(t.details)))return{ok:!1,error:{...s,details:{...s.details??{},androidSnapshotTimeoutScreenshot:await ed(e)}}}}async function ed(e){try{var s,r,i;let o=await t.mkdtemp(a.join(n.tmpdir(),"agent-device-android-snapshot-timeout-")),d=a.join(o,"snapshot-timeout-overlay-refs.png"),p=(s=await f(e.device,"screenshot",[d],void 0,{...S(e.logPath,{screenshotNoStabilize:!0},e.session?.appBundleId,e.session?.trace?.outPath),surface:e.session?.surface}),r=d,(i=s,"object"==typeof i&&null!==i&&"path"in i&&"string"==typeof i.path)?s.path:r);await t.access(p);let u=await ep(p,e.session);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_captured",data:{path:p,overlayRefCount:"overlayRefCount"in u?u.overlayRefCount:void 0,overlayRefsAnnotated:"overlayRefsAnnotated"in u?u.overlayRefsAnnotated:void 0}}),u}catch(t){let e=o(t);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_failed",data:{error:e.message}}),{captureFailed:!0,error:e.message}}}async function ep(e,t){if(!t?.snapshot)return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"unavailable",overlayRefCount:0};try{let n=await v({screenshotPath:e,snapshot:t.snapshot});return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:n.length>0,overlayRefCount:n.length,overlayRefSource:"session-snapshot",overlayRefs:n}}catch(n){let t=o(n);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_overlay_failed",data:{path:e,error:t.message}}),{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"session-snapshot",overlayRefCount:0,overlayAnnotationError:t.message}}}async function eu(e){return await ef({...e,command:"snapshot",unsupportedMessage:"snapshot is not supported on this device",execute:async({runtime:e,sessionName:t,req:n,snapshotScope:a})=>{let s=await e.capture.snapshot({session:t,interactiveOnly:n.flags?.snapshotInteractiveOnly,compact:n.flags?.snapshotCompact,depth:n.flags?.snapshotDepth,scope:a,raw:n.flags?.snapshotRaw,forceFull:n.flags?.snapshotForceFull});return{data:s,record:{kind:"snapshot",nodes:s.nodes.length,truncated:s.truncated}}}})}async function ec(e){return await ef({...e,command:"diff",unsupportedMessage:"diff is not supported on this device",execute:async({runtime:e,sessionName:t,req:n,snapshotScope:a})=>{let s=await e.capture.diffSnapshot({session:t,interactiveOnly:n.flags?.snapshotInteractiveOnly,compact:n.flags?.snapshotCompact,depth:n.flags?.snapshotDepth,scope:a,raw:n.flags?.snapshotRaw});return{data:s,record:{kind:"diff",mode:"snapshot",baselineInitialized:s.baselineInitialized,summary:s.summary}}}})}async function ef(e){let{req:t,sessionName:n,logPath:a,sessionStore:s}=e,{session:o,device:i}=await M(s,n,t.flags);if(!g(e.command,i))return w("UNSUPPORTED_OPERATION",e.unsupportedMessage);let l=m(t.flags?.snapshotScope,o);return l.ok?await P(o,i,async()=>{var d,p;let f,m,h=function(e){let{req:t,sessionName:n,logPath:a,sessionStore:s,session:o,device:i,snapshotScope:l}=e;return O({backend:function(e){let{req:t,logPath:n,session:a,device:s,snapshotScope:o}=e;return{platform:s.platform,captureSnapshot:async(e,r)=>{let i=await c({device:s,session:a,flags:t.flags,outPath:r?.outPath??t.flags?.out,logPath:n,snapshotScope:o});return{snapshot:i.snapshot,analysis:i.analysis,androidSnapshot:i.androidSnapshot,freshness:i.freshness,appName:a?.appBundleId?a.appName??a.appBundleId:void 0,appBundleId:a?.appBundleId}}}}({req:t,logPath:a,session:o,device:i,snapshotScope:l}),...R("snapshot"),sessions:u({sessionName:n,getSession:()=>s.get(n),recordOptions:{includeSnapshot:!0},setRecord:e=>{var a;let o=function(e){if(!e.snapshot)throw new r("UNKNOWN","snapshot runtime did not produce session state");return e}(e),l=s.get(n);s.set(n,function(e){var t,n;let{current:a,sessionName:s,device:o,record:r,refScopedSnapshot:i}=e,l=(t=a,n=r,i&&n.snapshot?.nodes.length===0&&t?.snapshot!==void 0),d=l?a.snapshot:r.snapshot,p=A({session:a,sessionName:s,device:o,snapshot:d,appBundleId:r.appBundleId});return p.snapshotScopeSource=function(e){let{current:t,keepCurrentSnapshot:n,refScopedSnapshot:a}=e;if(a)return n?t?.snapshotScopeSource:t?.snapshotScopeSource??t?.snapshot}({current:a,keepCurrentSnapshot:l,refScopedSnapshot:i}),r.appName&&(p.appName=r.appName),p}({current:l,sessionName:n,device:i,record:o,refScopedSnapshot:(a=t,a.flags?.snapshotScope?.trim().startsWith("@")===!0)}))}})})}({req:t,sessionName:n,logPath:a,sessionStore:s,session:o,device:i,snapshotScope:l.scope});try{m=await e.execute({runtime:h,sessionName:n,req:t,snapshotScope:l.scope})}catch(n){let t=await el({error:n,command:e.command,logPath:a,session:o,device:i});if(!t)throw n;return t}return(f=(d={req:t,sessionName:n,sessionStore:s,result:m.record}).sessionStore.get(d.sessionName))&&d.sessionStore.recordAction(f,{command:d.req.command,positionals:d.req.positionals??[],flags:d.req.flags??{},result:"snapshot"===(p=d.result).kind?{nodes:p.nodes,truncated:p.truncated}:{mode:p.mode,baselineInitialized:p.baselineInitialized,summary:p.summary}}),{ok:!0,data:m.data}}):l}let em=U.snapshot,eh={snapshot:async({req:e,sessionName:t,logPath:n,sessionStore:a})=>await eu({req:e,sessionName:t,logPath:n,sessionStore:a}),diff:async({req:e,sessionName:t,logPath:n,sessionStore:a})=>e.positionals?.[0]!=="snapshot"?w("INVALID_ARGS","diff currently supports only: diff snapshot"):await ec({req:e,sessionName:t,logPath:n,sessionStore:a}),wait:async({req:e,sessionName:t,logPath:n,sessionStore:a})=>await C({req:e,sessionName:t,logPath:n,sessionStore:a}),alert:async({req:e,sessionName:t,logPath:n,sessionStore:a})=>{let{session:s,device:o}=await M(a,t,e.flags);return await P(s,o,async()=>await en({req:e,logPath:n,sessionStore:a,session:s,device:o}))},settings:async({req:e,sessionName:t,logPath:n,sessionStore:a})=>{let s,o,r,i=(s=e.positionals?.[0]?.toLowerCase(),o=e.positionals?.[1]?.toLowerCase(),r=e.positionals?.[2]?.toLowerCase(),s&&o&&("permission"!==s||r)&&("location"!==s||"set"!==o||e.positionals?.[2]&&e.positionals?.[3])?{ok:!0,parsed:{setting:s,state:o,permissionTarget:r,latitude:e.positionals?.[2],longitude:e.positionals?.[3]}}:w("INVALID_ARGS",k));if(!i.ok)return i;let{session:l,device:d}=await M(a,t,e.flags);return await P(l,d,async()=>await ei({req:e,logPath:n,sessionStore:a,session:l,device:d,parsed:i.parsed}))}};async function ew(e){let t=e.req.command;if(!em.has(t))return null;let n=eh[t];return n?await n(e):w("COMMAND_FAILED",`Snapshot command has no handler: ${t}`)}export{x as snapshot_namespaceObject};
+import{__webpack_require__ as e}from"./rslib-runtime.js";import{promises as t}from"node:fs";import a from"node:os";import n from"node:path";import{centerOfRect as s}from"./4057.js";import{normalizeError as o,AppError as r}from"./9152.js";import{withDiagnosticTimer as i,emitDiagnostic as l}from"./7599.js";import{pressAndroid as d,backAndroid as p}from"./input-actions.js";import{createDaemonRuntimeSessionStore as u,captureSnapshot as c,dispatchCommand as f,resolveSnapshotScope as m,runMacOsAlertAction as h,errorResponse as w,runIosRunnerCommand as _,isCommandSupportedOnDevice as g,snapshotAndroid as y,annotateScreenshotWithRefs as v,context_contextFromFlags as S}from"./2415.js";import{successText as b}from"./1998.js";import{sleep as I}from"./4829.js";import{buildSnapshotSession as A,dispatchWaitViaRuntime as C,createDaemonRuntimePolicy as R,recordIfSession as N,resolveSessionDevice as M,withSessionlessRunnerCleanup as P}from"./selector-runtime.js";import{parseTimeout as D}from"./6085.js";import{SETTINGS_INVALID_ARGS_MESSAGE as k,isMacOsSettingSupported as E,getUnsupportedMacOsSettingMessage as K}from"./1352.js";import{createAgentDevice as O}from"./9533.js";import{DAEMON_COMMAND_GROUPS as U}from"./5792.js";var x={};e.r(x),e.d(x,{SNAPSHOT_COMMAND_HANDLERS:()=>eh,handleSnapshotCommands:()=>ew});let H=new Set(["com.android.permissioncontroller","com.google.android.permissioncontroller","com.google.android.packageinstaller","com.android.packageinstaller"]),L=new Set(["android","com.android.systemui"]),B=/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i,$=/^android:id\/button[123]$/i,T=/(?:^|:)id\/permission_/i,q=/\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i,F=/^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i,j=/^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i;function z(e,t){let a=t[0];if(!a)return;let n=G(e,a.index);for(let a of t.slice(1)){let t=new Set(G(e,a.index));for(let e=n.length-1;e>=0;e-=1)t.has(n[e])||n.splice(e,1)}return n[n.length-1]}function G(e,t){let a=new Map(e.map(e=>[e.index,e])),n=[],s=a.get(t);for(;s;)n.push(s.index),s=void 0===s.parentIndex?void 0:a.get(s.parentIndex);return n.reverse()}function V(e,t){let a=new Map;for(let t of e){if(void 0===t.parentIndex)continue;let e=a.get(t.parentIndex)??[];e.push(t),a.set(t.parentIndex,e)}let n=new Set([t]),s=[t];for(let e of s)for(let t of a.get(e)??[])n.has(t.index)||(n.add(t.index),s.push(t.index));return e.filter(e=>n.has(e.index))}function W(e){let t=e.type??"",a=e.identifier??"";return!!(e.hittable||/\bbutton\b/i.test(t)||$.test(a)||/(?:^|:)id\/permission_(?:allow|deny)/i.test(a))}function J(e){if(!e)return"";let t=[e.label,e.value].filter(e=>"string"==typeof e&&e.trim().length>0);return t[0]?.trim()??""}async function Q(e,t,a={}){if("wait"===t)return await X(e,a.timeoutMs??1e4);if("get"===t){var n;let t=await ee(e);return{kind:"alertStatus",platform:"android",action:"get",alert:n=t?.alert??null,...n?b("Alert visible"):b("No alert visible")}}return await Y(e,t)}async function X(e,t){let a=Date.now(),n=await Z(e,t);if(!n)throw new r("COMMAND_FAILED","alert wait timed out");return{kind:"alertWait",platform:"android",action:"wait",alert:n.alert,waitedMs:Date.now()-a,...b("Alert visible")}}async function Y(e,t){var a;let n,s,o=await Z(e,2e3);if(!o)throw new r("COMMAND_FAILED","alert not found",{hint:"If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref."});let i=(a=o.buttons,n="accept"===t?"accept":"dismiss",(s=a.find(e=>e.role===n))||("dismiss"===t?a.find(e=>"neutral"===e.role)??null:1===a.length?a[0]??null:null));if(i)return await d(e,i.x,i.y),et(t,o.alert,i.label);if("dismiss"===t)return await p(e),et(t,o.alert,"Back");throw new r("COMMAND_FAILED","alert accept found an alert but no accept button",{alert:o.alert,hint:"Inspect alert get --json for visible buttons, then use press by visible label/ref if needed."})}async function Z(e,t){let a=Date.now();for(;Date.now()-a<t;){let t=await ee(e);if(t)return t;await I(300)}return null}async function ee(e){return function(e){var t,a,n,o;let r,i=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",H.has(a)||T.test(t.identifier??"")});if(t.length)return{nodes:t,source:"permission"};let a=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",L.has(a)&&q.test(J(t))});if(0===t.length)return[];let a=z(e,t);return void 0===a?t:V(e,a).filter(e=>e.bundleId&&L.has(e.bundleId))}(e);return a.length?{nodes:a,source:"system-dialog"}:{nodes:function(e){var t;let a,n,s=e.filter(e=>{var t;return t=e.type??"",/(?:^|[.$])[^.]*Dialog$/i.test(t)}),o=e.filter(e=>B.test(e.identifier??"")),r=s.length?[...s,...o]:(a=(t=o).some(e=>$.test(e.identifier??"")),n=t.some(e=>!$.test(e.identifier??"")),a&&n?t:[]);if(0===r.length)return[];let i=z(e,r);return void 0===i?r:V(e,i)}(e),source:"native-dialog"}}(e),l=i.nodes;if(0===l.length)return null;let d=function(e){let t=new Set,a=[];for(let n of e){let e=J(n);if(!e||!n.rect||!W(n))continue;let o=e.trim().toLowerCase();if(!o||t.has(o))continue;t.add(o);let r=s(n.rect);a.push({label:e,x:r.x,y:r.y,role:function(e,t){var a;let n=(a=e.identifier??"",/(?:^|:)id\/button1$/i.test(a)?"accept":/(?:^|:)id\/button2$/i.test(a)?"dismiss":/(?:^|:)id\/button3$/i.test(a)?"neutral":/(?:^|:)id\/permission_allow/i.test(a)?"accept":/(?:^|:)id\/permission_deny/i.test(a)?"dismiss":null);return n||(F.test(t.trim())?"accept":j.test(t.trim())?"dismiss":"neutral")}(n,e)})}return a}(l),p=l.filter(e=>J(e)&&!W(e)),u=J((t=p).find(e=>/(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(e.identifier??"")))||J(t[0]),c=(a=p,n=u,(r=a.map(e=>J(e)).filter(e=>e&&e!==n)).length?[...new Set(r)].join("\n"):void 0),f=(o=l,o.find(e=>e.bundleId)?.bundleId);return{alert:{...u?{title:u}:{},...c?{message:c}:{},buttons:d.map(e=>e.label),platform:"android",source:i.source,...f?{packageName:f}:{}},buttons:d}}((await i("snapshot_capture",async()=>await y(e,{helperWaitForIdleTimeoutMs:0,includeHiddenContentHints:!1}),{backend:"android",purpose:"alert"})).nodes)}function et(e,t,a){return{kind:"alertHandled",platform:"android",action:e,handled:!0,alert:t,button:a,...b(`Alert ${e}ed`)}}async function ea(e){var t;let{req:a,logPath:n,session:s,device:o}=e,r="accept"===(t=a.positionals?.[0])||"dismiss"===t||"wait"===t?t:"get",i=s?"frontmost-app"===s.surface?{surface:"frontmost-app"}:{bundleId:s.appBundleId,surface:s.surface}:{};if(!g("alert",o))return w("UNSUPPORTED_OPERATION","alert is not supported on this device");if("android"===o.platform){let t=D(a.positionals?.[1])??1e4;return er(e,await Q(o,r,{timeoutMs:t}))}if("macos"===o.platform){let t=async e=>await h(e,i);return await en(e,r,t)}let l={verbose:a.flags?.verbose,logPath:n,traceLogPath:s?.trace?.outPath,requestId:a.meta?.requestId},d=async e=>await _(o,{command:"alert",action:e,appBundleId:s?.appBundleId},l);return await en(e,r,d)}async function en(e,t,a){if("wait"===t)return await es(e,a);let n="accept"===t||"dismiss"===t?t:"get";return"accept"===n||"dismiss"===n?await eo(e,n,a):er(e,await a("get"))}async function es(e,t){let a=D(e.req.positionals?.[1])??1e4,n=Date.now();for(;Date.now()-n<a;){try{return er(e,await t("get"))}catch{}await I(300)}return w("COMMAND_FAILED","alert wait timed out")}async function eo(e,t,a){var n,s;let o,i,l=Date.now();for(;Date.now()-l<2e3;){try{return er(e,await a(t))}catch(t){i=t;let e=String(t?.message??"").toLowerCase();if(!e.includes("alert not found")&&!e.includes("no alert"))break}await I(300)}throw(n=i)instanceof r&&(s=n,(o=String(s?.message??"").toLowerCase()).includes("alert not found")||o.includes("no alert"))?new r(n.code,n.message,{...n.details??{},hint:"If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref."}):n}function er(e,t){return N(e.sessionStore,e.session,e.req,t),{ok:!0,data:t}}async function ei(e){let{req:t,logPath:a,sessionStore:n,session:s,device:o,parsed:r}=e,{setting:i,state:l,appBundleId:d,permissionTarget:p,latitude:u,longitude:c}=r;if(!g("settings",o))return w("UNSUPPORTED_OPERATION","settings is not supported on this device");if("macos"===o.platform&&!E(i))return w("INVALID_ARGS",K(i));let m=d??s?.appBundleId;if("clear-app-state"===i&&!m)return w("INVALID_ARGS","settings clear-app-state requires an app id when no app is bound to the session");let h="clear-app-state"===i?[i,l,m??""]:"permission"===i?[i,l,p??"",t.positionals?.[3]??"",m??""]:"location"===i&&"set"===l?[i,l,u??"",c??"",m??""]:[i,l,m??""],_=await f(o,"settings",h,t.flags?.out,{...S(a,t.flags,m,s?.trace?.outPath)});return N(n,s,t,_??{setting:i,state:l}),{ok:!0,data:_??{setting:i,state:l}}}async function el(e){var t,a;let n;if("snapshot"!==e.command||"android"!==e.device.platform)return;let s=o(e.error);if("COMMAND_FAILED"===(t=s).code&&(a=t,n=`${a.message}
+${a.hint??""}`,/Android UI hierarchy dump timed out/i.test(n)||/Stock UIAutomator fallback was skipped/i.test(n)||/Android accessibility snapshots can be blocked/i.test(n)||function(e){if(!e||"object"!=typeof e)return!1;let t=String(e.errorType??""),a=String(e.message??"");return/TimeoutException/i.test(t)||/timed out/i.test(a)}(t.details?.helper)||function(e){var t;if(!e)return!1;let a=e?.timeoutMs,n=e?.cmd,s=Array.isArray(t=e?.args)?t.map(String):"string"==typeof t?t.split(/\s+/):[];return"number"==typeof a&&"adb"===n&&s.includes("uiautomator")&&s.includes("dump")}(t.details)))return{ok:!1,error:{...s,details:{...s.details??{},androidSnapshotTimeoutScreenshot:await ed(e)}}}}async function ed(e){try{var s,r,i;let o=await t.mkdtemp(n.join(a.tmpdir(),"agent-device-android-snapshot-timeout-")),d=n.join(o,"snapshot-timeout-overlay-refs.png"),p=(s=await f(e.device,"screenshot",[d],void 0,{...S(e.logPath,{screenshotNoStabilize:!0},e.session?.appBundleId,e.session?.trace?.outPath),surface:e.session?.surface}),r=d,(i=s,"object"==typeof i&&null!==i&&"path"in i&&"string"==typeof i.path)?s.path:r);await t.access(p);let u=await ep(p,e.session);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_captured",data:{path:p,overlayRefCount:"overlayRefCount"in u?u.overlayRefCount:void 0,overlayRefsAnnotated:"overlayRefsAnnotated"in u?u.overlayRefsAnnotated:void 0}}),u}catch(t){let e=o(t);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_failed",data:{error:e.message}}),{captureFailed:!0,error:e.message}}}async function ep(e,t){if(!t?.snapshot)return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"unavailable",overlayRefCount:0};try{let a=await v({screenshotPath:e,snapshot:t.snapshot});return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:a.length>0,overlayRefCount:a.length,overlayRefSource:"session-snapshot",overlayRefs:a}}catch(a){let t=o(a);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_overlay_failed",data:{path:e,error:t.message}}),{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"session-snapshot",overlayRefCount:0,overlayAnnotationError:t.message}}}async function eu(e){return await ef({...e,command:"snapshot",unsupportedMessage:"snapshot is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.snapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,compact:a.flags?.snapshotCompact,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw,forceFull:a.flags?.snapshotForceFull});return{data:s,record:{kind:"snapshot",nodes:s.nodes.length,truncated:s.truncated}}}})}async function ec(e){return await ef({...e,command:"diff",unsupportedMessage:"diff is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.diffSnapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,compact:a.flags?.snapshotCompact,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw});return{data:s,record:{kind:"diff",mode:"snapshot",baselineInitialized:s.baselineInitialized,summary:s.summary}}}})}async function ef(e){let{req:t,sessionName:a,logPath:n,sessionStore:s}=e,{session:o,device:i}=await M(s,a,t.flags);if(!g(e.command,i))return w("UNSUPPORTED_OPERATION",e.unsupportedMessage);let l=m(t.flags?.snapshotScope,o);return l.ok?await P(o,i,async()=>{var d,p;let f,m,h=function(e){let{req:t,sessionName:a,logPath:n,sessionStore:s,session:o,device:i,snapshotScope:l}=e;return O({backend:function(e){let{req:t,logPath:a,session:n,device:s,snapshotScope:o}=e;return{platform:s.platform,captureSnapshot:async(e,r)=>{let i=await c({device:s,session:n,flags:t.flags,outPath:r?.outPath??t.flags?.out,logPath:a,snapshotScope:o});return{snapshot:i.snapshot,analysis:i.analysis,androidSnapshot:i.androidSnapshot,freshness:i.freshness,appName:n?.appBundleId?n.appName??n.appBundleId:void 0,appBundleId:n?.appBundleId}}}}({req:t,logPath:n,session:o,device:i,snapshotScope:l}),...R("snapshot"),sessions:u({sessionName:a,getSession:()=>s.get(a),recordOptions:{includeSnapshot:!0},setRecord:e=>{var n;let o=function(e){if(!e.snapshot)throw new r("UNKNOWN","snapshot runtime did not produce session state");return e}(e),l=s.get(a);s.set(a,function(e){var t,a;let{current:n,sessionName:s,device:o,record:r,refScopedSnapshot:i}=e,l=(t=n,a=r,i&&a.snapshot?.nodes.length===0&&t?.snapshot!==void 0),d=l?n.snapshot:r.snapshot,p=A({session:n,sessionName:s,device:o,snapshot:d,appBundleId:r.appBundleId});return p.snapshotScopeSource=function(e){let{current:t,keepCurrentSnapshot:a,refScopedSnapshot:n}=e;if(n)return a?t?.snapshotScopeSource:t?.snapshotScopeSource??t?.snapshot}({current:n,keepCurrentSnapshot:l,refScopedSnapshot:i}),r.appName&&(p.appName=r.appName),p}({current:l,sessionName:a,device:i,record:o,refScopedSnapshot:(n=t,n.flags?.snapshotScope?.trim().startsWith("@")===!0)}))}})})}({req:t,sessionName:a,logPath:n,sessionStore:s,session:o,device:i,snapshotScope:l.scope});try{m=await e.execute({runtime:h,sessionName:a,req:t,snapshotScope:l.scope})}catch(a){let t=await el({error:a,command:e.command,logPath:n,session:o,device:i});if(!t)throw a;return t}return(f=(d={req:t,sessionName:a,sessionStore:s,result:m.record}).sessionStore.get(d.sessionName))&&d.sessionStore.recordAction(f,{command:d.req.command,positionals:d.req.positionals??[],flags:d.req.flags??{},result:"snapshot"===(p=d.result).kind?{nodes:p.nodes,truncated:p.truncated}:{mode:p.mode,baselineInitialized:p.baselineInitialized,summary:p.summary}}),{ok:!0,data:m.data}}):l}let em=U.snapshot,eh={snapshot:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await eu({req:e,sessionName:t,logPath:a,sessionStore:n}),diff:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>e.positionals?.[0]!=="snapshot"?w("INVALID_ARGS","diff currently supports only: diff snapshot"):await ec({req:e,sessionName:t,logPath:a,sessionStore:n}),wait:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await C({req:e,sessionName:t,logPath:a,sessionStore:n}),alert:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let{session:s,device:o}=await M(n,t,e.flags);return await P(s,o,async()=>await ea({req:e,logPath:a,sessionStore:n,session:s,device:o}))},settings:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let s,o,r,i=(s=e.positionals?.[0]?.toLowerCase(),o=e.positionals?.[1]?.toLowerCase(),r=e.positionals?.[2]?.toLowerCase(),"clear-app-state"===s?{ok:!0,parsed:{setting:s,state:"clear",appBundleId:"clear"===o?e.positionals?.[2]:e.positionals?.[1]}}:s&&o&&("permission"!==s||r)&&("location"!==s||"set"!==o||e.positionals?.[2]&&e.positionals?.[3])?{ok:!0,parsed:{setting:s,state:o,permissionTarget:r,latitude:e.positionals?.[2],longitude:e.positionals?.[3]}}:w("INVALID_ARGS",k));if(!i.ok)return i;let{session:l,device:d}=await M(n,t,e.flags);return await P(l,d,async()=>await ei({req:e,logPath:a,sessionStore:n,session:l,device:d,parsed:i.parsed}))}};async function ew(e){let t=e.req.command;if(!em.has(t))return null;let a=eh[t];return a?await a(e):w("COMMAND_FAILED",`Snapshot command has no handler: ${t}`)}export{x as snapshot_namespaceObject};

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -17,10 +17,10 @@ extension RunnerTests {
     switch outcome {
     case .performed:
       return nil
-    case .unsupported(let message):
+    case .unsupported(let message, let hint):
       return Response(
         ok: false,
-        error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message)
+        error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
       )
     }
   }
@@ -646,12 +646,26 @@ extension RunnerTests {
         scope: command.scope,
         raw: command.raw ?? false
       )
-      if options.raw {
+      do {
+        let payload: DataPayload
+        if options.raw {
+          payload = try snapshotRaw(app: activeApp, options: options)
+        } else {
+          payload = try snapshotFast(app: activeApp, options: options)
+        }
         needsPostSnapshotInteractionDelay = true
-        return Response(ok: true, data: snapshotRaw(app: activeApp, options: options))
+        return Response(ok: true, data: payload)
+      } catch let failure as SnapshotCaptureFailure {
+        // Other thrown errors fall through to executeOnMainSafely's generic error response.
+        return Response(
+          ok: false,
+          error: ErrorPayload(
+            code: failure.code,
+            message: failure.message,
+            hint: failure.hint
+          )
+        )
       }
-      needsPostSnapshotInteractionDelay = true
-      return Response(ok: true, data: snapshotFast(app: activeApp, options: options))
     case .screenshot:
       let screenshot: XCUIScreenshot
 #if os(macOS)

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -283,6 +283,13 @@ extension RunnerTests {
 
   func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
     let point = CGPoint(x: x, y: y)
+    let textInputCandidates = textInputCandidatesAt(app: app, point: point)
+    for element in textInputCandidates where prefersExpandedTextRead(element) {
+      if let text = readableText(for: element) {
+        return text
+      }
+    }
+
     let candidates = app.descendants(matching: .any).allElementsBoundByIndex
       .filter { element in
         element.exists && !element.frame.isEmpty && element.frame.contains(point)
@@ -316,6 +323,18 @@ extension RunnerTests {
   }
 
   func clearTextInput(_ element: XCUIElement) {
+    // Skip the clear (delete burst + moveCaretToEnd edge-tap) ONLY when we can confirm the
+    // field is empty. Why skip: the edge-tap computes a point from the element frame, which can
+    // be stale after the field repositions on focus (e.g. the Settings search bar jumps
+    // bottom->top and reveals a "Suggestions" list) — tapping there navigates away instead of
+    // clearing; and replacing into an already-empty field is a no-op anyway.
+    // editableTextValue returns nil for secure (and unknown) fields, where we CANNOT confirm
+    // emptiness — those must still be cleared, or replace would concatenate stale + new text.
+    // So distinguish nil (clear) from "" (skip).
+    if let existing = editableTextValue(for: element, treatingPlaceholderAsEmpty: true),
+       existing.isEmpty {
+      return
+    }
 #if !os(tvOS)
     moveCaretToEnd(element: element)
 #endif
@@ -325,20 +344,28 @@ extension RunnerTests {
   }
 
   func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
-    let point = CGPoint(x: x, y: y)
-    var matched: XCUIElement?
+    return textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first
+  }
+
+  private func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [XCUIElement] {
+    var candidates: [XCUIElement] = []
     let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      // Query the text-input element types directly instead of enumerating the entire tree
+      // (app.descendants(.any).allElementsBoundByIndex snapshots every element and is ~10x
+      // slower — it dominated fill latency because resolveTextEntryElement re-runs this on
+      // each verify/repair poll once the focused field reference goes stale).
       // Prefer the smallest matching field so nested editable controls win over large containers.
-      let candidates = app.descendants(matching: .any).allElementsBoundByIndex
+      candidates = [
+        app.textFields,
+        app.secureTextFields,
+        app.searchFields,
+        app.textViews,
+      ]
+        .flatMap { $0.allElementsBoundByIndex }
         .filter { element in
           guard element.exists else { return false }
-          switch element.elementType {
-          case .textField, .secureTextField, .searchField, .textView:
-            let frame = element.frame
-            return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2)
-          default:
-            return false
-          }
+          let frame = element.frame
+          return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2)
         }
         .sorted { left, right in
           let leftArea = max(1, left.frame.width * left.frame.height)
@@ -354,16 +381,15 @@ extension RunnerTests {
           }
           return left.elementType.rawValue < right.elementType.rawValue
         }
-      matched = candidates.first
     })
     if let exceptionMessage {
       NSLog(
         "AGENT_DEVICE_RUNNER_TEXT_INPUT_AT_POINT_IGNORED_EXCEPTION=%@",
         exceptionMessage
       )
-      return nil
+      return []
     }
-    return matched
+    return candidates
   }
 
   private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool {
@@ -411,11 +437,17 @@ extension RunnerTests {
     return target
 #else
     let latest = target
+    let keyboardVisibleAtEntry = isKeyboardVisible(app: app)
     let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout)
     while Date() < deadline {
       if let focused = focusedTextInput(app: app) {
         return focused
       }
+      // focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to
+      // visible after our tap as the focus-moved signal. Don't fast-path when it was already up.
+      if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
+        return latest
+      }
       sleepFor(TextEntryTiming.pollInterval)
     }
     return latest
@@ -866,6 +898,7 @@ extension RunnerTests {
   ) -> XCUIElement? {
 #if os(iOS)
     var latest = resolveTextEntryElement(app: app, target: target)
+    let keyboardVisibleAtEntry = isKeyboardVisible(app: app)
     let deadline = Date().addingTimeInterval(timeout)
     let hardwareKeyboardFallback = Date().addingTimeInterval(
       min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout)
@@ -878,6 +911,14 @@ extension RunnerTests {
           return focused
         }
       }
+      // Fast-path on a keyboard hidden->visible transition: our tapped field gained focus, so
+      // return immediately instead of burning the full readinessTimeout (warmup-first-char echo
+      // + post-type verify/repair remain as drop safety nets). When the keyboard was ALREADY up
+      // (back-to-back fills), this isn't a focus signal — fall through to the settle/timeout so
+      // text isn't sent to the previously-focused field.
+      if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
+        return latest
+      }
       sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app)
       if !sawSoftwareKeyboard && Date() >= hardwareKeyboardFallback && latest != nil {
         return latest
@@ -934,6 +975,15 @@ extension RunnerTests {
     return visibleKeyboardFrame(app: app) != nil
   }
 
+  /// A focus-moved signal for iOS text entry, where `focusedTextInput` is intentionally nil.
+  /// The software keyboard TRANSITIONING from hidden (at entry) to visible means the field we
+  /// just tapped gained first-responder. If the keyboard was ALREADY up (e.g. back-to-back
+  /// fills into different fields), its visibility is not evidence focus moved to the new field,
+  /// so callers must keep waiting rather than typing into the previously-focused field.
+  private func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool {
+    return !wasVisibleAtEntry && isKeyboardVisible(app: app)
+  }
+
   private func keyboardElementExists(app: XCUIApplication) -> Bool {
 #if os(iOS)
     var exists = false
@@ -978,6 +1028,14 @@ extension RunnerTests {
       return (wasVisible: true, dismissed: !visible, visible: visible)
     }
 
+    if tapKeyboardReturnControl(app: app, allowCoordinateFallback: true) {
+      sleepFor(0.2)
+      let visible = isKeyboardVisible(app: app)
+      if !visible {
+        return (wasVisible: true, dismissed: true, visible: false)
+      }
+    }
+
     return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
 #endif
   }
@@ -1098,7 +1156,10 @@ extension RunnerTests {
 #endif
   }
 
-  private func tapKeyboardReturnControl(app: XCUIApplication) -> Bool {
+  private func tapKeyboardReturnControl(
+    app: XCUIApplication,
+    allowCoordinateFallback: Bool = false
+  ) -> Bool {
 #if os(iOS)
     for label in ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] {
       let candidates = [
@@ -1109,6 +1170,21 @@ extension RunnerTests {
         hittable.tap()
         return true
       }
+      if allowCoordinateFallback,
+         let keyboardFrame = visibleKeyboardFrame(app: app),
+         let framed = candidates.first(where: {
+           guard $0.exists else { return false }
+           let frame = $0.frame
+           return !frame.isEmpty && keyboardFrame.contains(CGPoint(x: frame.midX, y: frame.midY))
+         }) {
+        let frame = framed.frame
+        switch tapAt(app: app, x: frame.midX, y: frame.midY) {
+        case .performed:
+          return true
+        case .unsupported:
+          return false
+        }
+      }
     }
 #endif
     return false
@@ -1510,11 +1586,69 @@ extension RunnerTests {
   }
 
   func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
-    return performCoordinatePinch(app: app, scale: scale, x: x, y: y)
+#if os(iOS)
+    // A coordinate tap+drag is a single-finger gesture: React Native reads it as a pan
+    // and the pinch scale never changes (#629). Drive the two-finger XCTest synthesis
+    // path (the same one transformGesture uses) with zero translation/rotation so RN's
+    // pinch recognizer actually fires.
+    let frame = interactionRoot(app: app).frame
+    let centerX = x ?? Double(frame.midX)
+    let centerY = y ?? Double(frame.midY)
+    return transformGesture(
+      app: app,
+      x: centerX,
+      y: centerY,
+      dx: 0,
+      dy: 0,
+      scale: scale,
+      degrees: 0,
+      durationMs: 300
+    )
+#elseif os(tvOS)
+    return .unsupported(
+      message: "pinch is not supported on tvOS",
+      hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)."
+    )
+#else
+    return .unsupported(
+      message: "pinch is not supported on macOS",
+      hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)."
+    )
+#endif
   }
 
   func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
-    return performCoordinateRotateGesture(app: app, degrees: degrees, x: x, y: y, velocity: velocity)
+#if os(iOS)
+    // Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634)
+    // with zero translation/scale so React Native's rotation recognizer actually fires. The native
+    // XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture
+    // handler does not read reliably — the same class of problem #629/#634 fixed for pinch.
+    // velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract
+    // keeps it for compatibility and direction is carried entirely by the sign of `degrees`.
+    let frame = interactionRoot(app: app).frame
+    let centerX = x ?? Double(frame.midX)
+    let centerY = y ?? Double(frame.midY)
+    return transformGesture(
+      app: app,
+      x: centerX,
+      y: centerY,
+      dx: 0,
+      dy: 0,
+      scale: 1,
+      degrees: degrees,
+      durationMs: 300
+    )
+#elseif os(tvOS)
+    return .unsupported(
+      message: "rotate-gesture is not supported on tvOS",
+      hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)."
+    )
+#else
+    return .unsupported(
+      message: "rotate-gesture is not supported on macOS",
+      hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)."
+    )
+#endif
   }
 
   func transformGesture(
@@ -1540,13 +1674,22 @@ extension RunnerTests {
       radius: transformGestureRadius(frame: target.frame, scale: scale),
       durationMs: durationMs
     ) {
-      return .unsupported(message)
+      return .unsupported(
+        message: message,
+        hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)."
+      )
     }
     return .performed
 #elseif os(tvOS)
-    return .unsupported("transformGesture is not supported on tvOS")
+    return .unsupported(
+      message: "transformGesture is not supported on tvOS",
+      hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)."
+    )
 #else
-    return .unsupported("transformGesture is not supported on macOS")
+    return .unsupported(
+      message: "transformGesture is not supported on macOS",
+      hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)."
+    )
 #endif
   }
 
@@ -1558,57 +1701,6 @@ extension RunnerTests {
     return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
   }
 
-  private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
-#if os(tvOS)
-    return .unsupported("pinch is not supported on tvOS")
-#else
-    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
-
-    // Use double-tap + drag gesture for reliable map zoom
-    // Zoom in (scale > 1): tap then drag UP
-    // Zoom out (scale < 1): tap then drag DOWN
-
-    // Determine center point (use provided x/y or screen center)
-    let centerX = x.map { $0 / target.frame.width } ?? 0.5
-    let centerY = y.map { $0 / target.frame.height } ?? 0.5
-    let center = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: centerY))
-
-    // Calculate drag distance based on scale (clamped to reasonable range)
-    // Larger scale = more drag distance
-    let dragAmount: CGFloat
-    if scale > 1.0 {
-      // Zoom in: drag up (negative Y direction in normalized coords)
-      dragAmount = min(0.4, CGFloat(scale - 1.0) * 0.2)
-    } else {
-      // Zoom out: drag down (positive Y direction)
-      dragAmount = min(0.4, CGFloat(1.0 - scale) * 0.4)
-    }
-
-    let endY = scale > 1.0 ? (centerY - Double(dragAmount)) : (centerY + Double(dragAmount))
-    let endPoint = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: max(0.1, min(0.9, endY))))
-
-    // Tap first (first tap of double-tap)
-    center.tap()
-
-    // Immediately press and drag (second tap + drag)
-    center.press(forDuration: 0.05, thenDragTo: endPoint)
-    return .performed
-#endif
-  }
-
-  private func performCoordinateRotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
-#if os(iOS)
-    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
-    let radians = CGFloat(degrees * .pi / 180.0)
-    target.rotate(radians, withVelocity: CGFloat(velocity))
-    return .performed
-#elseif os(tvOS)
-    return .unsupported("rotate-gesture is not supported on tvOS")
-#else
-    return .unsupported("rotate-gesture is not supported on macOS")
-#endif
-  }
-
   private func interactionRoot(app: XCUIApplication) -> XCUIElement {
     let windows = app.windows.allElementsBoundByIndex
     if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
@@ -1619,7 +1711,10 @@ extension RunnerTests {
 
   private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
 #if os(tvOS)
-    return .unsupported("coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
+    return .unsupported(
+      message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
+      hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
+    )
 #else
     interactionCoordinate(app: app, x: x, y: y).tap()
     return .performed
@@ -1628,7 +1723,10 @@ extension RunnerTests {
 
   private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
 #if os(tvOS)
-    return .unsupported("coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
+    return .unsupported(
+      message: "coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
+      hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
+    )
 #else
     interactionCoordinate(app: app, x: x, y: y).doubleTap()
     return .performed
@@ -1637,7 +1735,10 @@ extension RunnerTests {
 
   private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome {
 #if os(tvOS)
-    return .unsupported("coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element")
+    return .unsupported(
+      message: "coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element",
+      hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then long-select it."
+    )
 #else
     interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration)
     return .performed
@@ -1653,7 +1754,10 @@ extension RunnerTests {
     holdDuration: TimeInterval
   ) -> RunnerInteractionOutcome {
 #if os(tvOS)
-    return .unsupported("coordinate drag is not supported on tvOS")
+    return .unsupported(
+      message: "coordinate drag is not supported on tvOS",
+      hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead."
+    )
 #else
     let start = interactionCoordinate(app: app, x: x, y: y)
     let end = interactionCoordinate(app: app, x: x2, y: y2)

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -216,14 +216,14 @@ extension RunnerTests {
   // MARK: - Command Classification
 
   func isReadOnlyCommand(_ command: Command) -> Bool {
-    switch command.command {
-    case .interactionFrame, .findText, .readText, .snapshot, .screenshot:
+    switch command.command.traits.readOnly {
+    case .always:
       return true
-    case .alert:
-      let action = (command.action ?? "get").lowercased()
-      return action == "get"
-    default:
+    case .never:
       return false
+    case .conditional:
+      // Today only `alert` is conditional: read-only when getting, mutating otherwise.
+      return (command.action ?? "get").lowercased() == "get"
     }
   }
 
@@ -234,36 +234,11 @@ extension RunnerTests {
   }
 
   func isInteractionCommand(_ command: CommandType) -> Bool {
-    switch command {
-    case
-      .tap,
-      .longPress,
-      .drag,
-      .remotePress,
-      .type,
-      .swipe,
-      .back,
-      .backInApp,
-      .backSystem,
-      .rotate,
-      .appSwitcher,
-      .keyboardDismiss,
-      .pinch,
-      .rotateGesture,
-      .transformGesture:
-      return true
-    default:
-      return false
-    }
+    return command.traits.isInteraction
   }
 
   func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
-    switch command {
-    case .shutdown, .recordStop, .screenshot, .uptime:
-      return true
-    default:
-      return false
-    }
+    return command.traits.isLifecycle
   }
 
   // MARK: - Interaction Stabilization