agent-device 0.16.13 → 0.17.0

package/dist/src/snapshot.js

@@ -1,2 +1,2 @@
-import{__webpack_require__ as e}from"./rslib-runtime.js";import{promises as t}from"node:fs";import a from"node:os";import n from"node:path";import{centerOfRect as s}from"./4057.js";import{normalizeError as o,AppError as r}from"./9152.js";import{withDiagnosticTimer as i,emitDiagnostic as l}from"./7599.js";import{pressAndroid as d,backAndroid as p}from"./input-actions.js";import{createDaemonRuntimeSessionStore as u,captureSnapshot as c,dispatchCommand as f,resolveSnapshotScope as m,runMacOsAlertAction as h,errorResponse as w,runIosRunnerCommand as _,isCommandSupportedOnDevice as g,snapshotAndroid as y,annotateScreenshotWithRefs as v,context_contextFromFlags as S}from"./2415.js";import{successText as b}from"./1998.js";import{sleep as I}from"./4829.js";import{buildSnapshotSession as A,dispatchWaitViaRuntime as C,createDaemonRuntimePolicy as R,recordIfSession as N,resolveSessionDevice as M,withSessionlessRunnerCleanup as D}from"./selector-runtime.js";import{parseTimeout as P}from"./6085.js";import{SETTINGS_INVALID_ARGS_MESSAGE as k,isMacOsSettingSupported as E,getUnsupportedMacOsSettingMessage as O}from"./1352.js";import{createAgentDevice as K}from"./9533.js";import{DAEMON_COMMAND_GROUPS as U}from"./5792.js";var x={};e.r(x),e.d(x,{SNAPSHOT_COMMAND_HANDLERS:()=>eh,handleSnapshotCommands:()=>ew});let H=new Set(["com.android.permissioncontroller","com.google.android.permissioncontroller","com.google.android.packageinstaller","com.android.packageinstaller"]),L=new Set(["android","com.android.systemui"]),$=/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i,B=/^android:id\/button[123]$/i,T=/(?:^|:)id\/permission_/i,q=/\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i,F=/^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i,j=/^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i;function z(e,t){let a=t[0];if(!a)return;let n=G(e,a.index);for(let a of t.slice(1)){let t=new Set(G(e,a.index));for(let e=n.length-1;e>=0;e-=1)t.has(n[e])||n.splice(e,1)}return n[n.length-1]}function G(e,t){let a=new Map(e.map(e=>[e.index,e])),n=[],s=a.get(t);for(;s;)n.push(s.index),s=void 0===s.parentIndex?void 0:a.get(s.parentIndex);return n.reverse()}function V(e,t){let a=new Map;for(let t of e){if(void 0===t.parentIndex)continue;let e=a.get(t.parentIndex)??[];e.push(t),a.set(t.parentIndex,e)}let n=new Set([t]),s=[t];for(let e of s)for(let t of a.get(e)??[])n.has(t.index)||(n.add(t.index),s.push(t.index));return e.filter(e=>n.has(e.index))}function W(e){let t=e.type??"",a=e.identifier??"";return!!(e.hittable||/\bbutton\b/i.test(t)||B.test(a)||/(?:^|:)id\/permission_(?:allow|deny)/i.test(a))}function J(e){if(!e)return"";let t=[e.label,e.value].filter(e=>"string"==typeof e&&e.trim().length>0);return t[0]?.trim()??""}async function Q(e,t,a={}){if("wait"===t)return await X(e,a.timeoutMs??1e4);if("get"===t){var n;let t=await ee(e);return{kind:"alertStatus",platform:"android",action:"get",alert:n=t?.alert??null,...n?b("Alert visible"):b("No alert visible")}}return await Y(e,t)}async function X(e,t){let a=Date.now(),n=await Z(e,t);if(!n)throw new r("COMMAND_FAILED","alert wait timed out");return{kind:"alertWait",platform:"android",action:"wait",alert:n.alert,waitedMs:Date.now()-a,...b("Alert visible")}}async function Y(e,t){var a;let n,s,o=await Z(e,2e3);if(!o)throw new r("COMMAND_FAILED","alert not found",{hint:"If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref."});let i=(a=o.buttons,n="accept"===t?"accept":"dismiss",(s=a.find(e=>e.role===n))||("dismiss"===t?a.find(e=>"neutral"===e.role)??null:1===a.length?a[0]??null:null));if(i)return await d(e,i.x,i.y),et(t,o.alert,i.label);if("dismiss"===t)return await p(e),et(t,o.alert,"Back");throw new r("COMMAND_FAILED","alert accept found an alert but no accept button",{alert:o.alert,hint:"Inspect alert get --json for visible buttons, then use press by visible label/ref if needed."})}async function Z(e,t){let a=Date.now();for(;Date.now()-a<t;){let t=await ee(e);if(t)return t;await I(300)}return null}async function ee(e){return function(e){var t,a,n,o;let r,i=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",H.has(a)||T.test(t.identifier??"")});if(t.length)return{nodes:t,source:"permission"};let a=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",L.has(a)&&q.test(J(t))});if(0===t.length)return[];let a=z(e,t);return void 0===a?t:V(e,a).filter(e=>e.bundleId&&L.has(e.bundleId))}(e);return a.length?{nodes:a,source:"system-dialog"}:{nodes:function(e){var t;let a,n,s=e.filter(e=>{var t;return t=e.type??"",/(?:^|[.$])[^.]*Dialog$/i.test(t)}),o=e.filter(e=>$.test(e.identifier??"")),r=s.length?[...s,...o]:(a=(t=o).some(e=>B.test(e.identifier??"")),n=t.some(e=>!B.test(e.identifier??"")),a&&n?t:[]);if(0===r.length)return[];let i=z(e,r);return void 0===i?r:V(e,i)}(e),source:"native-dialog"}}(e),l=i.nodes;if(0===l.length)return null;let d=function(e){let t=new Set,a=[];for(let n of e){let e=J(n);if(!e||!n.rect||!W(n))continue;let o=e.trim().toLowerCase();if(!o||t.has(o))continue;t.add(o);let r=s(n.rect);a.push({label:e,x:r.x,y:r.y,role:function(e,t){var a;let n=(a=e.identifier??"",/(?:^|:)id\/button1$/i.test(a)?"accept":/(?:^|:)id\/button2$/i.test(a)?"dismiss":/(?:^|:)id\/button3$/i.test(a)?"neutral":/(?:^|:)id\/permission_allow/i.test(a)?"accept":/(?:^|:)id\/permission_deny/i.test(a)?"dismiss":null);return n||(F.test(t.trim())?"accept":j.test(t.trim())?"dismiss":"neutral")}(n,e)})}return a}(l),p=l.filter(e=>J(e)&&!W(e)),u=J((t=p).find(e=>/(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(e.identifier??"")))||J(t[0]),c=(a=p,n=u,(r=a.map(e=>J(e)).filter(e=>e&&e!==n)).length?[...new Set(r)].join("\n"):void 0),f=(o=l,o.find(e=>e.bundleId)?.bundleId);return{alert:{...u?{title:u}:{},...c?{message:c}:{},buttons:d.map(e=>e.label),platform:"android",source:i.source,...f?{packageName:f}:{}},buttons:d}}((await i("snapshot_capture",async()=>await y(e,{helperWaitForIdleTimeoutMs:0,includeHiddenContentHints:!1}),{backend:"android",purpose:"alert"})).nodes)}function et(e,t,a){return{kind:"alertHandled",platform:"android",action:e,handled:!0,alert:t,button:a,...b(`Alert ${e}ed`)}}async function ea(e){var t;let{req:a,logPath:n,session:s,device:o}=e,r="accept"===(t=a.positionals?.[0])||"dismiss"===t||"wait"===t?t:"get",i=s?"frontmost-app"===s.surface?{surface:"frontmost-app"}:{bundleId:s.appBundleId,surface:s.surface}:{};if(!g("alert",o))return w("UNSUPPORTED_OPERATION","alert is not supported on this device");if("android"===o.platform){let t=P(a.positionals?.[1])??1e4;return er(e,await Q(o,r,{timeoutMs:t}))}if("macos"===o.platform){let t=async e=>await h(e,i);return await en(e,r,t)}let l={verbose:a.flags?.verbose,logPath:n,traceLogPath:s?.trace?.outPath,requestId:a.meta?.requestId},d=async e=>await _(o,{command:"alert",action:e,appBundleId:s?.appBundleId},l);return await en(e,r,d)}async function en(e,t,a){if("wait"===t)return await es(e,a);let n="accept"===t||"dismiss"===t?t:"get";return"accept"===n||"dismiss"===n?await eo(e,n,a):er(e,await a("get"))}async function es(e,t){let a=P(e.req.positionals?.[1])??1e4,n=Date.now();for(;Date.now()-n<a;){try{return er(e,await t("get"))}catch{}await I(300)}return w("COMMAND_FAILED","alert wait timed out")}async function eo(e,t,a){var n,s;let o,i,l=Date.now();for(;Date.now()-l<2e3;){try{return er(e,await a(t))}catch(t){i=t;let e=String(t?.message??"").toLowerCase();if(!e.includes("alert not found")&&!e.includes("no alert"))break}await I(300)}throw(n=i)instanceof r&&(s=n,(o=String(s?.message??"").toLowerCase()).includes("alert not found")||o.includes("no alert"))?new r(n.code,n.message,{...n.details??{},hint:"If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref."}):n}function er(e,t){return N(e.sessionStore,e.session,e.req,t),{ok:!0,data:t}}async function ei(e){let{req:t,logPath:a,sessionStore:n,session:s,device:o,parsed:r}=e,{setting:i,state:l,appBundleId:d,permissionTarget:p,latitude:u,longitude:c}=r;if(!g("settings",o))return w("UNSUPPORTED_OPERATION","settings is not supported on this device");if("macos"===o.platform&&!E(i))return w("INVALID_ARGS",O(i));let m=d??s?.appBundleId;if("clear-app-state"===i&&!m)return w("INVALID_ARGS","settings clear-app-state requires an app id when no app is bound to the session");let h="clear-app-state"===i?[i,l,m??""]:"permission"===i?[i,l,p??"",t.positionals?.[3]??"",m??""]:"location"===i&&"set"===l?[i,l,u??"",c??"",m??""]:[i,l,m??""],_=await f(o,"settings",h,t.flags?.out,{...S(a,t.flags,m,s?.trace?.outPath)});return N(n,s,t,_??{setting:i,state:l}),{ok:!0,data:_??{setting:i,state:l}}}async function el(e){var t,a;let n;if("snapshot"!==e.command||"android"!==e.device.platform)return;let s=o(e.error);if("COMMAND_FAILED"===(t=s).code&&(a=t,n=`${a.message}
-${a.hint??""}`,/Android UI hierarchy dump timed out/i.test(n)||/Stock UIAutomator fallback was skipped/i.test(n)||/Android accessibility snapshots can be blocked/i.test(n)||function(e){if(!e||"object"!=typeof e)return!1;let t=String(e.errorType??""),a=String(e.message??"");return/TimeoutException/i.test(t)||/timed out/i.test(a)}(t.details?.helper)||function(e){var t;if(!e)return!1;let a=e?.timeoutMs,n=e?.cmd,s=Array.isArray(t=e?.args)?t.map(String):"string"==typeof t?t.split(/\s+/):[];return"number"==typeof a&&"adb"===n&&s.includes("uiautomator")&&s.includes("dump")}(t.details)))return{ok:!1,error:{...s,details:{...s.details??{},androidSnapshotTimeoutScreenshot:await ed(e)}}}}async function ed(e){try{var s,r,i;let o=await t.mkdtemp(n.join(a.tmpdir(),"agent-device-android-snapshot-timeout-")),d=n.join(o,"snapshot-timeout-overlay-refs.png"),p=(s=await f(e.device,"screenshot",[d],void 0,{...S(e.logPath,{screenshotNoStabilize:!0},e.session?.appBundleId,e.session?.trace?.outPath),surface:e.session?.surface}),r=d,(i=s,"object"==typeof i&&null!==i&&"path"in i&&"string"==typeof i.path)?s.path:r);await t.access(p);let u=await ep(p,e.session);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_captured",data:{path:p,overlayRefCount:"overlayRefCount"in u?u.overlayRefCount:void 0,overlayRefsAnnotated:"overlayRefsAnnotated"in u?u.overlayRefsAnnotated:void 0}}),u}catch(t){let e=o(t);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_failed",data:{error:e.message}}),{captureFailed:!0,error:e.message}}}async function ep(e,t){if(!t?.snapshot)return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"unavailable",overlayRefCount:0};try{let a=await v({screenshotPath:e,snapshot:t.snapshot});return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:a.length>0,overlayRefCount:a.length,overlayRefSource:"session-snapshot",overlayRefs:a}}catch(a){let t=o(a);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_overlay_failed",data:{path:e,error:t.message}}),{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"session-snapshot",overlayRefCount:0,overlayAnnotationError:t.message}}}async function eu(e){return await ef({...e,command:"snapshot",unsupportedMessage:"snapshot is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.snapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,compact:a.flags?.snapshotCompact,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw,forceFull:a.flags?.snapshotForceFull});return{data:s,record:{kind:"snapshot",nodes:s.nodes.length,truncated:s.truncated}}}})}async function ec(e){return await ef({...e,command:"diff",unsupportedMessage:"diff is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.diffSnapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,compact:a.flags?.snapshotCompact,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw});return{data:s,record:{kind:"diff",mode:"snapshot",baselineInitialized:s.baselineInitialized,summary:s.summary}}}})}async function ef(e){var t,a;let{req:n,sessionName:s,logPath:o,sessionStore:i}=e,{session:l,device:d}=await M(i,s,n.flags);if(!g(e.command,d))return w("UNSUPPORTED_OPERATION",e.unsupportedMessage);let p=m(n.flags?.snapshotScope,l);if(!p.ok)return p;let f=(t=e.command,a=l,"ios"!==d.platform||a?.appBundleId?null:w("SESSION_NOT_FOUND",`iOS ${t} requires an active app session on the target device. Run open first (for example: open --session ${a?.name??"sim"} --platform ios --device "<name>" <app>).`));return f||await D(l,d,async()=>{var t,a;let f,m,h=function(e){let{req:t,sessionName:a,logPath:n,sessionStore:s,session:o,device:i,snapshotScope:l}=e;return K({backend:function(e){let{req:t,logPath:a,session:n,device:s,snapshotScope:o}=e;return{platform:s.platform,captureSnapshot:async(e,r)=>{let i=await c({device:s,session:n,flags:t.flags,outPath:r?.outPath??t.flags?.out,logPath:a,snapshotScope:o});return{snapshot:i.snapshot,analysis:i.analysis,androidSnapshot:i.androidSnapshot,freshness:i.freshness,appName:n?.appBundleId?n.appName??n.appBundleId:void 0,appBundleId:n?.appBundleId}}}}({req:t,logPath:n,session:o,device:i,snapshotScope:l}),...R("snapshot"),sessions:u({sessionName:a,getSession:()=>s.get(a),recordOptions:{includeSnapshot:!0},setRecord:e=>{var n;let o=function(e){if(!e.snapshot)throw new r("UNKNOWN","snapshot runtime did not produce session state");return e}(e),l=s.get(a);s.set(a,function(e){var t,a;let{current:n,sessionName:s,device:o,record:r,refScopedSnapshot:i}=e,l=(t=n,a=r,i&&a.snapshot?.nodes.length===0&&t?.snapshot!==void 0),d=l?n.snapshot:r.snapshot,p=A({session:n,sessionName:s,device:o,snapshot:d,appBundleId:r.appBundleId});return p.snapshotScopeSource=function(e){let{current:t,keepCurrentSnapshot:a,refScopedSnapshot:n}=e;if(n)return a?t?.snapshotScopeSource:t?.snapshotScopeSource??t?.snapshot}({current:n,keepCurrentSnapshot:l,refScopedSnapshot:i}),r.appName&&(p.appName=r.appName),p}({current:l,sessionName:a,device:i,record:o,refScopedSnapshot:(n=t,n.flags?.snapshotScope?.trim().startsWith("@")===!0)}))}})})}({req:n,sessionName:s,logPath:o,sessionStore:i,session:l,device:d,snapshotScope:p.scope});try{m=await e.execute({runtime:h,sessionName:s,req:n,snapshotScope:p.scope})}catch(a){let t=await el({error:a,command:e.command,logPath:o,session:l,device:d});if(!t)throw a;return t}return(f=(t={req:n,sessionName:s,sessionStore:i,result:m.record}).sessionStore.get(t.sessionName))&&t.sessionStore.recordAction(f,{command:t.req.command,positionals:t.req.positionals??[],flags:t.req.flags??{},result:"snapshot"===(a=t.result).kind?{nodes:a.nodes,truncated:a.truncated}:{mode:a.mode,baselineInitialized:a.baselineInitialized,summary:a.summary}}),{ok:!0,data:m.data}})}let em=U.snapshot,eh={snapshot:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await eu({req:e,sessionName:t,logPath:a,sessionStore:n}),diff:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>e.positionals?.[0]!=="snapshot"?w("INVALID_ARGS","diff currently supports only: diff snapshot"):await ec({req:e,sessionName:t,logPath:a,sessionStore:n}),wait:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await C({req:e,sessionName:t,logPath:a,sessionStore:n}),alert:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let{session:s,device:o}=await M(n,t,e.flags);return await D(s,o,async()=>await ea({req:e,logPath:a,sessionStore:n,session:s,device:o}))},settings:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let s,o,r,i=(s=e.positionals?.[0]?.toLowerCase(),o=e.positionals?.[1]?.toLowerCase(),r=e.positionals?.[2]?.toLowerCase(),"clear-app-state"===s?{ok:!0,parsed:{setting:s,state:"clear",appBundleId:"clear"===o?e.positionals?.[2]:e.positionals?.[1]}}:s&&o&&("permission"!==s||r)&&("location"!==s||"set"!==o||e.positionals?.[2]&&e.positionals?.[3])?{ok:!0,parsed:{setting:s,state:o,permissionTarget:r,latitude:e.positionals?.[2],longitude:e.positionals?.[3]}}:w("INVALID_ARGS",k));if(!i.ok)return i;let{session:l,device:d}=await M(n,t,e.flags);return await D(l,d,async()=>await ei({req:e,logPath:a,sessionStore:n,session:l,device:d,parsed:i.parsed}))}};async function ew(e){let t=e.req.command;if(!em.has(t))return null;let a=eh[t];return a?await a(e):w("COMMAND_FAILED",`Snapshot command has no handler: ${t}`)}export{x as snapshot_namespaceObject};
+import{__webpack_require__ as e}from"./rslib-runtime.js";import{promises as t}from"node:fs";import a from"node:os";import n from"node:path";import{centerOfRect as s}from"./4057.js";import{normalizeError as o,AppError as r}from"./9152.js";import{withDiagnosticTimer as i,emitDiagnostic as l}from"./7599.js";import{pressAndroid as d,backAndroid as p}from"./input-actions.js";import{createDaemonRuntimeSessionStore as u,captureSnapshot as c,dispatchCommand as f,resolveSnapshotScope as m,runMacOsAlertAction as h,errorResponse as w,runIosRunnerCommand as g,isCommandSupportedOnDevice as y,snapshotAndroid as _,annotateScreenshotWithRefs as v,context_contextFromFlags as S}from"./2415.js";import{successText as b}from"./1998.js";import{sleep as I}from"./4829.js";import{buildSnapshotSession as A,dispatchWaitViaRuntime as R,createDaemonRuntimePolicy as C,recordIfSession as N,resolveSessionDevice as M,withSessionlessRunnerCleanup as P}from"./selector-runtime.js";import{parseTimeout as k}from"./6085.js";import{SETTINGS_INVALID_ARGS_MESSAGE as D,isMacOsSettingSupported as E,getUnsupportedMacOsSettingMessage as K}from"./1352.js";import{createAgentDevice as O}from"./9533.js";var U={};e.r(U),e.d(U,{handleSnapshotCommands:()=>em});let x=new Set(["com.android.permissioncontroller","com.google.android.permissioncontroller","com.google.android.packageinstaller","com.android.packageinstaller"]),L=new Set(["android","com.android.systemui"]),B=/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i,H=/^android:id\/button[123]$/i,$=/(?:^|:)id\/permission_/i,q=/\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i,T=/^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i,F=/^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i;function j(e,t){let a=t[0];if(!a)return;let n=z(e,a.index);for(let a of t.slice(1)){let t=new Set(z(e,a.index));for(let e=n.length-1;e>=0;e-=1)t.has(n[e])||n.splice(e,1)}return n[n.length-1]}function z(e,t){let a=new Map(e.map(e=>[e.index,e])),n=[],s=a.get(t);for(;s;)n.push(s.index),s=void 0===s.parentIndex?void 0:a.get(s.parentIndex);return n.reverse()}function G(e,t){let a=new Map;for(let t of e){if(void 0===t.parentIndex)continue;let e=a.get(t.parentIndex)??[];e.push(t),a.set(t.parentIndex,e)}let n=new Set([t]),s=[t];for(let e of s)for(let t of a.get(e)??[])n.has(t.index)||(n.add(t.index),s.push(t.index));return e.filter(e=>n.has(e.index))}function V(e){let t=e.type??"",a=e.identifier??"";return!!(e.hittable||/\bbutton\b/i.test(t)||H.test(a)||/(?:^|:)id\/permission_(?:allow|deny)/i.test(a))}function W(e){if(!e)return"";let t=[e.label,e.value].filter(e=>"string"==typeof e&&e.trim().length>0);return t[0]?.trim()??""}async function J(e,t,a={}){if("wait"===t)return await Q(e,a.timeoutMs??1e4);if("get"===t){var n;let t=await Z(e);return{kind:"alertStatus",platform:"android",action:"get",alert:n=t?.alert??null,...n?b("Alert visible"):b("No alert visible")}}return await X(e,t)}async function Q(e,t){let a=Date.now(),n=await Y(e,t);if(!n)throw new r("COMMAND_FAILED","alert wait timed out");return{kind:"alertWait",platform:"android",action:"wait",alert:n.alert,waitedMs:Date.now()-a,...b("Alert visible")}}async function X(e,t){var a;let n,s,o=await Y(e,2e3);if(!o)throw new r("COMMAND_FAILED","alert not found",{hint:"If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref."});let i=(a=o.buttons,n="accept"===t?"accept":"dismiss",(s=a.find(e=>e.role===n))||("dismiss"===t?a.find(e=>"neutral"===e.role)??null:1===a.length?a[0]??null:null));if(i)return await d(e,i.x,i.y),ee(t,o.alert,i.label);if("dismiss"===t)return await p(e),ee(t,o.alert,"Back");throw new r("COMMAND_FAILED","alert accept found an alert but no accept button",{alert:o.alert,hint:"Inspect alert get --json for visible buttons, then use press by visible label/ref if needed."})}async function Y(e,t){let a=Date.now();for(;Date.now()-a<t;){let t=await Z(e);if(t)return t;await I(300)}return null}async function Z(e){return function(e){var t,a,n,o;let r,i=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",x.has(a)||$.test(t.identifier??"")});if(t.length)return{nodes:t,source:"permission"};let a=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",L.has(a)&&q.test(W(t))});if(0===t.length)return[];let a=j(e,t);return void 0===a?t:G(e,a).filter(e=>e.bundleId&&L.has(e.bundleId))}(e);return a.length?{nodes:a,source:"system-dialog"}:{nodes:function(e){var t;let a,n,s=e.filter(e=>{var t;return t=e.type??"",/(?:^|[.$])[^.]*Dialog$/i.test(t)}),o=e.filter(e=>B.test(e.identifier??"")),r=s.length?[...s,...o]:(a=(t=o).some(e=>H.test(e.identifier??"")),n=t.some(e=>!H.test(e.identifier??"")),a&&n?t:[]);if(0===r.length)return[];let i=j(e,r);return void 0===i?r:G(e,i)}(e),source:"native-dialog"}}(e),l=i.nodes;if(0===l.length)return null;let d=function(e){let t=new Set,a=[];for(let n of e){let e=W(n);if(!e||!n.rect||!V(n))continue;let o=e.trim().toLowerCase();if(!o||t.has(o))continue;t.add(o);let r=s(n.rect);a.push({label:e,x:r.x,y:r.y,role:function(e,t){var a;let n=(a=e.identifier??"",/(?:^|:)id\/button1$/i.test(a)?"accept":/(?:^|:)id\/button2$/i.test(a)?"dismiss":/(?:^|:)id\/button3$/i.test(a)?"neutral":/(?:^|:)id\/permission_allow/i.test(a)?"accept":/(?:^|:)id\/permission_deny/i.test(a)?"dismiss":null);return n||(T.test(t.trim())?"accept":F.test(t.trim())?"dismiss":"neutral")}(n,e)})}return a}(l),p=l.filter(e=>W(e)&&!V(e)),u=W((t=p).find(e=>/(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(e.identifier??"")))||W(t[0]),c=(a=p,n=u,(r=a.map(e=>W(e)).filter(e=>e&&e!==n)).length?[...new Set(r)].join("\n"):void 0),f=(o=l,o.find(e=>e.bundleId)?.bundleId);return{alert:{...u?{title:u}:{},...c?{message:c}:{},buttons:d.map(e=>e.label),platform:"android",source:i.source,...f?{packageName:f}:{}},buttons:d}}((await i("snapshot_capture",async()=>await _(e,{helperWaitForIdleTimeoutMs:0,includeHiddenContentHints:!1}),{backend:"android",purpose:"alert"})).nodes)}function ee(e,t,a){return{kind:"alertHandled",platform:"android",action:e,handled:!0,alert:t,button:a,...b(`Alert ${e}ed`)}}async function et(e){var t;let{req:a,logPath:n,session:s,device:o}=e,r="accept"===(t=a.positionals?.[0])||"dismiss"===t||"wait"===t?t:"get",i=s?"frontmost-app"===s.surface?{surface:"frontmost-app"}:{bundleId:s.appBundleId,surface:s.surface}:{};if(!y("alert",o))return w("UNSUPPORTED_OPERATION","alert is not supported on this device");if("android"===o.platform){let t=k(a.positionals?.[1])??1e4;return eo(e,await J(o,r,{timeoutMs:t}))}if("macos"===o.platform){let t=async e=>await h(e,i);return await ea(e,r,t)}let l={verbose:a.flags?.verbose,logPath:n,traceLogPath:s?.trace?.outPath,requestId:a.meta?.requestId},d=async e=>await g(o,{command:"alert",action:e,appBundleId:s?.appBundleId},l);return await ea(e,r,d)}async function ea(e,t,a){if("wait"===t)return await en(e,a);let n="accept"===t||"dismiss"===t?t:"get";return"accept"===n||"dismiss"===n?await es(e,n,a):eo(e,await a("get"))}async function en(e,t){let a=k(e.req.positionals?.[1])??1e4,n=Date.now();for(;Date.now()-n<a;){try{return eo(e,await t("get"))}catch{}await I(300)}return w("COMMAND_FAILED","alert wait timed out")}async function es(e,t,a){var n,s;let o,i,l=Date.now();for(;Date.now()-l<2e3;){try{return eo(e,await a(t))}catch(t){i=t;let e=String(t?.message??"").toLowerCase();if(!e.includes("alert not found")&&!e.includes("no alert"))break}await I(300)}throw(n=i)instanceof r&&(s=n,(o=String(s?.message??"").toLowerCase()).includes("alert not found")||o.includes("no alert"))?new r(n.code,n.message,{...n.details??{},hint:"If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref."}):n}function eo(e,t){return N(e.sessionStore,e.session,e.req,t),{ok:!0,data:t}}async function er(e){let{req:t,logPath:a,sessionStore:n,session:s,device:o,parsed:r}=e,{setting:i,state:l,appBundleId:d,permissionTarget:p,latitude:u,longitude:c}=r;if(!y("settings",o))return w("UNSUPPORTED_OPERATION","settings is not supported on this device");if("macos"===o.platform&&!E(i))return w("INVALID_ARGS",K(i));let m=d??s?.appBundleId;if("clear-app-state"===i&&!m)return w("INVALID_ARGS","settings clear-app-state requires an app id when no app is bound to the session");let h="clear-app-state"===i?[i,l,m??""]:"permission"===i?[i,l,p??"",t.positionals?.[3]??"",m??""]:"location"===i&&"set"===l?[i,l,u??"",c??"",m??""]:[i,l,m??""],g=await f(o,"settings",h,t.flags?.out,{...S(a,t.flags,m,s?.trace?.outPath)});return N(n,s,t,g??{setting:i,state:l}),{ok:!0,data:g??{setting:i,state:l}}}async function ei(e){var t,a;let n;if("snapshot"!==e.command||"android"!==e.device.platform)return;let s=o(e.error);if("COMMAND_FAILED"===(t=s).code&&(a=t,n=`${a.message}
+${a.hint??""}`,/Android UI hierarchy dump timed out/i.test(n)||/Stock UIAutomator fallback was skipped/i.test(n)||/Android accessibility snapshots can be blocked/i.test(n)||function(e){if(!e||"object"!=typeof e)return!1;let t=String(e.errorType??""),a=String(e.message??"");return/TimeoutException/i.test(t)||/timed out/i.test(a)}(t.details?.helper)||function(e){var t;if(!e)return!1;let a=e?.timeoutMs,n=e?.cmd,s=Array.isArray(t=e?.args)?t.map(String):"string"==typeof t?t.split(/\s+/):[];return"number"==typeof a&&"adb"===n&&s.includes("uiautomator")&&s.includes("dump")}(t.details)))return{ok:!1,error:{...s,details:{...s.details??{},androidSnapshotTimeoutScreenshot:await el(e)}}}}async function el(e){try{var s,r,i;let o=await t.mkdtemp(n.join(a.tmpdir(),"agent-device-android-snapshot-timeout-")),d=n.join(o,"snapshot-timeout-overlay-refs.png"),p=(s=await f(e.device,"screenshot",[d],void 0,{...S(e.logPath,{screenshotNoStabilize:!0},e.session?.appBundleId,e.session?.trace?.outPath),surface:e.session?.surface}),r=d,(i=s,"object"==typeof i&&null!==i&&"path"in i&&"string"==typeof i.path)?s.path:r);await t.access(p);let u=await ed(p,e.session);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_captured",data:{path:p,overlayRefCount:"overlayRefCount"in u?u.overlayRefCount:void 0,overlayRefsAnnotated:"overlayRefsAnnotated"in u?u.overlayRefsAnnotated:void 0}}),u}catch(t){let e=o(t);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_failed",data:{error:e.message}}),{captureFailed:!0,error:e.message}}}async function ed(e,t){if(!t?.snapshot)return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"unavailable",overlayRefCount:0};try{let a=await v({screenshotPath:e,snapshot:t.snapshot});return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:a.length>0,overlayRefCount:a.length,overlayRefSource:"session-snapshot",overlayRefs:a}}catch(a){let t=o(a);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_overlay_failed",data:{path:e,error:t.message}}),{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"session-snapshot",overlayRefCount:0,overlayAnnotationError:t.message}}}async function ep(e){return await ec({...e,command:"snapshot",unsupportedMessage:"snapshot is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.snapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,compact:a.flags?.snapshotCompact,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw,forceFull:a.flags?.snapshotForceFull});return{data:s,record:{kind:"snapshot",nodes:s.nodes.length,truncated:s.truncated}}}})}async function eu(e){return await ec({...e,command:"diff",unsupportedMessage:"diff is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.diffSnapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,compact:a.flags?.snapshotCompact,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw});return{data:s,record:{kind:"diff",mode:"snapshot",baselineInitialized:s.baselineInitialized,summary:s.summary}}}})}async function ec(e){var t,a;let{req:n,sessionName:s,logPath:o,sessionStore:i}=e,{session:l,device:d}=await M(i,s,n.flags);if(!y(e.command,d))return w("UNSUPPORTED_OPERATION",e.unsupportedMessage);let p=m(n.flags?.snapshotScope,l);if(!p.ok)return p;let f=(t=e.command,a=l,"ios"!==d.platform||a?.appBundleId?null:w("SESSION_NOT_FOUND",`iOS ${t} requires an active app session on the target device. Run open first (for example: open --session ${a?.name??"sim"} --platform ios --device "<name>" <app>).`));return f||await P(l,d,async()=>{var t,a;let f,m,h=function(e){let{req:t,sessionName:a,logPath:n,sessionStore:s,session:o,device:i,snapshotScope:l}=e;return O({backend:function(e){let{req:t,logPath:a,session:n,device:s,snapshotScope:o}=e;return{platform:s.platform,captureSnapshot:async(e,r)=>{let i=await c({device:s,session:n,flags:t.flags,outPath:r?.outPath??t.flags?.out,logPath:a,snapshotScope:o});return{snapshot:i.snapshot,analysis:i.analysis,androidSnapshot:i.androidSnapshot,freshness:i.freshness,appName:n?.appBundleId?n.appName??n.appBundleId:void 0,appBundleId:n?.appBundleId}}}}({req:t,logPath:n,session:o,device:i,snapshotScope:l}),...C("snapshot"),sessions:u({sessionName:a,getSession:()=>s.get(a),recordOptions:{includeSnapshot:!0},setRecord:e=>{var n;let o=function(e){if(!e.snapshot)throw new r("UNKNOWN","snapshot runtime did not produce session state");return e}(e),l=s.get(a);s.set(a,function(e){var t,a;let{current:n,sessionName:s,device:o,record:r,refScopedSnapshot:i}=e,l=(t=n,a=r,i&&a.snapshot?.nodes.length===0&&t?.snapshot!==void 0),d=l?n.snapshot:r.snapshot,p=A({session:n,sessionName:s,device:o,snapshot:d,appBundleId:r.appBundleId});return p.snapshotScopeSource=function(e){let{current:t,keepCurrentSnapshot:a,refScopedSnapshot:n}=e;if(n)return a?t?.snapshotScopeSource:t?.snapshotScopeSource??t?.snapshot}({current:n,keepCurrentSnapshot:l,refScopedSnapshot:i}),r.appName&&(p.appName=r.appName),p}({current:l,sessionName:a,device:i,record:o,refScopedSnapshot:(n=t,n.flags?.snapshotScope?.trim().startsWith("@")===!0)}))}})})}({req:n,sessionName:s,logPath:o,sessionStore:i,session:l,device:d,snapshotScope:p.scope});try{m=await e.execute({runtime:h,sessionName:s,req:n,snapshotScope:p.scope})}catch(a){let t=await ei({error:a,command:e.command,logPath:o,session:l,device:d});if(!t)throw a;return t}return(f=(t={req:n,sessionName:s,sessionStore:i,result:m.record}).sessionStore.get(t.sessionName))&&t.sessionStore.recordAction(f,{command:t.req.command,positionals:t.req.positionals??[],flags:t.req.flags??{},result:"snapshot"===(a=t.result).kind?{nodes:a.nodes,truncated:a.truncated}:{mode:a.mode,baselineInitialized:a.baselineInitialized,summary:a.summary}}),{ok:!0,data:m.data}})}let ef={snapshot:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await ep({req:e,sessionName:t,logPath:a,sessionStore:n}),diff:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>e.positionals?.[0]!=="snapshot"?w("INVALID_ARGS","diff currently supports only: diff snapshot"):await eu({req:e,sessionName:t,logPath:a,sessionStore:n}),wait:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await R({req:e,sessionName:t,logPath:a,sessionStore:n}),alert:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let{session:s,device:o}=await M(n,t,e.flags);return await P(s,o,async()=>await et({req:e,logPath:a,sessionStore:n,session:s,device:o}))},settings:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let s,o,r,i=(s=e.positionals?.[0]?.toLowerCase(),o=e.positionals?.[1]?.toLowerCase(),r=e.positionals?.[2]?.toLowerCase(),"clear-app-state"===s?{ok:!0,parsed:{setting:s,state:"clear",appBundleId:"clear"===o?e.positionals?.[2]:e.positionals?.[1]}}:s&&o&&("permission"!==s||r)&&("location"!==s||"set"!==o||e.positionals?.[2]&&e.positionals?.[3])?{ok:!0,parsed:{setting:s,state:o,permissionTarget:r,latitude:e.positionals?.[2],longitude:e.positionals?.[3]}}:w("INVALID_ARGS",D));if(!i.ok)return i;let{session:l,device:d}=await M(n,t,e.flags);return await P(l,d,async()=>await er({req:e,logPath:a,sessionStore:n,session:l,device:d,parsed:i.parsed}))}};async function em(e){let t=ef[e.req.command];return t?await t(e):null}export{U as snapshot_namespaceObject};

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h

@@ -21,6 +21,10 @@ NS_ASSUME_NONNULL_BEGIN
                                                    y2:(double)y2
                                             durationMs:(double)durationMs;
 
++ (NSString * _Nullable)synthesizeTapWithApplication:(id)application
+                                                   x:(double)x
+                                                   y:(double)y;
+
 @end
 
 NS_ASSUME_NONNULL_END

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m

@@ -53,6 +53,10 @@ static id RunnerSwipePointerPath(
   CGPoint end,
   double durationMs
 );
+static id RunnerTapPointerPath(
+  const RunnerXCTestEventBridge *bridge,
+  CGPoint point
+);
 static CGPoint RunnerPointerPointAt(
   double x,
   double y,
@@ -115,6 +119,18 @@ static double RunnerSmoothStep(double t);
   }
 }
 
++ (NSString * _Nullable)synthesizeTapWithApplication:(id)application
+                                                   x:(double)x
+                                                   y:(double)y {
+  @try {
+    return [self trySynthesizeTapWithApplication:application x:x y:y];
+  } @catch (NSException *exception) {
+    NSString *name = exception.name ?: @"NSException";
+    NSString *reason = exception.reason ?: @"private XCTest event synthesis failed";
+    return [NSString stringWithFormat:@"%@: %@", name, reason];
+  }
+}
+
 + (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application
                                                             x:(double)x
                                                             y:(double)y
@@ -224,6 +240,48 @@ static double RunnerSmoothStep(double t);
   return nil;
 }
 
++ (NSString * _Nullable)trySynthesizeTapWithApplication:(id)application
+                                                      x:(double)x
+                                                      y:(double)y {
+  RunnerXCTestEventBridge bridge;
+  NSString *missing = RunnerResolveXCTestEventBridge(application, &bridge);
+  if (missing != nil) {
+    return missing;
+  }
+
+  NSInteger interfaceOrientation =
+    ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.interfaceOrientationSelector);
+  NSInteger targetProcessID = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.processIDSelector);
+  if (targetProcessID <= 0) {
+    return @"private XCTest event synthesis unavailable: could not resolve target process ID";
+  }
+
+  id record = ((RunnerMsgSendInitRecord)objc_msgSend)(
+    [bridge.recordClass alloc],
+    bridge.initRecordSelector,
+    @"agent-device-tap",
+    interfaceOrientation
+  );
+  if (record == nil) {
+    return @"private XCTest event synthesis failed: could not create event record";
+  }
+  ((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.setTargetProcessIDSelector, targetProcessID);
+
+  id path = RunnerTapPointerPath(&bridge, CGPointMake(x, y));
+  if (path == nil) {
+    return @"private XCTest event synthesis failed: could not create pointer path";
+  }
+  ((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.addPathSelector, path);
+
+  NSError *error = nil;
+  BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, bridge.synthesizeSelector, &error);
+  if (!ok) {
+    NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false";
+    return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail];
+  }
+  return nil;
+}
+
 static NSString * _Nullable RunnerResolveXCTestEventBridge(
   id application,
   RunnerXCTestEventBridge *bridge
@@ -368,6 +426,19 @@ static id RunnerSwipePointerPath(
   return path;
 }
 
+static id RunnerTapPointerPath(
+  const RunnerXCTestEventBridge *bridge,
+  CGPoint point
+) {
+  id path =
+    ((RunnerMsgSendInitPath)objc_msgSend)([bridge->pathClass alloc], bridge->initPathSelector, point, 0.0);
+  if (path == nil) {
+    return nil;
+  }
+  ((RunnerMsgSendPathOffset)objc_msgSend)(path, bridge->liftSelector, 0.05);
+  return path;
+}
+
 static CGPoint RunnerPointerPointAt(
   double x,
   double y,

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -8,20 +8,18 @@ extension RunnerTests {
   }
 
   func resolveAlert(app activeApp: XCUIApplication) -> RunnerAlert? {
+#if !os(macOS)
+    if let systemModal = firstBlockingSystemModal(in: springboard) {
+      return runnerAlert(root: systemModal, ownerApp: springboard)
+    }
+#endif
     if let alert = firstExistingElement(in: activeApp.alerts.allElementsBoundByIndex) {
       return runnerAlert(root: alert, ownerApp: activeApp)
     }
     if let popup = firstDismissPopupWindow(in: activeApp) {
       return runnerAlert(root: popup, ownerApp: activeApp)
     }
-#if os(macOS)
-    return nil
-#else
-    if let systemModal = firstBlockingSystemModal(in: springboard) {
-      return runnerAlert(root: systemModal, ownerApp: springboard)
-    }
     return nil
-#endif
   }
 
   func handleAlert(_ alert: RunnerAlert, action: String) -> Response {
@@ -33,6 +31,27 @@ extension RunnerTests {
       if let response = unsupportedResponse(for: outcome) {
         return response
       }
+      sleepFor(0.2)
+      if alertStillVisible(alert, actionButtonLabel: button.label) {
+        let frame = button.frame
+        if !frame.isNull && !frame.isEmpty {
+          let coordinateOutcome = tapAt(app: alert.ownerApp, x: frame.midX, y: frame.midY)
+          if let response = unsupportedResponse(for: coordinateOutcome) {
+            return response
+          }
+          sleepFor(0.2)
+        }
+      }
+      if alertStillVisible(alert, actionButtonLabel: button.label) {
+        return Response(
+          ok: false,
+          error: ErrorPayload(
+            code: "INTERACTION_FAILED",
+            message: "alert \(action) did not dismiss the visible alert",
+            hint: "The alert button was found but the system still reports the alert after tapping it."
+          )
+        )
+      }
       return Response(ok: true, data: DataPayload(message: action == "accept" ? "accepted" : "dismissed"))
     }
 
@@ -53,6 +72,21 @@ extension RunnerTests {
     return RunnerAlert(root: root, ownerApp: ownerApp, buttons: buttons)
   }
 
+  private func alertStillVisible(_ alert: RunnerAlert, actionButtonLabel: String) -> Bool {
+    guard let current = resolveAlert(app: alert.ownerApp) else {
+      return false
+    }
+    let previousTitle = preferredAlertTitle(alert.root, buttons: alert.buttons)
+    let currentTitle = preferredAlertTitle(current.root, buttons: current.buttons)
+    if previousTitle == currentTitle {
+      return true
+    }
+    let normalizedActionLabel = actionButtonLabel.trimmingCharacters(in: .whitespacesAndNewlines)
+    return current.buttons.contains { button in
+      button.label.trimmingCharacters(in: .whitespacesAndNewlines) == normalizedActionLabel
+    }
+  }
+
   private func firstExistingElement(in elements: [XCUIElement]) -> XCUIElement? {
     elements.first { isVisibleElement($0) }
   }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -147,10 +147,64 @@ extension RunnerTests {
     return Response(ok: true, data: data)
   }
 
+  func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() {
+    let response = gestureResponse(
+      message: "tapped",
+      timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2),
+      fallback: GestureFallback(
+        strategy: "xctest-coordinate-tap",
+        message: "Runner synthesized coordinate tap is unavailable",
+        hint: "Using XCTest coordinate tap fallback."
+      )
+    )
+
+    XCTAssertEqual(response.ok, true)
+    XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap")
+    XCTAssertEqual(
+      response.data?.gestureFallbackMessage,
+      "Runner synthesized coordinate tap is unavailable"
+    )
+    XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.")
+  }
+
+  func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws {
+    let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#)
+    let response = Response(ok: true, data: DataPayload(message: "tapped"))
+
+    let failureResponse = xctestRecordedFailureResponse(command: command, response: response)
+
+    XCTAssertEqual(failureResponse?.ok, false)
+    XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE")
+    XCTAssertEqual(
+      failureResponse?.error?.message,
+      "XCTest recorded a failure while executing tap; the action may not have been performed."
+    )
+  }
+
+  func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws {
+    let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#)
+    let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#)
+    let runnerFatalResponse = Response(
+      ok: true,
+      data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable")
+    )
+
+    XCTAssertNil(
+      xctestRecordedFailureResponse(
+        command: snapshotCommand,
+        response: Response(ok: true, data: DataPayload(nodes: [], truncated: false))
+      )
+    )
+    XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse))
+  }
+
   func execute(command: Command) throws -> Response {
     if command.command == .status {
       return executeStatus(command: command)
     }
+    if command.command == .uptime {
+      return executeUptime()
+    }
     commandJournal.accept(command: command)
     return try executeAccepted(command: command)
   }
@@ -185,6 +239,13 @@ extension RunnerTests {
     return Response(ok: true, data: commandJournal.status(commandId: statusCommandId))
   }
 
+  func executeUptime() -> Response {
+    Response(
+      ok: true,
+      data: DataPayload(currentUptimeMs: currentUptimeMs())
+    )
+  }
+
   private func executeDispatched(command: Command) throws -> Response {
     if Thread.isMainThread {
       return try executeOnMainSafely(command: command)
@@ -229,6 +290,7 @@ extension RunnerTests {
     while true {
       var response: Response?
       var swiftError: Error?
+      let failureCountBefore = currentXCTestFailureCount()
       let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
         do {
           response = try self.executeOnMain(command: command)
@@ -238,8 +300,7 @@ extension RunnerTests {
       })
 
       if let exceptionMessage {
-        currentApp = nil
-        currentBundleId = nil
+        invalidateCachedTarget(reason: "objc_exception")
         if !hasRetried, shouldRetryException(command, message: exceptionMessage) {
           NSLog(
             "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=objc_exception",
@@ -265,14 +326,19 @@ extension RunnerTests {
           userInfo: [NSLocalizedDescriptionKey: "command returned no response"]
         )
       }
+      if didRecordXCTestFailure(since: failureCountBefore),
+        let failureResponse = xctestRecordedFailureResponse(command: command, response: response)
+      {
+        invalidateCachedTarget(reason: "xctest_recorded_failure")
+        return failureResponse
+      }
       if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) {
         NSLog(
           "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable",
           command.command.rawValue
         )
         hasRetried = true
-        currentApp = nil
-        currentBundleId = nil
+        invalidateCachedTarget(reason: "response_unavailable")
         sleepFor(retryCooldown)
         continue
       }
@@ -282,7 +348,9 @@ extension RunnerTests {
 
   private func executeOnMain(command: Command) throws -> Response {
     var activeApp = currentApp ?? app
-    if !isRunnerLifecycleCommand(command.command) {
+    if shouldSkipAppActivationPreflight(command) {
+      activeApp = resolveAppWithoutActivation(command: command)
+    } else if !isRunnerLifecycleCommand(command.command) {
       let normalizedBundleId = command.appBundleId?
         .trimmingCharacters(in: .whitespacesAndNewlines)
       let requestedBundleId = (normalizedBundleId?.isEmpty == true) ? nil : normalizedBundleId
@@ -408,10 +476,7 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(message: "failed to stop recording: \(error.localizedDescription)"))
       }
     case .uptime:
-      return Response(
-        ok: true,
-        data: DataPayload(currentUptimeMs: currentUptimeMs())
-      )
+      return executeUptime()
     case .tap:
       if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
         let match = findElement(
@@ -425,6 +490,7 @@ extension RunnerTests {
         }
         if let element = match.element {
           let frame = element.frame
+          let isTextEntry = isTextEntryElement(element)
           let touchFrame = frame.isEmpty
             ? nil
             : resolvedTouchVisualizationFrame(app: activeApp, x: frame.midX, y: frame.midY)
@@ -440,7 +506,9 @@ extension RunnerTests {
           if let response = unsupportedResponse(for: outcome) {
             return response
           }
-          waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
+          if isTextEntry {
+            waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
+          }
           return gestureResponse(
             message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped",
             timing: timing,
@@ -462,12 +530,27 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(message: "element not found"))
       }
       if let x = command.x, let y = command.y {
+        var fallback: GestureFallback?
+        if command.synthesized == true {
+          let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+            synthesizedTapAt(app: activeApp, x: x, y: y)
+          }
+          if case .performed = outcome {
+            return gestureResponse(message: "tapped", timing: timing)
+          }
+          fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome)
+        }
         let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
         let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) }
         if let response = unsupportedResponse(for: outcome) {
           return response
         }
-        return gestureResponse(message: "tapped", timing: timing, frame: .touch(touchFrame))
+        return gestureResponse(
+          message: "tapped",
+          timing: timing,
+          frame: .touch(touchFrame),
+          fallback: fallback
+        )
       }
       return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y"))
     case .mouseClick:
@@ -736,6 +819,7 @@ extension RunnerTests {
         needsPostSnapshotInteractionDelay = true
         return Response(ok: true, data: payload)
       } catch let failure as SnapshotCaptureFailure {
+        invalidateCachedTarget(reason: "ax_snapshot_failure")
         // Other thrown errors fall through to executeOnMainSafely's generic error response.
         return Response(
           ok: false,
@@ -935,6 +1019,65 @@ extension RunnerTests {
     }
   }
 
+  private func currentXCTestFailureCount() -> Int {
+    return testRun?.failureCount ?? 0
+  }
+
+  private func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
+    return currentXCTestFailureCount() > failureCountBefore
+  }
+
+  private func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? {
+    guard response.ok else { return nil }
+    if response.data?.runnerFatal == true {
+      return nil
+    }
+    guard !isReadOnlyCommand(command), !isRunnerLifecycleCommand(command.command) else {
+      return nil
+    }
+    return Response(
+      ok: false,
+      error: ErrorPayload(
+        code: "XCTEST_RECORDED_FAILURE",
+        message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.",
+        hint: "The iOS runner session will be restarted. Retry after a fresh snapshot, or use screenshot plus coordinate commands when the accessibility tree is unavailable."
+      )
+    )
+  }
+
+  private func runnerCommandFixture(_ json: String) throws -> Command {
+    try JSONDecoder().decode(Command.self, from: Data(json.utf8))
+  }
+
+  private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool {
+#if os(iOS)
+    // Coordinate-only synthesized taps can run after an AX-fatal screen because they do not need
+    // app activation, window lookup, keyboard lookup, or element resolution. Selector/text taps
+    // intentionally stay on the normal AX path because they need an element query.
+    return command.command == .tap
+      && command.synthesized == true
+      && command.x != nil
+      && command.y != nil
+      && command.text == nil
+      && command.selectorKey == nil
+#else
+    return false
+#endif
+  }
+
+  private func resolveAppWithoutActivation(command: Command) -> XCUIApplication {
+    guard let bundleId = command.appBundleId?
+      .trimmingCharacters(in: .whitespacesAndNewlines),
+      !bundleId.isEmpty
+    else {
+      return currentApp ?? app
+    }
+    if currentBundleId == bundleId, let currentApp {
+      return currentApp
+    }
+    return XCUIApplication(bundleIdentifier: bundleId)
+  }
+
   private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
     guard let text = command.text else {
       return Response(ok: false, error: ErrorPayload(message: "type requires text"))

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -154,6 +154,17 @@ final class RunnerCommandJournal {
 }
 
 extension RunnerTests {
+  func testUptimeBypassesCommandJournal() throws {
+    let command = runnerJournalCommand("uptime", id: "uptime-probe")
+
+    let response = try execute(command: command)
+    let status = commandJournal.status(commandId: "uptime-probe")
+
+    XCTAssertEqual(response.ok, true)
+    XCTAssertNotNil(response.data?.currentUptimeMs)
+    XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue)
+  }
+
   func testCommandJournalRetentionPolicy() throws {
     let journal = RunnerCommandJournal()
 

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Exceptions.swift

@@ -10,10 +10,7 @@ extension RunnerTests {
   /// exception telemetry later. `RunnerObjCExceptionCatcher.catchException` takes a non-escaping
   /// block, so `block` may capture `inout` state.
   func safely<T>(_ tag: String, _ fallback: T, _ block: () -> T) -> T {
-    var result = fallback
-    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
-      result = block()
-    })
+    let (result, exceptionMessage) = catchingObjCException(fallback: fallback, block)
     if let exceptionMessage {
       NSLog("AGENT_DEVICE_RUNNER_%@_IGNORED_EXCEPTION=%@", tag, exceptionMessage)
       return fallback
@@ -21,6 +18,17 @@ extension RunnerTests {
     return result
   }
 
+  func catchingObjCException<T>(
+    fallback: T,
+    _ block: () -> T
+  ) -> (result: T, exceptionMessage: String?) {
+    var result = fallback
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      result = block()
+    })
+    return (result, exceptionMessage)
+  }
+
   /// Optional-returning convenience: returns `nil` on exception (matching the common
   /// `var x: T?` + catch-and-return-nil shape).
   func safely<T>(_ tag: String, _ block: () -> T?) -> T? {

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -670,6 +670,32 @@ extension RunnerTests {
 #endif
   }
 
+  func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+#if os(iOS)
+    if let message = RunnerSynthesizedGesture.synthesizeTap(
+      withApplication: app,
+      x: x,
+      y: y
+    ) {
+      return .unsupported(
+        message: message,
+        hint: "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree."
+      )
+    }
+    return .performed
+#elseif os(tvOS)
+    return .unsupported(
+      message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
+      hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
+    )
+#else
+    return .unsupported(
+      message: "synthesized coordinate tap is not supported on macOS",
+      hint: "macOS automation has no touchscreen; use mouse-driven interactions instead."
+    )
+#endif
+  }
+
   func keyboardAvoidingDragPoints(
     app: XCUIApplication,
     x: Double,

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -87,6 +87,14 @@ extension RunnerTests {
     currentBundleId = nil
   }
 
+  func invalidateCachedTarget(reason: String) {
+    if currentApp != nil || currentBundleId != nil {
+      NSLog("AGENT_DEVICE_RUNNER_TARGET_CACHE_INVALIDATE reason=%@", reason)
+    }
+    currentApp = nil
+    currentBundleId = nil
+  }
+
   func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
     let state = target.state
 #if os(macOS)

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -193,6 +193,8 @@ struct DataPayload: Codable {
   let gestureFallback: String?
   let gestureFallbackMessage: String?
   let gestureFallbackHint: String?
+  let runnerFatal: Bool?
+  let runnerFatalReason: String?
 
   init(
     message: String? = nil,
@@ -224,7 +226,9 @@ struct DataPayload: Codable {
     orientation: String? = nil,
     gestureFallback: String? = nil,
     gestureFallbackMessage: String? = nil,
-    gestureFallbackHint: String? = nil
+    gestureFallbackHint: String? = nil,
+    runnerFatal: Bool? = nil,
+    runnerFatalReason: String? = nil
   ) {
     self.message = message
     self.text = text
@@ -256,6 +260,8 @@ struct DataPayload: Codable {
     self.gestureFallback = gestureFallback
     self.gestureFallbackMessage = gestureFallbackMessage
     self.gestureFallbackHint = gestureFallbackHint
+    self.runnerFatal = runnerFatal
+    self.runnerFatalReason = runnerFatalReason
   }
 }