agent-device 0.17.10 → 0.18.1

package/dist/src/simulator.js

@@ -0,0 +1,2 @@
+import{AppError as t}from"./485.js";import{classifyBootFailure as e,retryWithPolicy as o,bootFailureHint as i,Deadline as r}from"./3340.js";import{runXcrun as s,buildSimctlArgs as d,buildSimctlArgsForDevice as a,runAppleToolCommand as u}from"./9722.js";let l=["Simulator"],n=["Device Hub","Simulator"];function c(e,o){if("simulator"!==e.kind)throw new t("UNSUPPORTED_OPERATION",`${o} is only supported on iOS simulators`)}async function m(t={}){let e=t.deviceHub?n:l,o=t.background?["-g","-a"]:["-a"];for(let t of e)if(0===(await u("open",[...o,t],{allowFailure:!0,timeoutMs:1e4})).exitCode)return}async function f(d,u={}){let l,n;if("simulator"!==d.kind)return;if("Booted"===await C(d)){u.focusExisting&&await m({background:u.deviceHub,deviceHub:u.deviceHub});return}u.onColdBootStart?.(d);let c=r.fromTimeoutMs(18e4);try{await o(async({deadline:e})=>{if(e?.isExpired())throw new t("COMMAND_FAILED","iOS simulator boot deadline exceeded",{timeoutMs:18e4});let o=Math.max(1e3,e?.remainingMs()??18e4),i=await s(a(d,["boot",d.id]),{allowFailure:!0,timeoutMs:o});l={stdout:String(i.stdout??""),stderr:String(i.stderr??""),exitCode:i.exitCode};let r=`${l.stdout}
+${l.stderr}`.toLowerCase(),u=r.includes("already booted")||r.includes("current state: booted");if(0!==l.exitCode&&!u)throw new t("COMMAND_FAILED","simctl boot failed",{stdout:l.stdout,stderr:l.stderr,exitCode:l.exitCode});let c=await s(a(d,["bootstatus",d.id,"-b"]),{allowFailure:!0,timeoutMs:o});if(n={stdout:String(c.stdout??""),stderr:String(c.stderr??""),exitCode:c.exitCode},0!==n.exitCode)throw new t("COMMAND_FAILED","simctl bootstatus failed",{stdout:n.stdout,stderr:n.stderr,exitCode:n.exitCode});let m=await C(d);if("Booted"!==m)throw new t("COMMAND_FAILED","Simulator is still booting",{state:m})},{maxAttempts:3,baseDelayMs:500,maxDelayMs:2e3,jitter:.2,shouldRetry:t=>{let o=e({error:t,stdout:n?.stdout??l?.stdout,stderr:n?.stderr??l?.stderr,context:{platform:"ios",phase:"boot"}});return"IOS_BOOT_TIMEOUT"!==o&&"CI_RESOURCE_STARVATION_SUSPECTED"!==o}},{deadline:c,phase:"boot",classifyReason:t=>e({error:t,stdout:n?.stdout??l?.stdout,stderr:n?.stderr??l?.stderr,context:{platform:"ios",phase:"boot"}})})}catch(r){let o=e({error:r,stdout:n?.stdout??l?.stdout,stderr:n?.stderr??l?.stderr,context:{platform:"ios",phase:"boot"}});throw new t("COMMAND_FAILED","iOS simulator failed to boot",{platform:"ios",deviceId:d.id,timeoutMs:18e4,elapsedMs:c.elapsedMs(),reason:o,hint:i(o),boot:l,bootstatus:n})}await m({deviceHub:u.deviceHub})}async function S(t){let e=a(t,["shutdown",t.id]),o=await s(e,{allowFailure:!0,timeoutMs:15e3});return{success:0===o.exitCode,exitCode:o.exitCode,stdout:String(o.stdout??""),stderr:String(o.stderr??"")}}async function C(t){let e="string"==typeof t?t:t.id,o="string"==typeof t?d(["list","devices","-j"]):a(t,["list","devices","-j"]),i=await s(o,{allowFailure:!0,timeoutMs:6e4});if(0!==i.exitCode)return null;try{let t=JSON.parse(String(i.stdout??""));for(let o of Object.values(t.devices??{})){let t=o.find(t=>t.udid===e);if(t)return t.state}return null}catch{return null}}export{f as ensureBootedSimulator,C as getSimulatorState,c as requireSimulatorDevice,S as shutdownSimulator};

package/dist/src/snapshot.js

@@ -1,2 +1,2 @@
-import{__webpack_require__ as e}from"./rslib-runtime.js";import{promises as t}from"node:fs";import a from"node:os";import n from"node:path";import{centerOfRect as s}from"./4291.js";import{normalizeError as o,AppError as r}from"./733.js";import{withDiagnosticTimer as i,emitDiagnostic as l}from"./432.js";import{pressAndroid as d,backAndroid as p}from"./input-actions.js";import{createDaemonRuntimeSessionStore as u,captureSnapshot as c,dispatchCommand as f,resolveSnapshotScope as m,runMacOsAlertAction as h,errorResponse as w,runIosRunnerCommand as g,isCommandSupportedOnDevice as y,snapshotAndroid as _,annotateScreenshotWithRefs as v,context_contextFromFlags as S}from"./9722.js";import{SETTINGS_INVALID_ARGS_MESSAGE as b,parseTimeout as I,getUnsupportedMacOsSettingMessage as A,successText as R,isMacOsSettingSupported as C,snapshotCaptureAnnotationsFrom as N}from"./2948.js";import{sleep as D}from"./5560.js";import{buildSnapshotSession as P,dispatchWaitViaRuntime as k,createDaemonRuntimePolicy as M,recordIfSession as E,resolveSessionDevice as O,withSessionlessRunnerCleanup as K}from"./selector-runtime.js";import{createAgentDevice as U}from"./2577.js";import{summarizeSnapshotDiagnostics as x}from"./8358.js";var L={};e.r(L),e.d(L,{handleSnapshotCommands:()=>ew});let B=new Set(["com.android.permissioncontroller","com.google.android.permissioncontroller","com.google.android.packageinstaller","com.android.packageinstaller"]),$=new Set(["android","com.android.systemui"]),H=/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i,q=/^android:id\/button[123]$/i,F=/(?:^|:)id\/permission_/i,T=/\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i,j=/^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i,X=/^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i;function z(e,t){let a=t[0];if(!a)return;let n=G(e,a.index);for(let a of t.slice(1)){let t=new Set(G(e,a.index));for(let e=n.length-1;e>=0;e-=1)t.has(n[e])||n.splice(e,1)}return n[n.length-1]}function G(e,t){let a=new Map(e.map(e=>[e.index,e])),n=[],s=a.get(t);for(;s;)n.push(s.index),s=void 0===s.parentIndex?void 0:a.get(s.parentIndex);return n.reverse()}function V(e,t){let a=new Map;for(let t of e){if(void 0===t.parentIndex)continue;let e=a.get(t.parentIndex)??[];e.push(t),a.set(t.parentIndex,e)}let n=new Set([t]),s=[t];for(let e of s)for(let t of a.get(e)??[])n.has(t.index)||(n.add(t.index),s.push(t.index));return e.filter(e=>n.has(e.index))}function W(e){let t=e.type??"",a=e.identifier??"";return!!(e.hittable||/\bbutton\b/i.test(t)||q.test(a)||/(?:^|:)id\/permission_(?:allow|deny)/i.test(a))}function J(e){if(!e)return"";let t=[e.label,e.value].filter(e=>"string"==typeof e&&e.trim().length>0);return t[0]?.trim()??""}async function Q(e,t,a={}){if("wait"===t)return await Y(e,a.timeoutMs??1e4);if("get"===t){var n;let t=await et(e);return{kind:"alertStatus",platform:"android",action:"get",alert:n=t?.alert??null,...n?R("Alert visible"):R("No alert visible")}}return await Z(e,t)}async function Y(e,t){let a=Date.now(),n=await ee(e,t);if(!n)throw new r("COMMAND_FAILED","alert wait timed out");return{kind:"alertWait",platform:"android",action:"wait",alert:n.alert,waitedMs:Date.now()-a,...R("Alert visible")}}async function Z(e,t){var a;let n,s,o=await ee(e,2e3);if(!o)throw new r("COMMAND_FAILED","alert not found",{hint:"If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref."});let i=(a=o.buttons,n="accept"===t?"accept":"dismiss",(s=a.find(e=>e.role===n))||("dismiss"===t?a.find(e=>"neutral"===e.role)??null:1===a.length?a[0]??null:null));if(i)return await d(e,i.x,i.y),ea(t,o.alert,i.label);if("dismiss"===t)return await p(e),ea(t,o.alert,"Back");throw new r("COMMAND_FAILED","alert accept found an alert but no accept button",{alert:o.alert,hint:"Inspect alert get --json for visible buttons, then use press by visible label/ref if needed."})}async function ee(e,t){let a=Date.now();for(;Date.now()-a<t;){let t=await et(e);if(t)return t;await D(300)}return null}async function et(e){return function(e){var t,a,n,o;let r,i=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",B.has(a)||F.test(t.identifier??"")});if(t.length)return{nodes:t,source:"permission"};let a=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",$.has(a)&&T.test(J(t))});if(0===t.length)return[];let a=z(e,t);return void 0===a?t:V(e,a).filter(e=>e.bundleId&&$.has(e.bundleId))}(e);return a.length?{nodes:a,source:"system-dialog"}:{nodes:function(e){var t;let a,n,s=e.filter(e=>{var t;return t=e.type??"",/(?:^|[.$])[^.]*Dialog$/i.test(t)}),o=e.filter(e=>H.test(e.identifier??"")),r=s.length?[...s,...o]:(a=(t=o).some(e=>q.test(e.identifier??"")),n=t.some(e=>!q.test(e.identifier??"")),a&&n?t:[]);if(0===r.length)return[];let i=z(e,r);return void 0===i?r:V(e,i)}(e),source:"native-dialog"}}(e),l=i.nodes;if(0===l.length)return null;let d=function(e){let t=new Set,a=[];for(let n of e){let e=J(n);if(!e||!n.rect||!W(n))continue;let o=e.trim().toLowerCase();if(!o||t.has(o))continue;t.add(o);let r=s(n.rect);a.push({label:e,x:r.x,y:r.y,role:function(e,t){var a;let n=(a=e.identifier??"",/(?:^|:)id\/button1$/i.test(a)?"accept":/(?:^|:)id\/button2$/i.test(a)?"dismiss":/(?:^|:)id\/button3$/i.test(a)?"neutral":/(?:^|:)id\/permission_allow/i.test(a)?"accept":/(?:^|:)id\/permission_deny/i.test(a)?"dismiss":null);return n||(j.test(t.trim())?"accept":X.test(t.trim())?"dismiss":"neutral")}(n,e)})}return a}(l),p=l.filter(e=>J(e)&&!W(e)),u=J((t=p).find(e=>/(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(e.identifier??"")))||J(t[0]),c=(a=p,n=u,(r=a.map(e=>J(e)).filter(e=>e&&e!==n)).length?[...new Set(r)].join("\n"):void 0),f=(o=l,o.find(e=>e.bundleId)?.bundleId);return{alert:{...u?{title:u}:{},...c?{message:c}:{},buttons:d.map(e=>e.label),platform:"android",source:i.source,...f?{packageName:f}:{}},buttons:d}}((await i("snapshot_capture",async()=>await _(e,{helperWaitForIdleTimeoutMs:0,includeHiddenContentHints:!1}),{backend:"android",purpose:"alert"})).nodes)}function ea(e,t,a){return{kind:"alertHandled",platform:"android",action:e,handled:!0,alert:t,button:a,...R(`Alert ${e}ed`)}}async function en(e){var t;let{req:a,logPath:n,session:s,device:o}=e,r="accept"===(t=a.positionals?.[0])||"dismiss"===t||"wait"===t?t:"get",i=s?"frontmost-app"===s.surface?{surface:"frontmost-app"}:{bundleId:s.appBundleId,surface:s.surface}:{};if(!y("alert",o))return w("UNSUPPORTED_OPERATION","alert is not supported on this device");if("android"===o.platform){let t=I(a.positionals?.[1])??1e4;return ei(e,await Q(o,r,{timeoutMs:t}))}if("macos"===o.platform){let t=async e=>await h(e,i);return await es(e,r,t)}let l={verbose:a.flags?.verbose,logPath:n,traceLogPath:s?.trace?.outPath,requestId:a.meta?.requestId,iosXctestrunFile:a.flags?.iosXctestrunFile,iosXctestDerivedDataPath:a.flags?.iosXctestDerivedDataPath,iosXctestEnvDir:a.flags?.iosXctestEnvDir},d=async e=>await g(o,{command:"alert",action:e,appBundleId:s?.appBundleId},l);return await es(e,r,d)}async function es(e,t,a){if("wait"===t)return await eo(e,a);let n="accept"===t||"dismiss"===t?t:"get";return"accept"===n||"dismiss"===n?await er(e,n,a):ei(e,await a("get"))}async function eo(e,t){let a=I(e.req.positionals?.[1])??1e4,n=Date.now();for(;Date.now()-n<a;){try{return ei(e,await t("get"))}catch{}await D(300)}return w("COMMAND_FAILED","alert wait timed out")}async function er(e,t,a){var n,s;let o,i,l=Date.now();for(;Date.now()-l<2e3;){try{return ei(e,await a(t))}catch(t){i=t;let e=String(t?.message??"").toLowerCase();if(!e.includes("alert not found")&&!e.includes("no alert"))break}await D(300)}throw(n=i)instanceof r&&(s=n,(o=String(s?.message??"").toLowerCase()).includes("alert not found")||o.includes("no alert"))?new r(n.code,n.message,{...n.details??{},hint:"If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref."}):n}function ei(e,t){return E(e.sessionStore,e.session,e.req,t),{ok:!0,data:t}}async function el(e){let{req:t,logPath:a,sessionStore:n,session:s,device:o,parsed:r}=e,{setting:i,state:l,appBundleId:d,permissionTarget:p,latitude:u,longitude:c}=r;if(!y("settings",o))return w("UNSUPPORTED_OPERATION","settings is not supported on this device");if("macos"===o.platform&&!C(i))return w("INVALID_ARGS",A(i));let m=d??s?.appBundleId;if("clear-app-state"===i&&!m)return w("INVALID_ARGS","settings clear-app-state requires an app id when no app is bound to the session");let h="clear-app-state"===i?[i,l,m??""]:"permission"===i?[i,l,p??"",t.positionals?.[3]??"",m??""]:"location"===i&&"set"===l?[i,l,u??"",c??"",m??""]:[i,l,m??""],g=await f(o,"settings",h,t.flags?.out,{...S(a,t.flags,m,s?.trace?.outPath)});return E(n,s,t,g??{setting:i,state:l}),{ok:!0,data:g??{setting:i,state:l}}}async function ed(e){var t,a;let n;if("snapshot"!==e.command||"android"!==e.device.platform)return;let s=o(e.error);if("COMMAND_FAILED"===(t=s).code&&(a=t,n=`${a.message}
-${a.hint??""}`,/Android UI hierarchy dump timed out/i.test(n)||/Stock UIAutomator fallback was skipped/i.test(n)||/Android accessibility snapshots can be blocked/i.test(n)||function(e){if(!e||"object"!=typeof e)return!1;let t=String(e.errorType??""),a=String(e.message??"");return/TimeoutException/i.test(t)||/timed out/i.test(a)}(t.details?.helper)||function(e){var t;if(!e)return!1;let a=e?.timeoutMs,n=e?.cmd,s=Array.isArray(t=e?.args)?t.map(String):"string"==typeof t?t.split(/\s+/):[];return"number"==typeof a&&"adb"===n&&s.includes("uiautomator")&&s.includes("dump")}(t.details)))return{ok:!1,error:{...s,details:{...s.details??{},androidSnapshotTimeoutScreenshot:await ep(e)}}}}async function ep(e){try{var s,r,i;let o=await t.mkdtemp(n.join(a.tmpdir(),"agent-device-android-snapshot-timeout-")),d=n.join(o,"snapshot-timeout-overlay-refs.png"),p=(s=await f(e.device,"screenshot",[d],void 0,{...S(e.logPath,{screenshotNoStabilize:!0},e.session?.appBundleId,e.session?.trace?.outPath),surface:e.session?.surface}),r=d,(i=s,"object"==typeof i&&null!==i&&"path"in i&&"string"==typeof i.path)?s.path:r);await t.access(p);let u=await eu(p,e.session);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_captured",data:{path:p,overlayRefCount:"overlayRefCount"in u?u.overlayRefCount:void 0,overlayRefsAnnotated:"overlayRefsAnnotated"in u?u.overlayRefsAnnotated:void 0}}),u}catch(t){let e=o(t);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_failed",data:{error:e.message}}),{captureFailed:!0,error:e.message}}}async function eu(e,t){if(!t?.snapshot)return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"unavailable",overlayRefCount:0};try{let a=await v({screenshotPath:e,snapshot:t.snapshot});return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:a.length>0,overlayRefCount:a.length,overlayRefSource:"session-snapshot",overlayRefs:a}}catch(a){let t=o(a);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_overlay_failed",data:{path:e,error:t.message}}),{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"session-snapshot",overlayRefCount:0,overlayAnnotationError:t.message}}}async function ec(e){return await em({...e,command:"snapshot",unsupportedMessage:"snapshot is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.snapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw,forceFull:a.flags?.snapshotForceFull});return{data:s,record:{kind:"snapshot",nodes:s.nodes.length,truncated:s.truncated}}}})}async function ef(e){return await em({...e,command:"diff",unsupportedMessage:"diff is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.diffSnapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw});return{data:s,record:{kind:"diff",mode:"snapshot",baselineInitialized:s.baselineInitialized,summary:s.summary}}}})}async function em(e){var t,a;let{req:n,sessionName:s,logPath:o,sessionStore:i}=e,{session:l,device:d}=await O(i,s,n.flags);if(!y(e.command,d))return w("UNSUPPORTED_OPERATION",e.unsupportedMessage);let p=m(n.flags?.snapshotScope,l);if(!p.ok)return p;let f=(t=e.command,a=l,"ios"!==d.platform||a?.appBundleId?null:w("SESSION_NOT_FOUND",`iOS ${t} requires an active app session on the target device. Run open first (for example: open --session ${a?.name??"sim"} --platform ios --device "<name>" <app>).`));return f||await K(l,d,async()=>{var t,a;let f,m,h=function(e){let{req:t,sessionName:a,logPath:n,sessionStore:s,session:o,device:i,snapshotScope:l}=e;return U({backend:function(e){let{req:t,logPath:a,session:n,device:s,snapshotScope:o}=e;return{platform:s.platform,captureSnapshot:async(e,r)=>{let i=await c({device:s,session:n,flags:t.flags,outPath:r?.outPath??t.flags?.out,logPath:a,snapshotScope:o}),l=x(n);return{snapshot:i.snapshot,...N(i),...l?{snapshotDiagnostics:l}:{},appName:n?.appBundleId?n.appName??n.appBundleId:void 0,appBundleId:n?.appBundleId}}}}({req:t,logPath:n,session:o,device:i,snapshotScope:l}),...M("snapshot"),sessions:u({sessionName:a,getSession:()=>s.get(a),recordOptions:{includeSnapshot:!0},setRecord:e=>{var n;let o=function(e){if(!e.snapshot)throw new r("UNKNOWN","snapshot runtime did not produce session state");return e}(e),l=s.get(a);s.set(a,function(e){var t,a;let{current:n,sessionName:s,device:o,record:r,refScopedSnapshot:i}=e,l=(t=n,a=r,i&&a.snapshot?.nodes.length===0&&t?.snapshot!==void 0),d=l?n.snapshot:r.snapshot,p=P({session:n,sessionName:s,device:o,snapshot:d,appBundleId:r.appBundleId});return p.snapshotScopeSource=function(e){let{current:t,keepCurrentSnapshot:a,refScopedSnapshot:n}=e;if(n)return a?t?.snapshotScopeSource:t?.snapshotScopeSource??t?.snapshot}({current:n,keepCurrentSnapshot:l,refScopedSnapshot:i}),r.appName&&(p.appName=r.appName),p}({current:l,sessionName:a,device:i,record:o,refScopedSnapshot:(n=t,n.flags?.snapshotScope?.trim().startsWith("@")===!0)}))}})})}({req:n,sessionName:s,logPath:o,sessionStore:i,session:l,device:d,snapshotScope:p.scope});try{m=await e.execute({runtime:h,sessionName:s,req:n,snapshotScope:p.scope})}catch(a){let t=await ed({error:a,command:e.command,logPath:o,session:l,device:d});if(!t)throw a;return t}return(f=(t={req:n,sessionName:s,sessionStore:i,result:m.record}).sessionStore.get(t.sessionName))&&t.sessionStore.recordAction(f,{command:t.req.command,positionals:t.req.positionals??[],flags:t.req.flags??{},result:"snapshot"===(a=t.result).kind?{nodes:a.nodes,truncated:a.truncated}:{mode:a.mode,baselineInitialized:a.baselineInitialized,summary:a.summary}}),{ok:!0,data:m.data}})}let eh={snapshot:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await ec({req:e,sessionName:t,logPath:a,sessionStore:n}),diff:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>e.positionals?.[0]!=="snapshot"?w("INVALID_ARGS","diff currently supports only: diff snapshot"):await ef({req:e,sessionName:t,logPath:a,sessionStore:n}),wait:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await k({req:e,sessionName:t,logPath:a,sessionStore:n}),alert:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let{session:s,device:o}=await O(n,t,e.flags);return await K(s,o,async()=>await en({req:e,logPath:a,sessionStore:n,session:s,device:o}))},settings:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let s,o,r,i=(s=e.positionals?.[0]?.toLowerCase(),o=e.positionals?.[1]?.toLowerCase(),r=e.positionals?.[2]?.toLowerCase(),"clear-app-state"===s?{ok:!0,parsed:{setting:s,state:"clear",appBundleId:"clear"===o?e.positionals?.[2]:e.positionals?.[1]}}:s&&o&&("permission"!==s||r)&&("location"!==s||"set"!==o||e.positionals?.[2]&&e.positionals?.[3])?{ok:!0,parsed:{setting:s,state:o,permissionTarget:r,latitude:e.positionals?.[2],longitude:e.positionals?.[3]}}:w("INVALID_ARGS",b));if(!i.ok)return i;let{session:l,device:d}=await O(n,t,e.flags);return await K(l,d,async()=>await el({req:e,logPath:a,sessionStore:n,session:l,device:d,parsed:i.parsed}))}};async function ew(e){let t=eh[e.req.command];return t?await t(e):null}export{L as snapshot_namespaceObject};
+import{__webpack_require__ as e}from"./rslib-runtime.js";import{promises as t}from"node:fs";import a from"node:os";import n from"node:path";import{centerOfRect as s}from"./6849.js";import{normalizeError as o,AppError as r}from"./485.js";import{withDiagnosticTimer as i,emitDiagnostic as l}from"./432.js";import{pressAndroid as d,backAndroid as p}from"./input-actions.js";import{runAppleRunnerCommand as u,createDaemonRuntimeSessionStore as c,captureSnapshot as f,dispatchCommand as m,requireCommandSupported as h,runMacOsAlertAction as w,errorResponse as g,annotateScreenshotWithRefs as y,resolveSnapshotScope as _,snapshotAndroid as v,context_contextFromFlags as S}from"./9722.js";import{SETTINGS_INVALID_ARGS_MESSAGE as b,parseTimeout as I,getUnsupportedMacOsSettingMessage as A,successText as C,isMacOsSettingSupported as R,snapshotCaptureAnnotationsFrom as N}from"./2948.js";import{sleep as M}from"./5560.js";import{buildAppleRunnerRequestOptions as k}from"./8414.js";import{buildSnapshotSession as D,dispatchWaitViaRuntime as K,createDaemonRuntimePolicy as x,recordIfSession as P,resolveSessionDevice as E,withSessionlessRunnerCleanup as U}from"./selector-runtime.js";import{createAgentDevice as L}from"./7871.js";import{summarizeSnapshotDiagnostics as O}from"./5913.js";var B={};e.r(B),e.d(B,{handleSnapshotCommands:()=>eg});let H=new Set(["com.android.permissioncontroller","com.google.android.permissioncontroller","com.google.android.packageinstaller","com.android.packageinstaller"]),$=new Set(["android","com.android.systemui"]),q=/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i,F=/^android:id\/button[123]$/i,j=/(?:^|:)id\/permission_/i,T=/\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i,z=/^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i,G=/^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i;function V(e,t){let a=t[0];if(!a)return;let n=W(e,a.index);for(let a of t.slice(1)){let t=new Set(W(e,a.index));for(let e=n.length-1;e>=0;e-=1)t.has(n[e])||n.splice(e,1)}return n[n.length-1]}function W(e,t){let a=new Map(e.map(e=>[e.index,e])),n=[],s=a.get(t);for(;s;)n.push(s.index),s=void 0===s.parentIndex?void 0:a.get(s.parentIndex);return n.reverse()}function J(e,t){let a=new Map;for(let t of e){if(void 0===t.parentIndex)continue;let e=a.get(t.parentIndex)??[];e.push(t),a.set(t.parentIndex,e)}let n=new Set([t]),s=[t];for(let e of s)for(let t of a.get(e)??[])n.has(t.index)||(n.add(t.index),s.push(t.index));return e.filter(e=>n.has(e.index))}function Q(e){let t=e.type??"",a=e.identifier??"";return!!(e.hittable||/\bbutton\b/i.test(t)||F.test(a)||/(?:^|:)id\/permission_(?:allow|deny)/i.test(a))}function X(e){if(!e)return"";let t=[e.label,e.value].filter(e=>"string"==typeof e&&e.trim().length>0);return t[0]?.trim()??""}async function Y(e,t,a={}){if("wait"===t)return await Z(e,a.timeoutMs??1e4);if("get"===t){var n;let t=await ea(e);return{kind:"alertStatus",platform:"android",action:"get",alert:n=t?.alert??null,...n?C("Alert visible"):C("No alert visible")}}return await ee(e,t)}async function Z(e,t){let a=Date.now(),n=await et(e,t);if(!n)throw new r("COMMAND_FAILED","alert wait timed out");return{kind:"alertWait",platform:"android",action:"wait",alert:n.alert,waitedMs:Date.now()-a,...C("Alert visible")}}async function ee(e,t){var a;let n,s,o=await et(e,2e3);if(!o)throw new r("COMMAND_FAILED","alert not found",{hint:"If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref."});let i=(a=o.buttons,n="accept"===t?"accept":"dismiss",(s=a.find(e=>e.role===n))||("dismiss"===t?a.find(e=>"neutral"===e.role)??null:1===a.length?a[0]??null:null));if(i)return await d(e,i.x,i.y),en(t,o.alert,i.label);if("dismiss"===t)return await p(e),en(t,o.alert,"Back");throw new r("COMMAND_FAILED","alert accept found an alert but no accept button",{alert:o.alert,hint:"Inspect alert get --json for visible buttons, then use press by visible label/ref if needed."})}async function et(e,t){let a=Date.now();for(;Date.now()-a<t;){let t=await ea(e);if(t)return t;await M(300)}return null}async function ea(e){return function(e){var t,a,n,o;let r,i=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",H.has(a)||j.test(t.identifier??"")});if(t.length)return{nodes:t,source:"permission"};let a=function(e){let t=e.filter(e=>{var t;let a;return a=(t=e).bundleId??"",$.has(a)&&T.test(X(t))});if(0===t.length)return[];let a=V(e,t);return void 0===a?t:J(e,a).filter(e=>e.bundleId&&$.has(e.bundleId))}(e);return a.length?{nodes:a,source:"system-dialog"}:{nodes:function(e){var t;let a,n,s=e.filter(e=>{var t;return t=e.type??"",/(?:^|[.$])[^.]*Dialog$/i.test(t)}),o=e.filter(e=>q.test(e.identifier??"")),r=s.length?[...s,...o]:(a=(t=o).some(e=>F.test(e.identifier??"")),n=t.some(e=>!F.test(e.identifier??"")),a&&n?t:[]);if(0===r.length)return[];let i=V(e,r);return void 0===i?r:J(e,i)}(e),source:"native-dialog"}}(e),l=i.nodes;if(0===l.length)return null;let d=function(e){let t=new Set,a=[];for(let n of e){let e=X(n);if(!e||!n.rect||!Q(n))continue;let o=e.trim().toLowerCase();if(!o||t.has(o))continue;t.add(o);let r=s(n.rect);a.push({label:e,x:r.x,y:r.y,role:function(e,t){var a;let n=(a=e.identifier??"",/(?:^|:)id\/button1$/i.test(a)?"accept":/(?:^|:)id\/button2$/i.test(a)?"dismiss":/(?:^|:)id\/button3$/i.test(a)?"neutral":/(?:^|:)id\/permission_allow/i.test(a)?"accept":/(?:^|:)id\/permission_deny/i.test(a)?"dismiss":null);return n||(z.test(t.trim())?"accept":G.test(t.trim())?"dismiss":"neutral")}(n,e)})}return a}(l),p=l.filter(e=>X(e)&&!Q(e)),u=X((t=p).find(e=>/(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(e.identifier??"")))||X(t[0]),c=(a=p,n=u,(r=a.map(e=>X(e)).filter(e=>e&&e!==n)).length?[...new Set(r)].join("\n"):void 0),f=(o=l,o.find(e=>e.bundleId)?.bundleId);return{alert:{...u?{title:u}:{},...c?{message:c}:{},buttons:d.map(e=>e.label),platform:"android",source:i.source,...f?{packageName:f}:{}},buttons:d}}((await i("snapshot_capture",async()=>await v(e,{helperWaitForIdleTimeoutMs:0,includeHiddenContentHints:!1}),{backend:"android",purpose:"alert"})).nodes)}function en(e,t,a){return{kind:"alertHandled",platform:"android",action:e,handled:!0,alert:t,button:a,...C(`Alert ${e}ed`)}}async function es(e){var t;let{req:a,logPath:n,session:s,device:o}=e,r="accept"===(t=a.positionals?.[0])||"dismiss"===t||"wait"===t?t:"get",i=s?"frontmost-app"===s.surface?{surface:"frontmost-app"}:{bundleId:s.appBundleId,surface:s.surface}:{},l=h("alert",o);if(l)return l;if("android"===o.platform){let t=I(a.positionals?.[1])??1e4;return el(e,await Y(o,r,{timeoutMs:t}))}if("macos"===o.platform){let t=async e=>await w(e,i);return await eo(e,r,t)}let d=k({req:a,logPath:n,traceLogPath:s?.trace?.outPath}),p=async e=>await u(o,{command:"alert",action:e,appBundleId:s?.appBundleId},d);return await eo(e,r,p)}async function eo(e,t,a){if("wait"===t)return await er(e,a);let n="accept"===t||"dismiss"===t?t:"get";return"accept"===n||"dismiss"===n?await ei(e,n,a):el(e,await a("get"))}async function er(e,t){let a=I(e.req.positionals?.[1])??1e4,n=Date.now();for(;Date.now()-n<a;){try{return el(e,await t("get"))}catch{}await M(300)}return g("COMMAND_FAILED","alert wait timed out")}async function ei(e,t,a){var n,s;let o,i,l=Date.now();for(;Date.now()-l<2e3;){try{return el(e,await a(t))}catch(t){i=t;let e=String(t?.message??"").toLowerCase();if(!e.includes("alert not found")&&!e.includes("no alert"))break}await M(300)}throw(n=i)instanceof r&&(s=n,(o=String(s?.message??"").toLowerCase()).includes("alert not found")||o.includes("no alert"))?new r(n.code,n.message,{...n.details??{},hint:"If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref."}):n}function el(e,t){return P(e.sessionStore,e.session,e.req,t),{ok:!0,data:t}}async function ed(e){let{req:t,logPath:a,sessionStore:n,session:s,device:o,parsed:r}=e,{setting:i,state:l,appBundleId:d,permissionTarget:p,latitude:u,longitude:c}=r,f=h("settings",o);if(f)return f;if("macos"===o.platform&&!R(i))return g("INVALID_ARGS",A(i));let w=d??s?.appBundleId;if("clear-app-state"===i&&!w)return g("INVALID_ARGS","settings clear-app-state requires an app id when no app is bound to the session");let y="clear-app-state"===i?[i,l,w??""]:"permission"===i?[i,l,p??"",t.positionals?.[3]??"",w??""]:"location"===i&&"set"===l?[i,l,u??"",c??"",w??""]:[i,l,w??""],_=await m(o,"settings",y,t.flags?.out,{...S(a,t.flags,w,s?.trace?.outPath)});return P(n,s,t,_??{setting:i,state:l}),{ok:!0,data:_??{setting:i,state:l}}}async function ep(e){var t,a;let n;if("snapshot"!==e.command||"android"!==e.device.platform)return;let s=o(e.error);if("COMMAND_FAILED"===(t=s).code&&(a=t,n=`${a.message}
+${a.hint??""}`,/Android UI hierarchy dump timed out/i.test(n)||/Stock UIAutomator fallback was skipped/i.test(n)||/Android accessibility snapshots can be blocked/i.test(n)||function(e){if(!e||"object"!=typeof e)return!1;let t=String(e.errorType??""),a=String(e.message??"");return/TimeoutException/i.test(t)||/timed out/i.test(a)}(t.details?.helper)||function(e){var t;if(!e)return!1;let a=e?.timeoutMs,n=e?.cmd,s=Array.isArray(t=e?.args)?t.map(String):"string"==typeof t?t.split(/\s+/):[];return"number"==typeof a&&"adb"===n&&s.includes("uiautomator")&&s.includes("dump")}(t.details)))return{ok:!1,error:{...s,details:{...s.details??{},androidSnapshotTimeoutScreenshot:await eu(e)}}}}async function eu(e){try{var s,r,i;let o=await t.mkdtemp(n.join(a.tmpdir(),"agent-device-android-snapshot-timeout-")),d=n.join(o,"snapshot-timeout-overlay-refs.png"),p=(s=await m(e.device,"screenshot",[d],void 0,{...S(e.logPath,{screenshotNoStabilize:!0},e.session?.appBundleId,e.session?.trace?.outPath),surface:e.session?.surface}),r=d,(i=s,"object"==typeof i&&null!==i&&"path"in i&&"string"==typeof i.path)?s.path:r);await t.access(p);let u=await ec(p,e.session);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_captured",data:{path:p,overlayRefCount:"overlayRefCount"in u?u.overlayRefCount:void 0,overlayRefsAnnotated:"overlayRefsAnnotated"in u?u.overlayRefsAnnotated:void 0}}),u}catch(t){let e=o(t);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_failed",data:{error:e.message}}),{captureFailed:!0,error:e.message}}}async function ec(e,t){if(!t?.snapshot)return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"unavailable",overlayRefCount:0};try{let a=await y({screenshotPath:e,snapshot:t.snapshot});return{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:a.length>0,overlayRefCount:a.length,overlayRefSource:"session-snapshot",overlayRefs:a}}catch(a){let t=o(a);return l({level:"warn",phase:"android_snapshot_timeout_screenshot_overlay_failed",data:{path:e,error:t.message}}),{path:e,overlayRefsRequested:!0,overlayRefsAnnotated:!1,overlayRefSource:"session-snapshot",overlayRefCount:0,overlayAnnotationError:t.message}}}async function ef(e){return await eh({...e,command:"snapshot",unsupportedMessage:"snapshot is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.snapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw,forceFull:a.flags?.snapshotForceFull});return{data:s,record:{kind:"snapshot",nodes:s.nodes.length,truncated:s.truncated}}}})}async function em(e){return await eh({...e,command:"diff",unsupportedMessage:"diff is not supported on this device",execute:async({runtime:e,sessionName:t,req:a,snapshotScope:n})=>{let s=await e.capture.diffSnapshot({session:t,interactiveOnly:a.flags?.snapshotInteractiveOnly,depth:a.flags?.snapshotDepth,scope:n,raw:a.flags?.snapshotRaw});return{data:s,record:{kind:"diff",mode:"snapshot",baselineInitialized:s.baselineInitialized,summary:s.summary}}}})}async function eh(e){var t,a;let{req:n,sessionName:s,logPath:o,sessionStore:i}=e,{session:l,device:d}=await E(i,s,n.flags),p=h(e.command,d,{message:e.unsupportedMessage});if(p)return p;let u=_(n.flags?.snapshotScope,l);if(!u.ok)return u;let m=(t=e.command,a=l,"ios"!==d.platform||a?.appBundleId?null:g("SESSION_NOT_FOUND",`iOS ${t} requires an active app session on the target device. Run open first (for example: open --session ${a?.name??"sim"} --platform ios --device "<name>" <app>).`));return m||await U(l,d,async()=>{var t,a;let p,m,h=function(e){let{req:t,sessionName:a,logPath:n,sessionStore:s,session:o,device:i,snapshotScope:l}=e;return L({backend:function(e){let{req:t,logPath:a,session:n,device:s,snapshotScope:o}=e;return{platform:s.platform,captureSnapshot:async(e,r)=>{let i=await f({device:s,session:n,flags:t.flags,outPath:r?.outPath??t.flags?.out,logPath:a,snapshotScope:o}),l=O(n);return{snapshot:i.snapshot,...N(i),...l?{snapshotDiagnostics:l}:{},appName:n?.appBundleId?n.appName??n.appBundleId:void 0,appBundleId:n?.appBundleId}}}}({req:t,logPath:n,session:o,device:i,snapshotScope:l}),...x("snapshot"),sessions:c({sessionName:a,getSession:()=>s.get(a),recordOptions:{includeSnapshot:!0},setRecord:e=>{var n;let o=function(e){if(!e.snapshot)throw new r("UNKNOWN","snapshot runtime did not produce session state");return e}(e),l=s.get(a);s.set(a,function(e){var t,a;let{current:n,sessionName:s,device:o,record:r,refScopedSnapshot:i}=e,l=(t=n,a=r,i&&a.snapshot?.nodes.length===0&&t?.snapshot!==void 0),d=l?n.snapshot:r.snapshot,p=D({session:n,sessionName:s,device:o,snapshot:d,appBundleId:r.appBundleId});return p.snapshotScopeSource=function(e){let{current:t,keepCurrentSnapshot:a,refScopedSnapshot:n}=e;if(n)return a?t?.snapshotScopeSource:t?.snapshotScopeSource??t?.snapshot}({current:n,keepCurrentSnapshot:l,refScopedSnapshot:i}),r.appName&&(p.appName=r.appName),p}({current:l,sessionName:a,device:i,record:o,refScopedSnapshot:(n=t,n.flags?.snapshotScope?.trim().startsWith("@")===!0)}))}})})}({req:n,sessionName:s,logPath:o,sessionStore:i,session:l,device:d,snapshotScope:u.scope});try{m=await e.execute({runtime:h,sessionName:s,req:n,snapshotScope:u.scope})}catch(a){let t=await ep({error:a,command:e.command,logPath:o,session:l,device:d});if(!t)throw a;return t}return(p=(t={req:n,sessionName:s,sessionStore:i,result:m.record}).sessionStore.get(t.sessionName))&&t.sessionStore.recordAction(p,{command:t.req.command,positionals:t.req.positionals??[],flags:t.req.flags??{},result:"snapshot"===(a=t.result).kind?{nodes:a.nodes,truncated:a.truncated}:{mode:a.mode,baselineInitialized:a.baselineInitialized,summary:a.summary}}),{ok:!0,data:m.data}})}let ew={snapshot:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await ef({req:e,sessionName:t,logPath:a,sessionStore:n}),diff:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>e.positionals?.[0]!=="snapshot"?g("INVALID_ARGS","diff currently supports only: diff snapshot"):await em({req:e,sessionName:t,logPath:a,sessionStore:n}),wait:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>await K({req:e,sessionName:t,logPath:a,sessionStore:n}),alert:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let{session:s,device:o}=await E(n,t,e.flags);return await U(s,o,async()=>await es({req:e,logPath:a,sessionStore:n,session:s,device:o}))},settings:async({req:e,sessionName:t,logPath:a,sessionStore:n})=>{let s,o,r,i=(s=e.positionals?.[0]?.toLowerCase(),o=e.positionals?.[1]?.toLowerCase(),r=e.positionals?.[2]?.toLowerCase(),"clear-app-state"===s?{ok:!0,parsed:{setting:s,state:"clear",appBundleId:"clear"===o?e.positionals?.[2]:e.positionals?.[1]}}:s&&o&&("permission"!==s||r)&&("location"!==s||"set"!==o||e.positionals?.[2]&&e.positionals?.[3])?{ok:!0,parsed:{setting:s,state:o,permissionTarget:r,latitude:e.positionals?.[2],longitude:e.positionals?.[3]}}:g("INVALID_ARGS",b));if(!i.ok)return i;let{session:l,device:d}=await E(n,t,e.flags);return await U(l,d,async()=>await ed({req:e,logPath:a,sessionStore:n,session:l,device:d,parsed:i.parsed}))}};async function eg(e){let t=ew[e.req.command];return t?await t(e):null}export{B as snapshot_namespaceObject};

package/dist/src/web.js

@@ -1 +1 @@
-import{AppError as e}from"./733.js";import{withDiagnosticTimer as t}from"./432.js";import{resolveWebProvider as r}from"./provider.js";function o(){return{open:(e,t)=>r().open(t?.url??e,{url:t?.url}),openDevice:()=>r().open("about:blank"),close:e=>r().close(e),tap:(e,t)=>r().click(e,t),doubleTap:()=>n("doubleTap"),swipe:()=>n("swipe"),pan:()=>n("pan"),fling:()=>n("fling"),longPress:()=>n("longPress"),focus:(e,t)=>r().click(e,t),type:(e,t)=>r().typeText(e,{delayMs:t}),fill:(e,t,o,n)=>r().fill(e,t,o,{delayMs:n}),scroll:(e,t)=>r().scroll(e,t),pinch:()=>n("pinch"),screenshot:(e,t)=>r().screenshot(e,t),setViewport:(e,t)=>r().setViewport(e,t),snapshot:async e=>{let o=await t("snapshot_capture",async()=>await r().snapshot(e),{backend:"web"});return{nodes:o.nodes,truncated:o.truncated??!1,backend:"web"}},back:()=>n("back"),home:()=>n("home"),rotate:()=>n("rotate"),rotateGesture:()=>n("rotateGesture"),transformGesture:()=>n("transformGesture"),appSwitcher:()=>n("appSwitcher"),readClipboard:()=>n("readClipboard"),writeClipboard:()=>n("writeClipboard"),setSetting:()=>n("setSetting")}}async function n(t){throw new e("UNSUPPORTED_OPERATION",`${t} is not supported on web`)}export{o as createWebInteractor};
+import{AppError as e}from"./485.js";import{withDiagnosticTimer as t}from"./432.js";import{resolveWebProvider as r}from"./provider.js";function o(){return{open:(e,t)=>r().open(t?.url??e,{url:t?.url}),openDevice:()=>r().open("about:blank"),close:e=>r().close(e),tap:(e,t)=>r().click(e,t),doubleTap:()=>n("doubleTap"),swipe:()=>n("swipe"),pan:()=>n("pan"),fling:()=>n("fling"),longPress:()=>n("longPress"),focus:(e,t)=>r().click(e,t),type:(e,t)=>r().typeText(e,{delayMs:t}),fill:(e,t,o,n)=>r().fill(e,t,o,{delayMs:n}),scroll:(e,t)=>r().scroll(e,t),pinch:()=>n("pinch"),screenshot:(e,t)=>r().screenshot(e,t),setViewport:(e,t)=>r().setViewport(e,t),snapshot:async e=>{let o=await t("snapshot_capture",async()=>await r().snapshot(e),{backend:"web"});return{nodes:o.nodes,truncated:o.truncated??!1,backend:"web"}},back:()=>n("back"),home:()=>n("home"),rotate:()=>n("rotate"),rotateGesture:()=>n("rotateGesture"),transformGesture:()=>n("transformGesture"),appSwitcher:()=>n("appSwitcher"),readClipboard:()=>n("readClipboard"),writeClipboard:()=>n("writeClipboard"),setSetting:()=>n("setSetting")}}async function n(t){throw new e("UNSUPPORTED_OPERATION",`${t} is not supported on web`)}export{o as createWebInteractor};

package/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift

@@ -0,0 +1,297 @@
+import AVFoundation
+import AudioToolbox
+import CoreMedia
+import Foundation
+import ScreenCaptureKit
+
+private let audioProbeSilenceDb = -90
+
+struct AudioProbeResponse: Codable {
+  let audio: String
+  let state: String
+  let active: Bool
+  let heard: Bool
+  let source: String
+  let backend: String
+  let durationMs: Int
+  let elapsedMs: Int
+  let bucketMs: Int
+  let sampleCount: Int
+  let sourceCount: Int
+  let rmsDbfs: [Int]
+  let peakDbfs: [Int]
+  let startedAt: String
+  let stoppedAt: String?
+  let reason: String?
+}
+
+private struct AudioProbeBucket {
+  var totalSquares: Double = 0
+  var totalSamples: Int = 0
+  var peak: Double = 0
+}
+
+private final class AudioProbeStatusWriter {
+  private let outPath: String
+  private let startedAt = Date()
+  private let durationMs: Int
+  private let bucketMs: Int
+  private let lock = NSLock()
+  private var current = AudioProbeBucket()
+  private var rmsDbfs: [Int] = []
+  private var peakDbfs: [Int] = []
+  private var heard = false
+  private var stoppedAt: Date?
+  private var reason: String?
+
+  init(outPath: String, durationMs: Int, bucketMs: Int) {
+    self.outPath = outPath
+    self.durationMs = durationMs
+    self.bucketMs = bucketMs
+  }
+
+  func add(sampleBuffer: CMSampleBuffer) {
+    guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else {
+      return
+    }
+    let format = AVAudioFormat(cmAudioFormatDescription: formatDescription)
+    let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
+    guard frameCount > 0,
+      let pcmBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount)
+    else {
+      return
+    }
+    let status = CMSampleBufferCopyPCMDataIntoAudioBufferList(
+      sampleBuffer,
+      at: 0,
+      frameCount: Int32(frameCount),
+      into: pcmBuffer.mutableAudioBufferList
+    )
+    guard status == noErr else {
+      return
+    }
+    pcmBuffer.frameLength = frameCount
+    add(audioBufferList: pcmBuffer.mutableAudioBufferList, format: format)
+  }
+
+  func flushRunning() throws {
+    lock.lock()
+    appendCurrentBucket()
+    let response = buildResponse(state: "running", active: true)
+    lock.unlock()
+    try write(response)
+  }
+
+  func finish(reason: String) throws {
+    lock.lock()
+    if current.totalSamples > 0 || rmsDbfs.isEmpty {
+      appendCurrentBucket()
+    }
+    self.stoppedAt = Date()
+    self.reason = reason
+    let response = buildResponse(state: "stopped", active: false)
+    lock.unlock()
+    try write(response)
+  }
+
+  private func add(audioBufferList: UnsafeMutablePointer<AudioBufferList>, format: AVAudioFormat) {
+    let streamDescription = format.streamDescription.pointee
+    let bitsPerChannel = Int(streamDescription.mBitsPerChannel)
+    guard bitsPerChannel > 0 else {
+      return
+    }
+    let bytesPerSample = max(1, bitsPerChannel / 8)
+    let isFloat = (streamDescription.mFormatFlags & kAudioFormatFlagIsFloat) != 0
+    let isSignedInteger = (streamDescription.mFormatFlags & kAudioFormatFlagIsSignedInteger) != 0
+    var bucket = AudioProbeBucket()
+    for buffer in UnsafeMutableAudioBufferListPointer(audioBufferList) {
+      guard let data = buffer.mData else {
+        continue
+      }
+      let sampleCount = Int(buffer.mDataByteSize) / bytesPerSample
+      if isFloat && bytesPerSample == 4 {
+        let samples = data.bindMemory(to: Float.self, capacity: sampleCount)
+        for index in 0..<sampleCount {
+          bucket.add(Double(samples[index]))
+        }
+      } else if isSignedInteger && bytesPerSample == 2 {
+        let samples = data.bindMemory(to: Int16.self, capacity: sampleCount)
+        for index in 0..<sampleCount {
+          bucket.add(Double(samples[index]) / Double(Int16.max))
+        }
+      } else if isSignedInteger && bytesPerSample == 4 {
+        let samples = data.bindMemory(to: Int32.self, capacity: sampleCount)
+        for index in 0..<sampleCount {
+          bucket.add(Double(samples[index]) / Double(Int32.max))
+        }
+      }
+    }
+    guard bucket.totalSamples > 0 else {
+      return
+    }
+    lock.lock()
+    current.totalSquares += bucket.totalSquares
+    current.totalSamples += bucket.totalSamples
+    current.peak = max(current.peak, bucket.peak)
+    lock.unlock()
+  }
+
+  private func appendCurrentBucket() {
+    let rms = current.totalSamples > 0
+      ? sqrt(current.totalSquares / Double(current.totalSamples))
+      : 0
+    let rmsDb = dbfs(rms)
+    let peakDb = dbfs(current.peak)
+    rmsDbfs.append(rmsDb)
+    peakDbfs.append(peakDb)
+    if rmsDb > audioProbeSilenceDb || peakDb > audioProbeSilenceDb {
+      heard = true
+    }
+    current = AudioProbeBucket()
+  }
+
+  private func buildResponse(state: String, active: Bool) -> AudioProbeResponse {
+    let end = stoppedAt ?? Date()
+    let elapsed = min(durationMs, max(0, Int(end.timeIntervalSince(startedAt) * 1000)))
+    return AudioProbeResponse(
+      audio: "probe",
+      state: state,
+      active: active,
+      heard: heard,
+      source: "system-audio",
+      backend: "macos-screencapturekit",
+      durationMs: durationMs,
+      elapsedMs: elapsed,
+      bucketMs: bucketMs,
+      sampleCount: rmsDbfs.count,
+      sourceCount: 1,
+      rmsDbfs: rmsDbfs,
+      peakDbfs: peakDbfs,
+      startedAt: iso8601(startedAt),
+      stoppedAt: stoppedAt.map(iso8601),
+      reason: reason
+    )
+  }
+
+  private func write(_ response: AudioProbeResponse) throws {
+    let outputURL = URL(fileURLWithPath: outPath)
+    try FileManager.default.createDirectory(
+      at: outputURL.deletingLastPathComponent(),
+      withIntermediateDirectories: true
+    )
+    let data = try JSONEncoder().encode(response)
+    try data.write(to: outputURL, options: .atomic)
+  }
+}
+
+private final class AudioProbeStreamOutput: NSObject, SCStreamOutput {
+  private let writer: AudioProbeStatusWriter
+
+  init(writer: AudioProbeStatusWriter) {
+    self.writer = writer
+  }
+
+  func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+    guard type == .audio else {
+      return
+    }
+    writer.add(sampleBuffer: sampleBuffer)
+  }
+}
+
+extension AudioProbeBucket {
+  fileprivate mutating func add(_ value: Double) {
+    guard value.isFinite else {
+      return
+    }
+    let clipped = min(1, max(-1, value))
+    totalSquares += clipped * clipped
+    totalSamples += 1
+    peak = max(peak, abs(clipped))
+  }
+}
+
+func runAudioProbe(durationMs: Int, bucketMs: Int, outPath: String) throws -> AudioProbeResponse {
+  guard #available(macOS 13.0, *) else {
+    throw HelperError.commandFailed("audio probe requires macOS 13 or newer")
+  }
+  let semaphore = DispatchSemaphore(value: 0)
+  var response: AudioProbeResponse?
+  var runError: Error?
+  Task {
+    do {
+      response = try await runAudioProbeAsync(durationMs: durationMs, bucketMs: bucketMs, outPath: outPath)
+    } catch {
+      runError = error
+    }
+    semaphore.signal()
+  }
+  semaphore.wait()
+  if let runError {
+    throw runError
+  }
+  guard let response else {
+    throw HelperError.commandFailed("audio probe failed")
+  }
+  return response
+}
+
+@available(macOS 13.0, *)
+private func runAudioProbeAsync(durationMs: Int, bucketMs: Int, outPath: String) async throws -> AudioProbeResponse {
+  let content: SCShareableContent
+  do {
+    content = try await SCShareableContent.current
+  } catch {
+    throw HelperError.commandFailed(
+      "audio probe requires Screen Recording permission on macOS",
+      details: ["permission": "screen-recording", "error": error.localizedDescription]
+    )
+  }
+  guard let display = content.displays.first else {
+    throw HelperError.commandFailed("audio probe could not resolve a macOS display")
+  }
+
+  let configuration = SCStreamConfiguration()
+  configuration.width = max(2, display.width)
+  configuration.height = max(2, display.height)
+  configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
+  configuration.queueDepth = 1
+  configuration.capturesAudio = true
+  configuration.sampleRate = 48_000
+  configuration.channelCount = 2
+  configuration.excludesCurrentProcessAudio = true
+
+  let filter = SCContentFilter(display: display, excludingWindows: [])
+  let writer = AudioProbeStatusWriter(outPath: outPath, durationMs: durationMs, bucketMs: bucketMs)
+  let output = AudioProbeStreamOutput(writer: writer)
+  let stream = SCStream(filter: filter, configuration: configuration, delegate: nil)
+  let queue = DispatchQueue(label: "com.callstack.agent-device.audio-probe")
+  try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: queue)
+  try await stream.startCapture()
+  try writer.flushRunning()
+
+  let deadline = Date().addingTimeInterval(Double(durationMs) / 1000)
+  while Date() < deadline {
+    let remainingMs = Int(deadline.timeIntervalSinceNow * 1000)
+    let sleepMs = max(1, min(bucketMs, remainingMs))
+    try await Task.sleep(nanoseconds: UInt64(sleepMs) * 1_000_000)
+    try writer.flushRunning()
+  }
+
+  try await stream.stopCapture()
+  try writer.finish(reason: "completed")
+  let data = try Data(contentsOf: URL(fileURLWithPath: outPath))
+  return try JSONDecoder().decode(AudioProbeResponse.self, from: data)
+}
+
+private func dbfs(_ value: Double) -> Int {
+  if !value.isFinite || value <= 0 {
+    return audioProbeSilenceDb
+  }
+  let db = Int((20 * log10(value)).rounded())
+  return max(audioProbeSilenceDb, min(0, db))
+}
+
+private func iso8601(_ date: Date) -> String {
+  ISO8601DateFormatter().string(from: date)
+}

package/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift

@@ -120,6 +120,8 @@ struct AgentDeviceMacOSHelper {
       return try handlePress(arguments: Array(arguments.dropFirst()))
     case "screenshot":
       return try handleScreenshot(arguments: Array(arguments.dropFirst()))
+    case "audio-probe":
+      return try handleAudioProbe(arguments: Array(arguments.dropFirst()))
     default:
       throw HelperError.invalidArgs("unknown command: \(command)")
     }
@@ -390,6 +392,27 @@ struct AgentDeviceMacOSHelper {
     try captureSurfaceScreenshot(surface: surface, outPath: outPath, fullscreen: fullscreen)
     return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface, fullscreen: fullscreen))
   }
+
+  static func handleAudioProbe(arguments: [String]) throws -> any Encodable {
+    let durationMs = intOption(arguments: arguments, name: "--duration-ms") ?? 10_000
+    let bucketMs = intOption(arguments: arguments, name: "--bucket-ms") ?? 1_000
+    guard durationMs >= 100, durationMs <= 120_000 else {
+      throw HelperError.invalidArgs("audio-probe --duration-ms must be in range 100..120000")
+    }
+    guard bucketMs >= 100, bucketMs <= 10_000 else {
+      throw HelperError.invalidArgs("audio-probe --bucket-ms must be in range 100..10000")
+    }
+    guard let outPath = optionValue(arguments: arguments, name: "--out")?
+      .trimmingCharacters(in: .whitespacesAndNewlines),
+      !outPath.isEmpty
+    else {
+      throw HelperError.invalidArgs("audio-probe requires --out <path>")
+    }
+
+    return SuccessEnvelope(
+      data: try runAudioProbe(durationMs: durationMs, bucketMs: bucketMs, outPath: outPath)
+    )
+  }
 }
 
 private func optionValue(arguments: [String], name: String) -> String? {
@@ -399,6 +422,13 @@ private func optionValue(arguments: [String], name: String) -> String? {
   return arguments[index + 1]
 }
 
+private func intOption(arguments: [String], name: String) -> Int? {
+  guard let value = optionValue(arguments: arguments, name: name) else {
+    return nil
+  }
+  return Int(value)
+}
+
 private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String {
   let targetApp: NSRunningApplication?
   if surface == "frontmost-app" || (surface == nil && bundleId != nil) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device",
-  "version": "0.17.10",
+  "version": "0.18.1",
   "description": "Agent-native CLI for AI app automation across iOS, Android, tvOS, Android TV, macOS, Linux, and web.",
   "mcpName": "io.github.callstackincubator/agent-device",
   "license": "MIT",
@@ -50,10 +50,6 @@
       "import": "./dist/src/android-adb.js",
       "types": "./dist/src/android-adb.d.ts"
     },
-    "./android-snapshot-helper": {
-      "import": "./dist/src/android-snapshot-helper.js",
-      "types": "./dist/src/android-snapshot-helper.d.ts"
-    },
     "./contracts": {
       "import": "./dist/src/contracts.js",
       "types": "./dist/src/contracts.d.ts"
@@ -81,6 +77,7 @@
     "clean:xcuitest:ios": "node scripts/clean-xcuitest-derived.mjs ios",
     "clean:xcuitest:macos": "node scripts/clean-xcuitest-derived.mjs macos",
     "clean:xcuitest:tvos": "node scripts/clean-xcuitest-derived.mjs tvos",
+    "clean:xcuitest:visionos": "node scripts/clean-xcuitest-derived.mjs visionos",
     "build:node": "pnpm build && pnpm clean:daemon",
     "build:xcuitest": "pnpm build:xcuitest:ios && pnpm build:xcuitest:macos",
     "build:xcuitest:ios": "AGENT_DEVICE_XCUITEST_PLATFORM=ios sh ./scripts/build-xcuitest-apple.sh",
@@ -88,6 +85,8 @@
     "build:xcuitest:macos": "AGENT_DEVICE_XCUITEST_PLATFORM=macos sh ./scripts/build-xcuitest-apple.sh",
     "build:xcuitest:tvos": "AGENT_DEVICE_XCUITEST_PLATFORM=tvos sh ./scripts/build-xcuitest-apple.sh",
     "build:xcuitest:tvos:clean": "pnpm clean:xcuitest:tvos && pnpm build:xcuitest:tvos",
+    "build:xcuitest:visionos": "AGENT_DEVICE_XCUITEST_PLATFORM=visionos sh ./scripts/build-xcuitest-apple.sh",
+    "build:xcuitest:visionos:clean": "pnpm clean:xcuitest:visionos && pnpm build:xcuitest:visionos",
     "build:android-snapshot-helper": "sh ./scripts/build-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") .tmp/android-snapshot-helper",
     "package:android-snapshot-helper": "sh ./scripts/package-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") v$(node -p \"require('./package.json').version\") .tmp/android-snapshot-helper",
     "package:android-snapshot-helper:npm": "rm -rf android-snapshot-helper/dist && sh ./scripts/package-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") v$(node -p \"require('./package.json').version\") android-snapshot-helper/dist",
@@ -109,11 +108,12 @@
     "fallow:all": "fallow --summary",
     "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",
     "check:fallow": "fallow audit",
+    "check:layering": "node --experimental-strip-types scripts/layering/check.ts",
     "check:quick": "pnpm lint && pnpm typecheck",
     "sync:mcp-metadata": "node scripts/sync-mcp-metadata.mjs",
     "check:mcp-metadata": "node scripts/sync-mcp-metadata.mjs --check",
     "version": "node scripts/sync-mcp-metadata.mjs && git add server.json",
-    "check:tooling": "pnpm lint && pnpm typecheck && pnpm check:mcp-metadata && pnpm build",
+    "check:tooling": "pnpm lint && pnpm typecheck && pnpm check:layering && pnpm check:mcp-metadata && pnpm build",
     "check:unit": "pnpm test:unit && pnpm test:smoke",
     "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
     "prepack": "pnpm check:mcp-metadata && pnpm build:all && pnpm package:android-snapshot-helper:npm && pnpm package:android-multitouch-helper:npm",
@@ -149,17 +149,21 @@
   "files": [
     "bin",
     "dist",
-    "ios-runner",
-    "!ios-runner/**/.build",
-    "!ios-runner/**/.swiftpm",
-    "!ios-runner/**/xcuserdata",
-    "!ios-runner/**/*.xcuserstate",
+    "apple-runner",
+    "!apple-runner/**/.build",
+    "!apple-runner/**/.swiftpm",
+    "!apple-runner/**/xcuserdata",
+    "!apple-runner/**/*.xcuserstate",
+    "!apple-runner/README.md",
+    "!apple-runner/RUNNER_PROTOCOL.md",
     "macos-helper",
     "!macos-helper/**/.build",
     "android-snapshot-helper/dist",
     "!android-snapshot-helper/dist/*.idsig",
+    "!android-snapshot-helper/README.md",
     "android-multitouch-helper/dist",
     "!android-multitouch-helper/dist/*.idsig",
+    "!android-multitouch-helper/README.md",
     "src/platforms/linux/atspi-dump.py",
     "skills",
     "server.json",

package/server.json

@@ -7,12 +7,12 @@
     "url": "https://github.com/callstack/agent-device",
     "source": "github"
   },
-  "version": "0.17.10",
+  "version": "0.18.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "agent-device",
-      "version": "0.17.10",
+      "version": "0.18.1",
       "transport": {
         "type": "stdio"
       }

package/android-multitouch-helper/README.md

@@ -1,41 +0,0 @@
-# Android MultiTouch Helper
-
-Small instrumentation APK used to inject Android touch gestures through
-`UiAutomation.injectInputEvent`. The helper accepts a compact base64 JSON payload so local ADB,
-remote ADB tunnels, and remote providers that allow `adb install -t` plus `am instrument` can use
-the same contract.
-
-The helper is separate from `android-snapshot-helper` because the payload and output protocol are
-gesture-specific. The install/version/cache lifecycle should stay aligned with the snapshot helper.
-
-## Build
-
-```sh
-VERSION="$(node -p 'require("./package.json").version')"
-sh ./scripts/build-android-multitouch-helper.sh "$VERSION" .tmp/android-multitouch-helper
-```
-
-## Run
-
-```sh
-PAYLOAD="$(printf '%s' '{"kind":"transform","x":672,"y":1500,"dx":80,"dy":-40,"scale":1.8,"degrees":35,"durationMs":700}' | base64)"
-adb install -r -t ".tmp/android-multitouch-helper/agent-device-android-multitouch-helper-$VERSION.apk"
-adb shell am instrument -w \
-  -e payloadBase64 "$PAYLOAD" \
-  com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation
-```
-
-## Output Contract
-
-The APK emits instrumentation result records using
-`agentDeviceProtocol=android-multitouch-helper-v1`.
-
-Successful results include:
-
-- `ok=true`
-- `helperApiVersion=1`
-- `kind` (`swipe`, `pinch`, `rotate`, or `transform`)
-- `injectedEvents`
-- `elapsedMs`
-
-Failures return `ok=false`, `errorType`, and `message`.

package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.17.10.apk.sha256

@@ -1 +0,0 @@
-e3c1bb4b43e7e4bbcceaa4975ec7c00f606208a435d575209a9f0819a7adc5de  agent-device-android-multitouch-helper-0.17.10.apk