agent-device 0.14.9 → 0.15.1

package/dist/src/remote-config.d.ts

@@ -1,4 +1,20 @@
-export declare type RemoteConfigProfile = {
+declare type RemoteConfigMetroOptions = {
+    metroProjectRoot?: string;
+    metroKind?: 'auto' | 'react-native' | 'expo';
+    metroPublicBaseUrl?: string;
+    metroProxyBaseUrl?: string;
+    metroBearerToken?: string;
+    metroPreparePort?: number;
+    metroListenHost?: string;
+    metroStatusHost?: string;
+    metroStartupTimeoutMs?: number;
+    metroProbeTimeoutMs?: number;
+    metroRuntimeFile?: string;
+    metroNoReuseExisting?: boolean;
+    metroNoInstallDeps?: boolean;
+};
+
+export declare type RemoteConfigProfile = RemoteConfigMetroOptions & {
     stateDir?: string;
     daemonBaseUrl?: string;
     daemonAuthToken?: string;
@@ -17,19 +33,6 @@ export declare type RemoteConfigProfile = {
     iosSimulatorDeviceSet?: string;
     androidDeviceAllowlist?: string;
     session?: string;
-    metroProjectRoot?: string;
-    metroKind?: 'auto' | 'react-native' | 'expo';
-    metroPublicBaseUrl?: string;
-    metroProxyBaseUrl?: string;
-    metroBearerToken?: string;
-    metroPreparePort?: number;
-    metroListenHost?: string;
-    metroStatusHost?: string;
-    metroStartupTimeoutMs?: number;
-    metroProbeTimeoutMs?: number;
-    metroRuntimeFile?: string;
-    metroNoReuseExisting?: boolean;
-    metroNoInstallDeps?: boolean;
 };
 
 export declare type RemoteConfigProfileOptions = {

package/dist/src/selectors.d.ts

@@ -46,6 +46,7 @@ declare type RawSnapshotNode = {
     surface?: string;
     hiddenContentAbove?: boolean;
     hiddenContentBelow?: boolean;
+    presentationHints?: string[];
 };
 
 declare type Rect = {
@@ -104,6 +105,7 @@ declare type SnapshotState = {
     truncated?: boolean;
     backend?: SnapshotBackend;
     comparisonSafe?: boolean;
+    presentationKey?: string;
 };
 
 export declare function tryParseSelectorChain(expression: string): SelectorChain | null;

package/dist/src/server.js

@@ -1,20 +1,2 @@
-import{buildCommandUsageText as e,buildUsageText as t}from"./command-schema.js";import{readVersion as r}from"./9671.js";let n="agent-device",o=["workflow","debugging","react-devtools","remote","macos","dogfood"];function i(e={}){let t=!1===e.global?"npx -y agent-device mcp":"agent-device mcp",r=!1===e.global?"No global install required.":"npm install -g agent-device",n=e.client?`
-Client hint: ${e.client}`:"";return`${r}
-
-MCP server command:
-  ${t}
-
-Generic MCP JSON:
-  {
-    "mcpServers": {
-      "agent-device": {
-        "command": "${!1===e.global?"npx":"agent-device"}",
-        "args": ${JSON.stringify(!1===e.global?["-y","agent-device","mcp"]:["mcp"])}
-      }
-    }
-  }
-
-Use this server for discovery and routing only. For device actions, call the CLI commands returned by the help tool, starting with:
-  agent-device help workflow${n}
-`}function a(r={}){if(r.topic&&r.command)throw Error("Provide either topic or command, not both.");let n=r.topic??r.command;if(!n)return t();if(r.topic&&!d(r.topic))throw Error(`Unknown help topic: ${r.topic}. Expected one of: ${o.join(", ")}`);let i=e(n);if(!i)throw Error(`Unknown command or help topic: ${n}`);return i}function s(){return[{uri:"agent-device://install",name:"agent-device MCP install",description:"Install and client configuration snippets.",mimeType:"text/markdown"},{uri:"agent-device://help",name:"agent-device command list",description:"Version-matched command list and global flags.",mimeType:"text/plain"},...o.map(e=>({uri:`agent-device://help/${e}`,name:`agent-device help ${e}`,description:`Version-matched ${e} workflow guidance.`,mimeType:"text/plain"}))]}function c(){return[l("agent-device-workflow","Plan a normal app automation loop."),l("agent-device-debugging","Collect focused debugging evidence."),l("agent-device-dogfood","Run exploratory QA with reproducible evidence."),l("agent-device-react-native-performance","Inspect React Native renders."),l("agent-device-macos","Inspect a macOS app or surface.")]}function l(e,t){return{name:e,description:t,arguments:[{name:"target",description:"Optional app, device, or task target.",required:!1}]}}function d(e){return o.includes(e)}function p(e){if("2.0"!==e.jsonrpc||"string"!=typeof e.method)return g(e.id??null,-32600,"Invalid JSON-RPC request.");if(void 0===e.id)return null;try{var l,p;return l=e.id,p=function(e,l){switch(e){case"initialize":return{protocolVersion:"2025-11-25",capabilities:{tools:{},resources:{},prompts:{}},serverInfo:{name:n,version:r()}};case"ping":return{};case"tools/list":return{tools:[{name:"status",description:"Report the installed agent-device MCP router and CLI metadata.",inputSchema:{type:"object",properties:{},additionalProperties:!1}},{name:"install",description:"Return install and MCP client configuration snippets for agent-device.",inputSchema:{type:"object",properties:{client:{type:"string",description:"Optional client name for labeling the returned guidance."},global:{type:"boolean",description:"Use a global agent-device binary when true; use npx when false."}},additionalProperties:!1}},{name:"help",description:"Return version-matched CLI help for a workflow topic or command.",inputSchema:{type:"object",properties:{topic:{type:"string",enum:o,description:"Agent workflow topic."},command:{type:"string",description:"CLI command name such as snapshot, open, logs, or react-devtools."}},additionalProperties:!1}}]};case"tools/call":var p,g,w,y=l;let b=m(y),$=h(b,"name"),k=f(b.arguments);try{if("status"===$)return u(JSON.stringify({name:n,registryName:"io.github.callstackincubator/agent-device",version:r(),transport:"stdio",command:"agent-device mcp",node:process.version,capabilities:{tools:["status","install","help"],resources:s().map(e=>e.uri),prompts:c().map(e=>e.name)},note:"This MCP server routes agents to the agent-device CLI. It does not expose device automation or shell execution tools."},null,2));if("install"===$)return u(i(k));if("help"===$)return u(a(k));throw Error(`Unknown tool: ${$}`)}catch(e){return u(e instanceof Error?e.message:String(e),!0)}return;case"resources/list":return{resources:s()};case"resources/read":let x;return p=l,{contents:[{uri:x=h(m(p),"uri"),mimeType:"agent-device://install"===x?"text/markdown":"text/plain",text:function(e){if("agent-device://install"===e)return i();if("agent-device://help"===e)return t();let r=e.startsWith("agent-device://help/")?e.slice(20):"";if(d(r))return a({topic:r});throw Error(`Unknown resource: ${e}`)}(x)}]};case"prompts/list":return{prompts:c()};case"prompts/get":let C;return g=l,C=m(g),function(e,t={}){let r={"agent-device-workflow":"workflow","agent-device-debugging":"debugging","agent-device-dogfood":"dogfood","agent-device-react-native-performance":"react-devtools","agent-device-macos":"macos"}[e];if(!r)throw Error(`Unknown prompt: ${e}`);let n=t.target?` Target: ${t.target}.`:"";return{description:`Use agent-device help ${r} before planning commands.${n}`,messages:[{role:"user",content:{type:"text",text:`Read the agent-device ${r} guidance through the MCP help tool, then produce a concise command plan using agent-device CLI commands only.${n}`}}]}}(h(C,"name"),(w=C.arguments,Object.fromEntries(Object.entries(f(w)).filter(e=>"string"==typeof e[1]))));default:throw new v(`Unsupported MCP method: ${e}`)}}(e.method,e.params),{jsonrpc:"2.0",id:l,result:p}}catch(t){if(t instanceof v)return g(e.id,-32601,t.message);return g(e.id,-32602,t instanceof Error?t.message:String(t))}}function u(e,t=!1){return{isError:t,content:[{type:"text",text:e}]}}function g(e,t,r){return{jsonrpc:"2.0",id:e,error:{code:t,message:r}}}function m(e){if(!e||"object"!=typeof e||Array.isArray(e))throw Error("Expected object parameters.");return e}function f(e){return void 0===e?{}:m(e)}function h(e,t){let r=e[t];if("string"!=typeof r||0===r.length)throw Error(`Expected ${t} to be a non-empty string.`);return r}class v extends Error{}async function w(){let e=new y(e=>{let t=function(e){if(Array.isArray(e)){let t=e.flatMap(e=>{var t;return(t=p(e))?[t]:[]});return t.length>0?t:null}return p(e)}(e);t&&b(t)});process.stdin.setEncoding("utf8"),process.stdin.on("data",t=>{try{e.push(t)}catch(e){b({jsonrpc:"2.0",id:null,error:{code:-32700,message:e instanceof Error?e.message:String(e)}})}}),await new Promise(e=>{process.stdin.on("end",e),process.stdin.on("close",e),process.stdin.resume()})}class y{buffer="";sink;constructor(e){this.sink=e}push(e){for(this.buffer+=e;;){let e=this.tryReadLineMessage();if(void 0!==e){this.emit(e);continue}break}}tryReadLineMessage(){let e=this.buffer.indexOf("\n");if(-1===e)return;let t=this.buffer.slice(0,e).trim();return this.buffer=this.buffer.slice(e+1),t.length>0?t:void 0}emit(e){let t=JSON.parse(e);Array.isArray(t),this.sink(t)}}function b(e){process.stdout.write(`${JSON.stringify(e)}
-`)}export{w as runAgentDeviceMcpServer};
+import{readVersion as e}from"./9671.js";function t(t){if("2.0"!==t.jsonrpc||"string"!=typeof t.method)return r(t.id??null,-32600,"Invalid JSON-RPC request.");if(void 0===t.id)return null;try{var i,s;return i=t.id,s=function(t,r){switch(t){case"initialize":return{protocolVersion:"2025-11-25",capabilities:{tools:{}},serverInfo:{name:"agent-device",version:e()}};case"ping":return{};case"tools/list":return{tools:[{name:"status",description:"Return discovery-only handoff metadata for installing, verifying, and using the agent-device CLI.",inputSchema:{type:"object",properties:{},additionalProperties:!1},outputSchema:{type:"object",properties:{packageName:{type:"string"},installedPackageVersion:{type:"string"},cliCommandName:{type:"string"},installCommand:{type:"string"},verifyCommand:{type:"string"},startingHelpCommand:{type:"string"},supportedTargets:{type:"array",items:{type:"string"}},capabilities:{type:"array",items:{type:"string"}},prerequisites:{type:"array",items:{type:"string"}},docsUrl:{type:"string"},agentDocsUrl:{type:"string"},firstCommands:{type:"array",items:{type:"string"}},automationInterface:{type:"string",const:"cli"},automationNote:{type:"string"},installRequiresHumanApproval:{type:"boolean",const:!0},installSafetyNote:{type:"string"}},required:["packageName","installedPackageVersion","cliCommandName","installCommand","verifyCommand","startingHelpCommand","supportedTargets","capabilities","prerequisites","docsUrl","agentDocsUrl","firstCommands","automationInterface","automationNote","installRequiresHumanApproval","installSafetyNote"],additionalProperties:!1}}]};case"tools/call":var i=r;let s=function(e,t){let r=e[t];if("string"!=typeof r||0===r.length)throw Error(`Expected ${t} to be a non-empty string.`);return r}(function(e){if(!e||"object"!=typeof e||Array.isArray(e))throw Error("Expected object parameters.");return e}(i),"name");try{if("status"===s){let t;return t={packageName:"agent-device",installedPackageVersion:e(),cliCommandName:"agent-device",installCommand:"npm install -g agent-device@latest",verifyCommand:"agent-device --version",startingHelpCommand:"agent-device help workflow",supportedTargets:["ios-simulator","android-emulator","ios-device","android-device","tvos-simulator","macos","linux"],capabilities:["inspect-ui","interact-with-elements","open-apps","install-app","capture-screenshot","accessibility-snapshot","collect-logs","collect-network","collect-performance","record-replay","react-native","expo","android-adb","ios-xcuitest"],prerequisites:["node>=22","xcode-for-ios","android-sdk-adb-for-android","macos-accessibility-permission-for-desktop"],docsUrl:"https://agent-device.dev/",agentDocsUrl:"https://incubator.callstack.com/agent-device/llms-full.txt",firstCommands:["agent-device help workflow","agent-device apps --platform ios","agent-device apps --platform android"],automationInterface:"cli",automationNote:"Device automation happens through the agent-device CLI. This MCP server is discovery-only and does not expose device automation tools.",installRequiresHumanApproval:!0,installSafetyNote:"Agents should not install or update the package unless the human has approved the environment change. If the CLI is missing, ask the human to run the install command, then run the verify command."},{isError:!1,structuredContent:t,content:[{type:"text",text:JSON.stringify(t,null,2)}]}}throw Error(`Unknown tool: ${s}`)}catch(e){return function(e,t=!1){return{isError:t,content:[{type:"text",text:e}]}}(e instanceof Error?e.message:String(e),!0)}return;default:throw new n(`Unsupported MCP method: ${t}`)}}(t.method,t.params),{jsonrpc:"2.0",id:i,result:s}}catch(e){if(e instanceof n)return r(t.id,-32601,e.message);return r(t.id,-32602,e instanceof Error?e.message:String(e))}}function r(e,t,r){return{jsonrpc:"2.0",id:e,error:{code:t,message:r}}}class n extends Error{}async function i(){let e=new s(e=>{let r=function(e){if(Array.isArray(e)){let r=e.flatMap(e=>{var r;return(r=t(e))?[r]:[]});return r.length>0?r:null}return t(e)}(e);r&&a(r)});process.stdin.setEncoding("utf8"),process.stdin.on("data",t=>{try{e.push(t)}catch(e){a({jsonrpc:"2.0",id:null,error:{code:-32700,message:e instanceof Error?e.message:String(e)}})}}),await new Promise(e=>{process.stdin.on("end",e),process.stdin.on("close",e),process.stdin.resume()})}class s{buffer="";sink;constructor(e){this.sink=e}push(e){for(this.buffer+=e;;){let e=this.tryReadLineMessage();if(void 0!==e){this.emit(e);continue}break}}tryReadLineMessage(){let e=this.buffer.indexOf("\n");if(-1===e)return;let t=this.buffer.slice(0,e).trim();return this.buffer=this.buffer.slice(e+1),t.length>0?t:void 0}emit(e){let t=JSON.parse(e);Array.isArray(t),this.sink(t)}}function a(e){process.stdout.write(`${JSON.stringify(e)}
+`)}export{i as runAgentDeviceMcpServer};

package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme

@@ -28,7 +28,13 @@
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       shouldUseLaunchSchemeArgsEnv = "YES"
-      shouldAutocreateTestPlan = "YES">
+      shouldAutocreateTestPlan = "NO">
+      <TestPlans>
+         <TestPlanReference
+            reference = "container:AgentDeviceRunnerUITests.xctestplan"
+            default = "YES">
+         </TestPlanReference>
+      </TestPlans>
       <Testables>
          <TestableReference
             skipped = "NO"

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -0,0 +1,155 @@
+import XCTest
+
+extension RunnerTests {
+  struct RunnerAlert {
+    let root: XCUIElement
+    let ownerApp: XCUIApplication
+    let buttons: [XCUIElement]
+  }
+
+  func resolveAlert(app activeApp: XCUIApplication) -> RunnerAlert? {
+    if let alert = firstExistingElement(in: activeApp.alerts.allElementsBoundByIndex) {
+      return runnerAlert(root: alert, ownerApp: activeApp)
+    }
+    if let popup = firstDismissPopupWindow(in: activeApp) {
+      return runnerAlert(root: popup, ownerApp: activeApp)
+    }
+#if os(macOS)
+    return nil
+#else
+    if let systemModal = firstBlockingSystemModal(in: springboard) {
+      return runnerAlert(root: systemModal, ownerApp: springboard)
+    }
+    return nil
+#endif
+  }
+
+  func handleAlert(_ alert: RunnerAlert, action: String) -> Response {
+    if action == "accept" || action == "dismiss" {
+      guard let button = chooseAlertButton(alert.buttons, action: action) else {
+        return Response(ok: false, error: ErrorPayload(message: "alert \(action) button not found"))
+      }
+      let outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)")
+      if let response = unsupportedResponse(for: outcome) {
+        return response
+      }
+      return Response(ok: true, data: DataPayload(message: action == "accept" ? "accepted" : "dismissed"))
+    }
+
+    return Response(
+      ok: true,
+      data: DataPayload(
+        message: preferredAlertTitle(alert.root, buttons: alert.buttons),
+        items: alert.buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) }
+      )
+    )
+  }
+
+  private func runnerAlert(root: XCUIElement, ownerApp: XCUIApplication) -> RunnerAlert? {
+    let buttons = actionableElements(in: root).filter { isEnabledElement($0) }
+    guard !buttons.isEmpty else {
+      return nil
+    }
+    return RunnerAlert(root: root, ownerApp: ownerApp, buttons: buttons)
+  }
+
+  private func firstExistingElement(in elements: [XCUIElement]) -> XCUIElement? {
+    elements.first { isVisibleElement($0) }
+  }
+
+  private func firstDismissPopupWindow(in app: XCUIApplication) -> XCUIElement? {
+    safeElementsQuery {
+      app.windows.allElementsBoundByIndex
+    }.first { window in
+      if !isVisibleElement(window) { return false }
+      if isDismissPopupMarker(window.label) || isDismissPopupMarker(window.identifier) {
+        return true
+      }
+      return safeElementsQuery {
+        window.descendants(matching: .any).allElementsBoundByIndex
+      }.contains { descendant in
+        isDismissPopupMarker(descendant.label) || isDismissPopupMarker(descendant.identifier)
+      }
+    }
+  }
+
+  private func chooseAlertButton(_ buttons: [XCUIElement], action: String) -> XCUIElement? {
+    if action == "accept" {
+      if let accept = buttons.first(where: { isAcceptButton($0.label) }) {
+        return accept
+      }
+      return buttons.count == 1 && !isDismissButton(buttons[0].label) ? buttons[0] : nil
+    }
+
+    return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last
+  }
+
+  private func isAcceptButton(_ label: String) -> Bool {
+    let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
+    return [
+      "ok",
+      "allow",
+      "yes",
+      "continue",
+      "done",
+      "open settings"
+    ].contains(normalized) || normalized.hasPrefix("confirm")
+  }
+
+  private func isDismissButton(_ label: String) -> Bool {
+    [
+      "cancel",
+      "close",
+      "dismiss",
+      "don't allow",
+      "don’t allow",
+      "not now",
+      "no",
+      "keep browsing",
+      "later"
+    ].contains(label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased())
+  }
+
+  private func preferredAlertTitle(_ element: XCUIElement, buttons: [XCUIElement]) -> String {
+    let buttonLabels = Set(buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() })
+    let descendants = element.descendants(matching: .any).allElementsBoundByIndex
+    for descendant in descendants {
+      let text = descendant.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      if text.isEmpty ||
+        isGenericAlertLabel(text) ||
+        buttonLabels.contains(text.lowercased()) ||
+        descendant.elementType == .navigationBar ||
+        actionableTypes.contains(descendant.elementType)
+      {
+        continue
+      }
+      return text
+    }
+    let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    return label.isEmpty || isGenericAlertLabel(label) ? "Alert" : label
+  }
+
+  private func isGenericAlertLabel(_ label: String) -> Bool {
+    let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
+    return isDismissPopupMarker(normalized) ||
+      normalized.hasPrefix("vertical scroll bar") ||
+      normalized.hasPrefix("horizontal scroll bar") ||
+      normalized == "tab bar"
+  }
+
+  private func isVisibleElement(_ element: XCUIElement) -> Bool {
+    element.exists && !element.frame.isNull && !element.frame.isEmpty
+  }
+
+  private func isEnabledElement(_ element: XCUIElement) -> Bool {
+    var enabled = false
+    _ = RunnerObjCExceptionCatcher.catchException({
+      enabled = element.exists && element.isEnabled
+    })
+    return enabled
+  }
+
+  private func isDismissPopupMarker(_ label: String) -> Bool {
+    label.trimmingCharacters(in: .whitespacesAndNewlines).caseInsensitiveCompare("dismiss popup") == .orderedSame
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -13,7 +13,7 @@ extension RunnerTests {
     return (gestureStartUptimeMs, currentUptimeMs())
   }
 
-  private func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
+  func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
     switch outcome {
     case .performed:
       return nil
@@ -140,11 +140,16 @@ extension RunnerTests {
       if let bundleId = requestedBundleId, targetNeedsActivation(activeApp) {
         activeApp = activateTarget(bundleId: bundleId, reason: "stale_target")
       } else if requestedBundleId == nil, targetNeedsActivation(activeApp) {
-        app.activate()
+        ensureRunnerHostAppActive(reason: "missing_app_bundle")
         activeApp = app
       }
 
-      if !activeApp.waitForExistence(timeout: appExistenceTimeout) {
+      let skipExistenceWait = canUseFastForegroundAppGuard(
+        activeApp: activeApp,
+        requestedBundleId: requestedBundleId,
+        command: command.command
+      )
+      if !skipExistenceWait && !activeApp.waitForExistence(timeout: appExistenceTimeout) {
         if let bundleId = requestedBundleId {
           activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait")
           guard activeApp.waitForExistence(timeout: appExistenceTimeout) else {
@@ -159,10 +164,15 @@ extension RunnerTests {
         if let bundleId = requestedBundleId, activeApp.state != .runningForeground {
           activeApp = activateTarget(bundleId: bundleId, reason: "interaction_foreground_guard")
         } else if requestedBundleId == nil, activeApp.state != .runningForeground {
-          app.activate()
+          ensureRunnerHostAppActive(reason: "interaction_missing_app_bundle")
           activeApp = app
         }
-        if !activeApp.waitForExistence(timeout: 2) {
+        let skipInteractionExistenceWait = canUseFastForegroundAppGuard(
+          activeApp: activeApp,
+          requestedBundleId: requestedBundleId,
+          command: command.command
+        )
+        if !skipInteractionExistenceWait && !activeApp.waitForExistence(timeout: 2) {
           if let bundleId = requestedBundleId {
             return Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))
           }
@@ -241,6 +251,40 @@ extension RunnerTests {
         data: DataPayload(currentUptimeMs: currentUptimeMs())
       )
     case .tap:
+      if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
+        let match = findElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue)
+        if match.isAmbiguous {
+          return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
+        }
+        if let element = match.element {
+          let frame = element.frame
+          let touchFrame = frame.isEmpty
+            ? nil
+            : resolvedTouchVisualizationFrame(app: activeApp, x: frame.midX, y: frame.midY)
+          var outcome = RunnerInteractionOutcome.performed
+          let timing = measureGesture {
+            withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+              outcome = activateElement(app: activeApp, element: element, action: "tap by selector")
+            }
+          }
+          if let response = unsupportedResponse(for: outcome) {
+            return response
+          }
+          return Response(
+            ok: true,
+            data: DataPayload(
+              message: "tapped",
+              gestureStartUptimeMs: timing.gestureStartUptimeMs,
+              gestureEndUptimeMs: timing.gestureEndUptimeMs,
+              x: touchFrame?.x,
+              y: touchFrame?.y,
+              referenceWidth: touchFrame?.referenceWidth,
+              referenceHeight: touchFrame?.referenceHeight
+            )
+          )
+        }
+        return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
+      }
       if let text = command.text {
         if let element = findElement(app: activeApp, text: text) {
           var outcome = RunnerInteractionOutcome.performed
@@ -412,11 +456,25 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
       }
       let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
-      let dragFrame = resolvedDragVisualizationFrame(app: activeApp, x: x, y: y, x2: x2, y2: y2)
+      let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
+      let dragFrame = resolvedDragVisualizationFrame(
+        app: activeApp,
+        x: dragPoints.x,
+        y: dragPoints.y,
+        x2: dragPoints.x2,
+        y2: dragPoints.y2
+      )
       var outcome = RunnerInteractionOutcome.performed
       let timing = measureGesture {
         withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-          outcome = dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+          outcome = dragAt(
+            app: activeApp,
+            x: dragPoints.x,
+            y: dragPoints.y,
+            x2: dragPoints.x2,
+            y2: dragPoints.y2,
+            holdDuration: holdDuration
+          )
         }
       }
       if let response = unsupportedResponse(for: outcome) {
@@ -447,6 +505,7 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
       }
       let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
+      let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
       var outcome = RunnerInteractionOutcome.performed
       let timing = measureGesture {
         withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
@@ -456,9 +515,23 @@ extension RunnerTests {
             }
             let reverse = pattern == "ping-pong" && (idx % 2 == 1)
             if reverse {
-              outcome = dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
+              outcome = dragAt(
+                app: activeApp,
+                x: dragPoints.x2,
+                y: dragPoints.y2,
+                x2: dragPoints.x,
+                y2: dragPoints.y,
+                holdDuration: holdDuration
+              )
             } else {
-              outcome = dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+              outcome = dragAt(
+                app: activeApp,
+                x: dragPoints.x,
+                y: dragPoints.y,
+                x2: dragPoints.x2,
+                y2: dragPoints.y2,
+                holdDuration: holdDuration
+              )
             }
           }
         }
@@ -487,45 +560,11 @@ extension RunnerTests {
       }
       return Response(ok: true, data: DataPayload(message: "remote pressed"))
     case .type:
-      guard let text = command.text else {
-        return Response(ok: false, error: ErrorPayload(message: "type requires text"))
-      }
-      let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
-      let target: XCUIElement?
-      if let x = command.x, let y = command.y {
-        target = textInputAt(app: activeApp, x: x, y: y) ?? focusedTextInput(app: activeApp)
-      } else {
-        target = focusedTextInput(app: activeApp)
-      }
-      func typeIntoTarget(_ value: String) {
-        if let focused = target {
-          focused.typeText(value)
-        } else {
-          activeApp.typeText(value)
-        }
-      }
-      if command.clearFirst == true {
-        guard let focused = target else {
-          let message =
-            (command.x != nil && command.y != nil)
-            ? "no text input found at the provided coordinates to clear"
-            : "no focused text input to clear"
-          return Response(ok: false, error: ErrorPayload(message: message))
-        }
-        clearTextInput(focused)
-      }
-      if delaySeconds > 0 && text.count > 1 {
-        let chunks = Array(text)
-        for (index, character) in chunks.enumerated() {
-          typeIntoTarget(String(character))
-          if index + 1 < chunks.count {
-            Thread.sleep(forTimeInterval: delaySeconds)
-          }
-        }
-      } else {
-        typeIntoTarget(text)
+      var response: Response?
+      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+        response = executeTypeCommand(activeApp: activeApp, command: command)
       }
-      return Response(ok: true, data: DataPayload(message: "typed"))
+      return response ?? Response(ok: false, error: ErrorPayload(message: "type produced no response"))
     case .interactionFrame:
       let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
       return Response(
@@ -573,6 +612,11 @@ extension RunnerTests {
       }
       let found = findElement(app: activeApp, text: text) != nil
       return Response(ok: true, data: DataPayload(found: found))
+    case .querySelector:
+      guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else {
+        return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue"))
+      }
+      return queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue)
     case .readText:
       guard let x = command.x, let y = command.y else {
         return Response(ok: false, error: ErrorPayload(message: "readText requires x and y"))
@@ -604,7 +648,7 @@ extension RunnerTests {
         targetApp.activate()
         activeApp = targetApp
         // Brief wait for the app transition animation to complete
-        Thread.sleep(forTimeInterval: 0.5)
+        sleepFor(0.5)
       }
       if command.fullscreen == true {
         screenshot = XCUIScreen.main.screenshot()
@@ -687,32 +731,10 @@ extension RunnerTests {
       )
     case .alert:
       let action = (command.action ?? "get").lowercased()
-      let alert = activeApp.alerts.firstMatch
-      if !alert.exists {
+      guard let alert = resolveAlert(app: activeApp) else {
         return Response(ok: false, error: ErrorPayload(message: "alert not found"))
       }
-      if action == "accept" {
-        guard let button = alert.buttons.allElementsBoundByIndex.first else {
-          return Response(ok: false, error: ErrorPayload(message: "alert accept button not found"))
-        }
-        let outcome = activateElement(app: activeApp, element: button, action: "alert accept")
-        if let response = unsupportedResponse(for: outcome) {
-          return response
-        }
-        return Response(ok: true, data: DataPayload(message: "accepted"))
-      }
-      if action == "dismiss" {
-        guard let button = alert.buttons.allElementsBoundByIndex.last else {
-          return Response(ok: false, error: ErrorPayload(message: "alert dismiss button not found"))
-        }
-        let outcome = activateElement(app: activeApp, element: button, action: "alert dismiss")
-        if let response = unsupportedResponse(for: outcome) {
-          return response
-        }
-        return Response(ok: true, data: DataPayload(message: "dismissed"))
-      }
-      let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label }
-      return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels))
+      return handleAlert(alert, action: action)
     case .pinch:
       guard let scale = command.scale, scale > 0 else {
         return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0"))
@@ -734,4 +756,41 @@ extension RunnerTests {
       )
     }
   }
+
+  private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
+    guard let text = command.text else {
+      return Response(ok: false, error: ErrorPayload(message: "type requires text"))
+    }
+    let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
+    let textEntryMode = resolveTextEntryMode(command)
+    let target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y)
+    if textEntryMode == .replacement {
+      guard target.element != nil else {
+        let message =
+          (command.x != nil && command.y != nil)
+          ? "no text input found at the provided coordinates to clear"
+          : "no focused text input to clear"
+        return Response(ok: false, error: ErrorPayload(message: message))
+      }
+    }
+    let textResult = typeTextReliably(
+      app: activeApp,
+      target: target,
+      text: text,
+      delaySeconds: delaySeconds,
+      repairMode: textEntryMode
+    )
+    if textResult.verified == false {
+      let expected = textResult.expectedText ?? ""
+      let observed = textResult.observedText ?? ""
+      return Response(
+        ok: false,
+        error: ErrorPayload(
+          code: "TEXT_ENTRY_MISMATCH",
+          message: "text entry verification failed: expected \"\(expected)\", observed \"\(observed)\""
+        )
+      )
+    }
+    return Response(ok: true, data: DataPayload(message: textResult.repaired ? "typed after repair" : "typed"))
+  }
 }