agent-device 0.14.9 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +7 -4
  2. package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.14.9.apk → agent-device-android-snapshot-helper-0.15.0.apk} +0 -0
  3. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.15.0.apk.sha256 +1 -0
  4. package/android-snapshot-helper/dist/{agent-device-android-snapshot-helper-0.14.9.manifest.json → agent-device-android-snapshot-helper-0.15.0.manifest.json} +6 -6
  5. package/dist/src/1769.js +7 -0
  6. package/dist/src/2151.js +429 -0
  7. package/dist/src/221.js +4 -4
  8. package/dist/src/2842.js +1 -0
  9. package/dist/src/3572.js +1 -0
  10. package/dist/src/4057.js +1 -1
  11. package/dist/src/840.js +2 -0
  12. package/dist/src/9542.js +2 -2
  13. package/dist/src/9639.js +2 -2
  14. package/dist/src/android-adb.d.ts +38 -9
  15. package/dist/src/android-adb.js +1 -1
  16. package/dist/src/android-snapshot-helper.d.ts +23 -0
  17. package/dist/src/cli.js +60 -57
  18. package/dist/src/contracts.d.ts +1 -0
  19. package/dist/src/finders.d.ts +1 -0
  20. package/dist/src/index.d.ts +19 -22
  21. package/dist/src/internal/companion-tunnel.js +1 -1
  22. package/dist/src/internal/daemon.js +51 -23
  23. package/dist/src/remote-config.d.ts +17 -14
  24. package/dist/src/selectors.d.ts +2 -0
  25. package/dist/src/server.js +2 -20
  26. package/ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/xcshareddata/xcschemes/AgentDeviceRunner.xcscheme +7 -1
  27. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +128 -47
  28. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +734 -10
  29. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +93 -7
  30. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +5 -0
  31. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +9 -0
  32. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +1 -0
  33. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +1 -2
  34. package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan +26 -0
  35. package/package.json +25 -11
  36. package/server.json +3 -3
  37. package/skills/agent-device/SKILL.md +2 -7
  38. package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.14.9.apk.sha256 +0 -1
  39. package/dist/src/180.js +0 -1
  40. package/dist/src/6108.js +0 -26
  41. package/dist/src/6642.js +0 -1
  42. package/dist/src/7462.js +0 -1
  43. package/dist/src/8809.js +0 -8
  44. package/dist/src/command-schema.js +0 -382
@@ -1,4 +1,20 @@
1
- export declare type RemoteConfigProfile = {
1
+ declare type RemoteConfigMetroOptions = {
2
+ metroProjectRoot?: string;
3
+ metroKind?: 'auto' | 'react-native' | 'expo';
4
+ metroPublicBaseUrl?: string;
5
+ metroProxyBaseUrl?: string;
6
+ metroBearerToken?: string;
7
+ metroPreparePort?: number;
8
+ metroListenHost?: string;
9
+ metroStatusHost?: string;
10
+ metroStartupTimeoutMs?: number;
11
+ metroProbeTimeoutMs?: number;
12
+ metroRuntimeFile?: string;
13
+ metroNoReuseExisting?: boolean;
14
+ metroNoInstallDeps?: boolean;
15
+ };
16
+
17
+ export declare type RemoteConfigProfile = RemoteConfigMetroOptions & {
2
18
  stateDir?: string;
3
19
  daemonBaseUrl?: string;
4
20
  daemonAuthToken?: string;
@@ -17,19 +33,6 @@ export declare type RemoteConfigProfile = {
17
33
  iosSimulatorDeviceSet?: string;
18
34
  androidDeviceAllowlist?: string;
19
35
  session?: string;
20
- metroProjectRoot?: string;
21
- metroKind?: 'auto' | 'react-native' | 'expo';
22
- metroPublicBaseUrl?: string;
23
- metroProxyBaseUrl?: string;
24
- metroBearerToken?: string;
25
- metroPreparePort?: number;
26
- metroListenHost?: string;
27
- metroStatusHost?: string;
28
- metroStartupTimeoutMs?: number;
29
- metroProbeTimeoutMs?: number;
30
- metroRuntimeFile?: string;
31
- metroNoReuseExisting?: boolean;
32
- metroNoInstallDeps?: boolean;
33
36
  };
34
37
 
35
38
  export declare type RemoteConfigProfileOptions = {
@@ -46,6 +46,7 @@ declare type RawSnapshotNode = {
46
46
  surface?: string;
47
47
  hiddenContentAbove?: boolean;
48
48
  hiddenContentBelow?: boolean;
49
+ presentationHints?: string[];
49
50
  };
50
51
 
51
52
  declare type Rect = {
@@ -104,6 +105,7 @@ declare type SnapshotState = {
104
105
  truncated?: boolean;
105
106
  backend?: SnapshotBackend;
106
107
  comparisonSafe?: boolean;
108
+ presentationKey?: string;
107
109
  };
108
110
 
109
111
  export declare function tryParseSelectorChain(expression: string): SelectorChain | null;
@@ -1,20 +1,2 @@
1
- import{buildCommandUsageText as e,buildUsageText as t}from"./command-schema.js";import{readVersion as r}from"./9671.js";let n="agent-device",o=["workflow","debugging","react-devtools","remote","macos","dogfood"];function i(e={}){let t=!1===e.global?"npx -y agent-device mcp":"agent-device mcp",r=!1===e.global?"No global install required.":"npm install -g agent-device",n=e.client?`
2
- Client hint: ${e.client}`:"";return`${r}
3
-
4
- MCP server command:
5
- ${t}
6
-
7
- Generic MCP JSON:
8
- {
9
- "mcpServers": {
10
- "agent-device": {
11
- "command": "${!1===e.global?"npx":"agent-device"}",
12
- "args": ${JSON.stringify(!1===e.global?["-y","agent-device","mcp"]:["mcp"])}
13
- }
14
- }
15
- }
16
-
17
- Use this server for discovery and routing only. For device actions, call the CLI commands returned by the help tool, starting with:
18
- agent-device help workflow${n}
19
- `}function a(r={}){if(r.topic&&r.command)throw Error("Provide either topic or command, not both.");let n=r.topic??r.command;if(!n)return t();if(r.topic&&!d(r.topic))throw Error(`Unknown help topic: ${r.topic}. Expected one of: ${o.join(", ")}`);let i=e(n);if(!i)throw Error(`Unknown command or help topic: ${n}`);return i}function s(){return[{uri:"agent-device://install",name:"agent-device MCP install",description:"Install and client configuration snippets.",mimeType:"text/markdown"},{uri:"agent-device://help",name:"agent-device command list",description:"Version-matched command list and global flags.",mimeType:"text/plain"},...o.map(e=>({uri:`agent-device://help/${e}`,name:`agent-device help ${e}`,description:`Version-matched ${e} workflow guidance.`,mimeType:"text/plain"}))]}function c(){return[l("agent-device-workflow","Plan a normal app automation loop."),l("agent-device-debugging","Collect focused debugging evidence."),l("agent-device-dogfood","Run exploratory QA with reproducible evidence."),l("agent-device-react-native-performance","Inspect React Native renders."),l("agent-device-macos","Inspect a macOS app or surface.")]}function l(e,t){return{name:e,description:t,arguments:[{name:"target",description:"Optional app, device, or task target.",required:!1}]}}function d(e){return o.includes(e)}function p(e){if("2.0"!==e.jsonrpc||"string"!=typeof e.method)return g(e.id??null,-32600,"Invalid JSON-RPC request.");if(void 0===e.id)return null;try{var l,p;return l=e.id,p=function(e,l){switch(e){case"initialize":return{protocolVersion:"2025-11-25",capabilities:{tools:{},resources:{},prompts:{}},serverInfo:{name:n,version:r()}};case"ping":return{};case"tools/list":return{tools:[{name:"status",description:"Report the installed agent-device MCP router and CLI metadata.",inputSchema:{type:"object",properties:{},additionalProperties:!1}},{name:"install",description:"Return install and MCP client configuration snippets for agent-device.",inputSchema:{type:"object",properties:{client:{type:"string",description:"Optional client name for labeling the returned guidance."},global:{type:"boolean",description:"Use a global agent-device binary when true; use npx when false."}},additionalProperties:!1}},{name:"help",description:"Return version-matched CLI help for a workflow topic or command.",inputSchema:{type:"object",properties:{topic:{type:"string",enum:o,description:"Agent workflow topic."},command:{type:"string",description:"CLI command name such as snapshot, open, logs, or react-devtools."}},additionalProperties:!1}}]};case"tools/call":var p,g,w,y=l;let b=m(y),$=h(b,"name"),k=f(b.arguments);try{if("status"===$)return u(JSON.stringify({name:n,registryName:"io.github.callstackincubator/agent-device",version:r(),transport:"stdio",command:"agent-device mcp",node:process.version,capabilities:{tools:["status","install","help"],resources:s().map(e=>e.uri),prompts:c().map(e=>e.name)},note:"This MCP server routes agents to the agent-device CLI. It does not expose device automation or shell execution tools."},null,2));if("install"===$)return u(i(k));if("help"===$)return u(a(k));throw Error(`Unknown tool: ${$}`)}catch(e){return u(e instanceof Error?e.message:String(e),!0)}return;case"resources/list":return{resources:s()};case"resources/read":let x;return p=l,{contents:[{uri:x=h(m(p),"uri"),mimeType:"agent-device://install"===x?"text/markdown":"text/plain",text:function(e){if("agent-device://install"===e)return i();if("agent-device://help"===e)return t();let r=e.startsWith("agent-device://help/")?e.slice(20):"";if(d(r))return a({topic:r});throw Error(`Unknown resource: ${e}`)}(x)}]};case"prompts/list":return{prompts:c()};case"prompts/get":let C;return g=l,C=m(g),function(e,t={}){let r={"agent-device-workflow":"workflow","agent-device-debugging":"debugging","agent-device-dogfood":"dogfood","agent-device-react-native-performance":"react-devtools","agent-device-macos":"macos"}[e];if(!r)throw Error(`Unknown prompt: ${e}`);let n=t.target?` Target: ${t.target}.`:"";return{description:`Use agent-device help ${r} before planning commands.${n}`,messages:[{role:"user",content:{type:"text",text:`Read the agent-device ${r} guidance through the MCP help tool, then produce a concise command plan using agent-device CLI commands only.${n}`}}]}}(h(C,"name"),(w=C.arguments,Object.fromEntries(Object.entries(f(w)).filter(e=>"string"==typeof e[1]))));default:throw new v(`Unsupported MCP method: ${e}`)}}(e.method,e.params),{jsonrpc:"2.0",id:l,result:p}}catch(t){if(t instanceof v)return g(e.id,-32601,t.message);return g(e.id,-32602,t instanceof Error?t.message:String(t))}}function u(e,t=!1){return{isError:t,content:[{type:"text",text:e}]}}function g(e,t,r){return{jsonrpc:"2.0",id:e,error:{code:t,message:r}}}function m(e){if(!e||"object"!=typeof e||Array.isArray(e))throw Error("Expected object parameters.");return e}function f(e){return void 0===e?{}:m(e)}function h(e,t){let r=e[t];if("string"!=typeof r||0===r.length)throw Error(`Expected ${t} to be a non-empty string.`);return r}class v extends Error{}async function w(){let e=new y(e=>{let t=function(e){if(Array.isArray(e)){let t=e.flatMap(e=>{var t;return(t=p(e))?[t]:[]});return t.length>0?t:null}return p(e)}(e);t&&b(t)});process.stdin.setEncoding("utf8"),process.stdin.on("data",t=>{try{e.push(t)}catch(e){b({jsonrpc:"2.0",id:null,error:{code:-32700,message:e instanceof Error?e.message:String(e)}})}}),await new Promise(e=>{process.stdin.on("end",e),process.stdin.on("close",e),process.stdin.resume()})}class y{buffer="";sink;constructor(e){this.sink=e}push(e){for(this.buffer+=e;;){let e=this.tryReadLineMessage();if(void 0!==e){this.emit(e);continue}break}}tryReadLineMessage(){let e=this.buffer.indexOf("\n");if(-1===e)return;let t=this.buffer.slice(0,e).trim();return this.buffer=this.buffer.slice(e+1),t.length>0?t:void 0}emit(e){let t=JSON.parse(e);Array.isArray(t),this.sink(t)}}function b(e){process.stdout.write(`${JSON.stringify(e)}
20
- `)}export{w as runAgentDeviceMcpServer};
1
+ import{readVersion as e}from"./9671.js";function t(t){if("2.0"!==t.jsonrpc||"string"!=typeof t.method)return r(t.id??null,-32600,"Invalid JSON-RPC request.");if(void 0===t.id)return null;try{var i,s;return i=t.id,s=function(t,r){switch(t){case"initialize":return{protocolVersion:"2025-11-25",capabilities:{tools:{}},serverInfo:{name:"agent-device",version:e()}};case"ping":return{};case"tools/list":return{tools:[{name:"status",description:"Return discovery-only handoff metadata for installing, verifying, and using the agent-device CLI.",inputSchema:{type:"object",properties:{},additionalProperties:!1},outputSchema:{type:"object",properties:{packageName:{type:"string"},installedPackageVersion:{type:"string"},cliCommandName:{type:"string"},installCommand:{type:"string"},verifyCommand:{type:"string"},startingHelpCommand:{type:"string"},supportedTargets:{type:"array",items:{type:"string"}},capabilities:{type:"array",items:{type:"string"}},prerequisites:{type:"array",items:{type:"string"}},docsUrl:{type:"string"},agentDocsUrl:{type:"string"},firstCommands:{type:"array",items:{type:"string"}},automationInterface:{type:"string",const:"cli"},automationNote:{type:"string"},installRequiresHumanApproval:{type:"boolean",const:!0},installSafetyNote:{type:"string"}},required:["packageName","installedPackageVersion","cliCommandName","installCommand","verifyCommand","startingHelpCommand","supportedTargets","capabilities","prerequisites","docsUrl","agentDocsUrl","firstCommands","automationInterface","automationNote","installRequiresHumanApproval","installSafetyNote"],additionalProperties:!1}}]};case"tools/call":var i=r;let s=function(e,t){let r=e[t];if("string"!=typeof r||0===r.length)throw Error(`Expected ${t} to be a non-empty string.`);return r}(function(e){if(!e||"object"!=typeof e||Array.isArray(e))throw Error("Expected object parameters.");return e}(i),"name");try{if("status"===s){let t;return t={packageName:"agent-device",installedPackageVersion:e(),cliCommandName:"agent-device",installCommand:"npm install -g agent-device@latest",verifyCommand:"agent-device --version",startingHelpCommand:"agent-device help workflow",supportedTargets:["ios-simulator","android-emulator","ios-device","android-device","tvos-simulator","macos","linux"],capabilities:["inspect-ui","interact-with-elements","open-apps","install-app","capture-screenshot","accessibility-snapshot","collect-logs","collect-network","collect-performance","record-replay","react-native","expo","android-adb","ios-xcuitest"],prerequisites:["node>=22","xcode-for-ios","android-sdk-adb-for-android","macos-accessibility-permission-for-desktop"],docsUrl:"https://agent-device.dev/",agentDocsUrl:"https://incubator.callstack.com/agent-device/llms-full.txt",firstCommands:["agent-device help workflow","agent-device apps --platform ios","agent-device apps --platform android"],automationInterface:"cli",automationNote:"Device automation happens through the agent-device CLI. This MCP server is discovery-only and does not expose device automation tools.",installRequiresHumanApproval:!0,installSafetyNote:"Agents should not install or update the package unless the human has approved the environment change. If the CLI is missing, ask the human to run the install command, then run the verify command."},{isError:!1,structuredContent:t,content:[{type:"text",text:JSON.stringify(t,null,2)}]}}throw Error(`Unknown tool: ${s}`)}catch(e){return function(e,t=!1){return{isError:t,content:[{type:"text",text:e}]}}(e instanceof Error?e.message:String(e),!0)}return;default:throw new n(`Unsupported MCP method: ${t}`)}}(t.method,t.params),{jsonrpc:"2.0",id:i,result:s}}catch(e){if(e instanceof n)return r(t.id,-32601,e.message);return r(t.id,-32602,e instanceof Error?e.message:String(e))}}function r(e,t,r){return{jsonrpc:"2.0",id:e,error:{code:t,message:r}}}class n extends Error{}async function i(){let e=new s(e=>{let r=function(e){if(Array.isArray(e)){let r=e.flatMap(e=>{var r;return(r=t(e))?[r]:[]});return r.length>0?r:null}return t(e)}(e);r&&a(r)});process.stdin.setEncoding("utf8"),process.stdin.on("data",t=>{try{e.push(t)}catch(e){a({jsonrpc:"2.0",id:null,error:{code:-32700,message:e instanceof Error?e.message:String(e)}})}}),await new Promise(e=>{process.stdin.on("end",e),process.stdin.on("close",e),process.stdin.resume()})}class s{buffer="";sink;constructor(e){this.sink=e}push(e){for(this.buffer+=e;;){let e=this.tryReadLineMessage();if(void 0!==e){this.emit(e);continue}break}}tryReadLineMessage(){let e=this.buffer.indexOf("\n");if(-1===e)return;let t=this.buffer.slice(0,e).trim();return this.buffer=this.buffer.slice(e+1),t.length>0?t:void 0}emit(e){let t=JSON.parse(e);Array.isArray(t),this.sink(t)}}function a(e){process.stdout.write(`${JSON.stringify(e)}
2
+ `)}export{i as runAgentDeviceMcpServer};
@@ -28,7 +28,13 @@
28
28
  selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
29
29
  selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
30
30
  shouldUseLaunchSchemeArgsEnv = "YES"
31
- shouldAutocreateTestPlan = "YES">
31
+ shouldAutocreateTestPlan = "NO">
32
+ <TestPlans>
33
+ <TestPlanReference
34
+ reference = "container:AgentDeviceRunnerUITests.xctestplan"
35
+ default = "YES">
36
+ </TestPlanReference>
37
+ </TestPlans>
32
38
  <Testables>
33
39
  <TestableReference
34
40
  skipped = "NO"
@@ -140,11 +140,16 @@ extension RunnerTests {
140
140
  if let bundleId = requestedBundleId, targetNeedsActivation(activeApp) {
141
141
  activeApp = activateTarget(bundleId: bundleId, reason: "stale_target")
142
142
  } else if requestedBundleId == nil, targetNeedsActivation(activeApp) {
143
- app.activate()
143
+ ensureRunnerHostAppActive(reason: "missing_app_bundle")
144
144
  activeApp = app
145
145
  }
146
146
 
147
- if !activeApp.waitForExistence(timeout: appExistenceTimeout) {
147
+ let skipExistenceWait = canUseFastForegroundAppGuard(
148
+ activeApp: activeApp,
149
+ requestedBundleId: requestedBundleId,
150
+ command: command.command
151
+ )
152
+ if !skipExistenceWait && !activeApp.waitForExistence(timeout: appExistenceTimeout) {
148
153
  if let bundleId = requestedBundleId {
149
154
  activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait")
150
155
  guard activeApp.waitForExistence(timeout: appExistenceTimeout) else {
@@ -159,10 +164,15 @@ extension RunnerTests {
159
164
  if let bundleId = requestedBundleId, activeApp.state != .runningForeground {
160
165
  activeApp = activateTarget(bundleId: bundleId, reason: "interaction_foreground_guard")
161
166
  } else if requestedBundleId == nil, activeApp.state != .runningForeground {
162
- app.activate()
167
+ ensureRunnerHostAppActive(reason: "interaction_missing_app_bundle")
163
168
  activeApp = app
164
169
  }
165
- if !activeApp.waitForExistence(timeout: 2) {
170
+ let skipInteractionExistenceWait = canUseFastForegroundAppGuard(
171
+ activeApp: activeApp,
172
+ requestedBundleId: requestedBundleId,
173
+ command: command.command
174
+ )
175
+ if !skipInteractionExistenceWait && !activeApp.waitForExistence(timeout: 2) {
166
176
  if let bundleId = requestedBundleId {
167
177
  return Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))
168
178
  }
@@ -241,6 +251,40 @@ extension RunnerTests {
241
251
  data: DataPayload(currentUptimeMs: currentUptimeMs())
242
252
  )
243
253
  case .tap:
254
+ if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
255
+ let match = findElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue)
256
+ if match.isAmbiguous {
257
+ return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
258
+ }
259
+ if let element = match.element {
260
+ let frame = element.frame
261
+ let touchFrame = frame.isEmpty
262
+ ? nil
263
+ : resolvedTouchVisualizationFrame(app: activeApp, x: frame.midX, y: frame.midY)
264
+ var outcome = RunnerInteractionOutcome.performed
265
+ let timing = measureGesture {
266
+ withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
267
+ outcome = activateElement(app: activeApp, element: element, action: "tap by selector")
268
+ }
269
+ }
270
+ if let response = unsupportedResponse(for: outcome) {
271
+ return response
272
+ }
273
+ return Response(
274
+ ok: true,
275
+ data: DataPayload(
276
+ message: "tapped",
277
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
278
+ gestureEndUptimeMs: timing.gestureEndUptimeMs,
279
+ x: touchFrame?.x,
280
+ y: touchFrame?.y,
281
+ referenceWidth: touchFrame?.referenceWidth,
282
+ referenceHeight: touchFrame?.referenceHeight
283
+ )
284
+ )
285
+ }
286
+ return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
287
+ }
244
288
  if let text = command.text {
245
289
  if let element = findElement(app: activeApp, text: text) {
246
290
  var outcome = RunnerInteractionOutcome.performed
@@ -412,11 +456,25 @@ extension RunnerTests {
412
456
  return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
413
457
  }
414
458
  let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
415
- let dragFrame = resolvedDragVisualizationFrame(app: activeApp, x: x, y: y, x2: x2, y2: y2)
459
+ let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
460
+ let dragFrame = resolvedDragVisualizationFrame(
461
+ app: activeApp,
462
+ x: dragPoints.x,
463
+ y: dragPoints.y,
464
+ x2: dragPoints.x2,
465
+ y2: dragPoints.y2
466
+ )
416
467
  var outcome = RunnerInteractionOutcome.performed
417
468
  let timing = measureGesture {
418
469
  withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
419
- outcome = dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
470
+ outcome = dragAt(
471
+ app: activeApp,
472
+ x: dragPoints.x,
473
+ y: dragPoints.y,
474
+ x2: dragPoints.x2,
475
+ y2: dragPoints.y2,
476
+ holdDuration: holdDuration
477
+ )
420
478
  }
421
479
  }
422
480
  if let response = unsupportedResponse(for: outcome) {
@@ -447,6 +505,7 @@ extension RunnerTests {
447
505
  return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
448
506
  }
449
507
  let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
508
+ let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
450
509
  var outcome = RunnerInteractionOutcome.performed
451
510
  let timing = measureGesture {
452
511
  withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
@@ -456,9 +515,23 @@ extension RunnerTests {
456
515
  }
457
516
  let reverse = pattern == "ping-pong" && (idx % 2 == 1)
458
517
  if reverse {
459
- outcome = dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
518
+ outcome = dragAt(
519
+ app: activeApp,
520
+ x: dragPoints.x2,
521
+ y: dragPoints.y2,
522
+ x2: dragPoints.x,
523
+ y2: dragPoints.y,
524
+ holdDuration: holdDuration
525
+ )
460
526
  } else {
461
- outcome = dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
527
+ outcome = dragAt(
528
+ app: activeApp,
529
+ x: dragPoints.x,
530
+ y: dragPoints.y,
531
+ x2: dragPoints.x2,
532
+ y2: dragPoints.y2,
533
+ holdDuration: holdDuration
534
+ )
462
535
  }
463
536
  }
464
537
  }
@@ -487,45 +560,11 @@ extension RunnerTests {
487
560
  }
488
561
  return Response(ok: true, data: DataPayload(message: "remote pressed"))
489
562
  case .type:
490
- guard let text = command.text else {
491
- return Response(ok: false, error: ErrorPayload(message: "type requires text"))
492
- }
493
- let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
494
- let target: XCUIElement?
495
- if let x = command.x, let y = command.y {
496
- target = textInputAt(app: activeApp, x: x, y: y) ?? focusedTextInput(app: activeApp)
497
- } else {
498
- target = focusedTextInput(app: activeApp)
499
- }
500
- func typeIntoTarget(_ value: String) {
501
- if let focused = target {
502
- focused.typeText(value)
503
- } else {
504
- activeApp.typeText(value)
505
- }
506
- }
507
- if command.clearFirst == true {
508
- guard let focused = target else {
509
- let message =
510
- (command.x != nil && command.y != nil)
511
- ? "no text input found at the provided coordinates to clear"
512
- : "no focused text input to clear"
513
- return Response(ok: false, error: ErrorPayload(message: message))
514
- }
515
- clearTextInput(focused)
516
- }
517
- if delaySeconds > 0 && text.count > 1 {
518
- let chunks = Array(text)
519
- for (index, character) in chunks.enumerated() {
520
- typeIntoTarget(String(character))
521
- if index + 1 < chunks.count {
522
- Thread.sleep(forTimeInterval: delaySeconds)
523
- }
524
- }
525
- } else {
526
- typeIntoTarget(text)
563
+ var response: Response?
564
+ withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
565
+ response = executeTypeCommand(activeApp: activeApp, command: command)
527
566
  }
528
- return Response(ok: true, data: DataPayload(message: "typed"))
567
+ return response ?? Response(ok: false, error: ErrorPayload(message: "type produced no response"))
529
568
  case .interactionFrame:
530
569
  let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
531
570
  return Response(
@@ -573,6 +612,11 @@ extension RunnerTests {
573
612
  }
574
613
  let found = findElement(app: activeApp, text: text) != nil
575
614
  return Response(ok: true, data: DataPayload(found: found))
615
+ case .querySelector:
616
+ guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else {
617
+ return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue"))
618
+ }
619
+ return queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue)
576
620
  case .readText:
577
621
  guard let x = command.x, let y = command.y else {
578
622
  return Response(ok: false, error: ErrorPayload(message: "readText requires x and y"))
@@ -604,7 +648,7 @@ extension RunnerTests {
604
648
  targetApp.activate()
605
649
  activeApp = targetApp
606
650
  // Brief wait for the app transition animation to complete
607
- Thread.sleep(forTimeInterval: 0.5)
651
+ sleepFor(0.5)
608
652
  }
609
653
  if command.fullscreen == true {
610
654
  screenshot = XCUIScreen.main.screenshot()
@@ -734,4 +778,41 @@ extension RunnerTests {
734
778
  )
735
779
  }
736
780
  }
781
+
782
+ private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
783
+ guard let text = command.text else {
784
+ return Response(ok: false, error: ErrorPayload(message: "type requires text"))
785
+ }
786
+ let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
787
+ let textEntryMode = resolveTextEntryMode(command)
788
+ let target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y)
789
+ if textEntryMode == .replacement {
790
+ guard target.element != nil else {
791
+ let message =
792
+ (command.x != nil && command.y != nil)
793
+ ? "no text input found at the provided coordinates to clear"
794
+ : "no focused text input to clear"
795
+ return Response(ok: false, error: ErrorPayload(message: message))
796
+ }
797
+ }
798
+ let textResult = typeTextReliably(
799
+ app: activeApp,
800
+ target: target,
801
+ text: text,
802
+ delaySeconds: delaySeconds,
803
+ repairMode: textEntryMode
804
+ )
805
+ if textResult.verified == false {
806
+ let expected = textResult.expectedText ?? ""
807
+ let observed = textResult.observedText ?? ""
808
+ return Response(
809
+ ok: false,
810
+ error: ErrorPayload(
811
+ code: "TEXT_ENTRY_MISMATCH",
812
+ message: "text entry verification failed: expected \"\(expected)\", observed \"\(observed)\""
813
+ )
814
+ )
815
+ }
816
+ return Response(ok: true, data: DataPayload(message: textResult.repaired ? "typed after repair" : "typed"))
817
+ }
737
818
  }