mobile-debug-mcp 0.34.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (500) hide show
  1. package/.agents/skills/mcp-builder/SKILL.md +44 -64
  2. package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
  3. package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
  4. package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
  5. package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
  6. package/.agents/skills/test-authoring/SKILL.md +7 -6
  7. package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
  8. package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
  9. package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
  10. package/.cursor-plugin/marketplace.json +1 -1
  11. package/.github/copilot-instructions.md +14 -36
  12. package/.github/workflows/ci.yml +4 -4
  13. package/AGENTS.md +12 -3
  14. package/CONTRIBUTING.md +79 -0
  15. package/README.md +218 -129
  16. package/agent-plugin/.antigravity-plugin/README.md +48 -0
  17. package/agent-plugin/.antigravity-plugin/plugin.json +18 -0
  18. package/agent-plugin/.claude-plugin/plugin.json +2 -2
  19. package/agent-plugin/.codex-plugin/plugin.json +1 -1
  20. package/agent-plugin/.cursor-plugin/plugin.json +2 -2
  21. package/agent-plugin/.mcp.json +1 -1
  22. package/agent-plugin/README.md +50 -11
  23. package/agent-plugin/cursor/.cursor-plugin/plugin.json +23 -0
  24. package/agent-plugin/cursor/LICENSE +201 -0
  25. package/agent-plugin/cursor/README.md +9 -0
  26. package/agent-plugin/cursor/skills/mobile-debug-mcp/SKILL.md +253 -0
  27. package/agent-plugin/mcp.json +1 -1
  28. package/agent-plugin/mcp_config.json +13 -0
  29. package/agent-plugin/plugin.json +1 -1
  30. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +62 -17
  31. package/dist/contracts/execution-controls.js +24 -0
  32. package/dist/contracts/selector.js +8 -0
  33. package/dist/contracts/target.js +15 -0
  34. package/dist/contracts/ui.js +1 -0
  35. package/dist/debugger/android.js +2 -1
  36. package/dist/debugger/diagnostics.js +54 -0
  37. package/dist/debugger/guard.js +4 -4
  38. package/dist/debugger/ios-worker.js +62 -0
  39. package/dist/debugger/ios.js +57 -0
  40. package/dist/debugger/protocol.js +33 -0
  41. package/dist/debugger/sessions.js +133 -74
  42. package/dist/debugger/worker.js +59 -46
  43. package/dist/interact/adjustment/adjustment-context.js +41 -0
  44. package/dist/interact/adjustment/adjustment-gestures.js +55 -0
  45. package/dist/interact/adjustment/adjustment-plan.js +18 -0
  46. package/dist/interact/adjustment/adjustment-resolution.js +127 -0
  47. package/dist/interact/adjustment/adjustment-result.js +19 -0
  48. package/dist/interact/adjustment/adjustment-semantic.js +64 -0
  49. package/dist/interact/adjustment/adjustment-verification.js +35 -0
  50. package/dist/interact/adjustment/adjustment.js +52 -0
  51. package/dist/interact/adjustment/control-geometry.js +87 -0
  52. package/dist/interact/adjustment/control-values.js +136 -0
  53. package/dist/interact/adjustment/slider-labels.js +84 -0
  54. package/dist/interact/assertions/screen-assertion.js +37 -0
  55. package/dist/interact/assertions/state-assertion-context.js +35 -0
  56. package/dist/interact/assertions/state-assertion.js +76 -0
  57. package/dist/interact/assertions/state-comparison.js +141 -0
  58. package/dist/interact/assertions/visibility-assertion.js +34 -0
  59. package/dist/interact/conditional-action.js +12 -7
  60. package/dist/interact/index.js +77 -3092
  61. package/dist/interact/input-service.js +1 -0
  62. package/dist/interact/interaction-trace.js +21 -0
  63. package/dist/interact/model.js +1 -0
  64. package/dist/interact/observation.js +1 -0
  65. package/dist/interact/targeting/candidate-ancestor.js +58 -0
  66. package/dist/interact/targeting/candidate-result.js +23 -0
  67. package/dist/interact/targeting/candidate-scoring.js +50 -0
  68. package/dist/interact/targeting/element-helpers.js +216 -0
  69. package/dist/interact/targeting/find-element.js +65 -0
  70. package/dist/interact/targeting/nearby-control.js +56 -0
  71. package/dist/interact/targeting/resolved-elements.js +21 -0
  72. package/dist/interact/targeting/tap-target.js +17 -0
  73. package/dist/interact/targeting/target-scope.js +104 -0
  74. package/dist/interact/waits/legacy-log-wait.js +38 -0
  75. package/dist/interact/waits/legacy-wait-observations.js +103 -0
  76. package/dist/interact/waits/legacy-wait-state.js +34 -0
  77. package/dist/interact/waits/legacy-wait.js +48 -0
  78. package/dist/interact/waits/screen-wait.js +49 -0
  79. package/dist/interact/waits/ui-change-helpers.js +157 -0
  80. package/dist/interact/waits/ui-change-state.js +18 -0
  81. package/dist/interact/waits/ui-change-wait.js +77 -0
  82. package/dist/interact/waits/ui-wait-condition.js +56 -0
  83. package/dist/interact/waits/ui-wait-state.js +38 -0
  84. package/dist/interact/waits/ui-wait.js +106 -0
  85. package/dist/manage/android.js +55 -51
  86. package/dist/manage/index.js +38 -75
  87. package/dist/manage/ios/build-types.js +1 -0
  88. package/dist/manage/ios/build.js +210 -0
  89. package/dist/manage/ios/project-discovery.js +29 -0
  90. package/dist/manage/ios/scheme.js +18 -0
  91. package/dist/manage/ios.js +44 -256
  92. package/dist/observe/android/context.js +1 -0
  93. package/dist/observe/android/current-screen.js +55 -0
  94. package/dist/observe/android/fingerprint.js +13 -0
  95. package/dist/observe/android/log-stream.js +132 -0
  96. package/dist/observe/android/logs.js +107 -0
  97. package/dist/observe/android/screenshot.js +70 -0
  98. package/dist/observe/android/ui-tree.js +107 -0
  99. package/dist/observe/android.js +16 -520
  100. package/dist/observe/evidence.js +114 -75
  101. package/dist/observe/index.js +16 -263
  102. package/dist/observe/ios/context.js +16 -0
  103. package/dist/observe/ios/fingerprint.js +12 -0
  104. package/dist/observe/ios/log-stream.js +105 -0
  105. package/dist/observe/ios/logs.js +168 -0
  106. package/dist/observe/ios/screenshot.js +22 -0
  107. package/dist/observe/ios/ui-elements.js +241 -0
  108. package/dist/observe/ios/ui-tree.js +137 -0
  109. package/dist/observe/ios-trace/artifact.js +63 -0
  110. package/dist/observe/ios-trace/capabilities.js +86 -0
  111. package/dist/observe/ios-trace/native.js +93 -0
  112. package/dist/observe/ios-trace/notification.js +35 -0
  113. package/dist/observe/ios-trace/process.js +98 -0
  114. package/dist/observe/ios-trace/sessions.js +244 -0
  115. package/dist/observe/ios.js +17 -715
  116. package/dist/observe/logs/model.js +1 -0
  117. package/dist/{network → observe/network}/index.js +2 -2
  118. package/dist/observe/readiness-service.js +5 -3
  119. package/dist/observe/readiness.js +33 -14
  120. package/dist/observe/screenshot-encoding.js +41 -0
  121. package/dist/observe/snapshot-collection.js +79 -0
  122. package/dist/observe/snapshot-logs.js +30 -0
  123. package/dist/observe/snapshot-metadata.js +43 -31
  124. package/dist/observe/snapshot-semantic.js +115 -0
  125. package/dist/observe/system-trace-error.js +7 -0
  126. package/dist/observe/system-trace-router.js +81 -0
  127. package/dist/observe/system-trace.js +26 -21
  128. package/dist/runtime/action-results.js +270 -0
  129. package/dist/runtime/clock.js +4 -0
  130. package/dist/server/common.js +1 -256
  131. package/dist/server/definitions/assert.js +162 -0
  132. package/dist/server/definitions/diagnostic.js +118 -0
  133. package/dist/server/definitions/input.js +274 -0
  134. package/dist/server/definitions/manage.js +194 -0
  135. package/dist/server/definitions/observe.js +149 -0
  136. package/dist/server/definitions/target.js +138 -0
  137. package/dist/server/definitions/wait.js +105 -0
  138. package/dist/server/handlers/action-execution.js +163 -0
  139. package/dist/server/handlers/correlation.js +23 -0
  140. package/dist/server/handlers/debugger-handlers.js +67 -0
  141. package/dist/server/handlers/debugger-response.js +9 -0
  142. package/dist/server/handlers/diagnostic.js +77 -0
  143. package/dist/server/handlers/handler-types.js +1 -0
  144. package/dist/server/handlers/input.js +154 -0
  145. package/dist/server/handlers/interact.js +113 -0
  146. package/dist/server/handlers/journey-assertions.js +35 -0
  147. package/dist/server/handlers/journey.js +147 -0
  148. package/dist/server/handlers/manage.js +119 -0
  149. package/dist/server/handlers/observe.js +95 -0
  150. package/dist/server/handlers/payload.js +4 -0
  151. package/dist/server/handlers/tool-handlers.js +170 -0
  152. package/dist/server/schemas/condition-schema.js +2 -0
  153. package/dist/server/{debugger-schema.js → schemas/debugger-schema.js} +20 -12
  154. package/dist/server/{journey-schema.js → schemas/journey-schema.js} +7 -22
  155. package/dist/server/schemas/system-trace-schema.js +22 -0
  156. package/dist/server/tool-definitions.js +58 -1135
  157. package/dist/server-core.js +2 -2
  158. package/dist/server.js +4 -4
  159. package/dist/system/android-status.js +101 -0
  160. package/dist/system/android.js +37 -27
  161. package/dist/system/index.js +13 -439
  162. package/dist/system/ios-status.js +132 -0
  163. package/dist/system/ios.js +11 -8
  164. package/dist/system/model.js +1 -0
  165. package/dist/system/probes.js +105 -0
  166. package/dist/system/status-summary.js +43 -0
  167. package/dist/system/unsupported-status.js +52 -0
  168. package/dist/utils/android/commands.js +89 -0
  169. package/dist/utils/android/gradle.js +139 -0
  170. package/dist/utils/android/logs.js +103 -0
  171. package/dist/utils/android/ui-tree.js +233 -0
  172. package/dist/utils/android/utils.js +6 -564
  173. package/dist/utils/cli/idb/idb-helper.js +4 -6
  174. package/dist/utils/cli/idb/install-idb.js +13 -9
  175. package/dist/utils/cli/ios/preflight-ios.js +42 -38
  176. package/dist/utils/ios/utils.js +48 -46
  177. package/dist/utils/java.js +65 -49
  178. package/dist/utils/resolve-device.js +4 -2
  179. package/dist/utils/ui/index.js +26 -26
  180. package/docs/CHANGELOG.md +67 -1
  181. package/docs/README.md +38 -0
  182. package/docs/agent-plugin-installation-verification.md +39 -0
  183. package/docs/benchmarks/README.md +18 -0
  184. package/docs/benchmarks/interaction-latency/README.md +14 -0
  185. package/docs/benchmarks/interaction-latency/final.md +4 -1
  186. package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
  187. package/docs/benchmarks/ui-tree-performance/README.md +24 -16
  188. package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
  189. package/docs/code-structure.md +120 -0
  190. package/docs/specs/001-state-verification/spec.md +16 -14
  191. package/docs/specs/002-richer-element-identity/spec.md +12 -10
  192. package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
  193. package/docs/specs/004-action-verification-routing/spec.md +3 -3
  194. package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
  195. package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
  196. package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
  197. package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
  198. package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
  199. package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
  200. package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
  201. package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
  202. package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
  203. package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
  204. package/docs/specs/014-actionability-resolution/spec.md +20 -18
  205. package/docs/specs/015-adjustable-control-support/spec.md +17 -15
  206. package/docs/specs/016-environment-tools/spec.md +20 -20
  207. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
  208. package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
  209. package/docs/specs/020-conditional-actions/intent.md +5 -3
  210. package/docs/specs/020-conditional-actions/plan.md +8 -9
  211. package/docs/specs/020-conditional-actions/spec.md +4 -2
  212. package/docs/specs/021-app-readiness-signals/intent.md +4 -2
  213. package/docs/specs/021-app-readiness-signals/plan.md +7 -6
  214. package/docs/specs/021-app-readiness-signals/spec.md +3 -1
  215. package/docs/specs/022-correlated-evidence/intent.md +4 -4
  216. package/docs/specs/022-correlated-evidence/plan.md +10 -11
  217. package/docs/specs/022-correlated-evidence/spec.md +3 -1
  218. package/docs/specs/023-android-system-traces/intent.md +3 -2
  219. package/docs/specs/023-android-system-traces/plan.md +5 -7
  220. package/docs/specs/023-android-system-traces/spec.md +3 -1
  221. package/docs/specs/024-android-heap-dump-capture/intent.md +3 -1
  222. package/docs/specs/025-android-runtime-debugger/intent.md +4 -3
  223. package/docs/specs/025-android-runtime-debugger/plan.md +3 -1
  224. package/docs/specs/025-android-runtime-debugger/spec.md +4 -3
  225. package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
  226. package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
  227. package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
  228. package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
  229. package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
  230. package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
  231. package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
  232. package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
  233. package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
  234. package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
  235. package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
  236. package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
  237. package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
  238. package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
  239. package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
  240. package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
  241. package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
  242. package/docs/specs/030-android-coroutine-validation/report.md +240 -0
  243. package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
  244. package/docs/specs/031-ios-system-traces/intent.md +38 -0
  245. package/docs/specs/031-ios-system-traces/investigation.md +92 -0
  246. package/docs/specs/031-ios-system-traces/late-run-failure-evidence.json +32 -0
  247. package/docs/specs/031-ios-system-traces/plan.md +138 -0
  248. package/docs/specs/031-ios-system-traces/pr102-monitor-evidence.json +22 -0
  249. package/docs/specs/031-ios-system-traces/simulator-evidence.json +47 -0
  250. package/docs/specs/031-ios-system-traces/spec.md +135 -0
  251. package/docs/specs/032-ios-runtime-debugger/evidence.json +176 -0
  252. package/docs/specs/032-ios-runtime-debugger/intent.md +38 -0
  253. package/docs/specs/032-ios-runtime-debugger/plan.md +95 -0
  254. package/docs/specs/032-ios-runtime-debugger/report.md +205 -0
  255. package/docs/specs/032-ios-runtime-debugger/spec.md +153 -0
  256. package/docs/tools/TOOLS.md +31 -18
  257. package/docs/tools/{evidence.md → correlated-evidence.md} +9 -1
  258. package/docs/tools/debugger.md +120 -4
  259. package/docs/tools/interact.md +42 -20
  260. package/docs/tools/manage.md +25 -44
  261. package/docs/tools/observe.md +55 -23
  262. package/docs/tools/system-traces.md +89 -14
  263. package/docs/tools/system.md +12 -5
  264. package/package.json +12 -8
  265. package/scripts/agent-plugin.mjs +55 -5
  266. package/scripts/architecture-graph.mjs +69 -0
  267. package/scripts/check-architecture.mjs +47 -0
  268. package/scripts/code-quality-baseline.json +1 -0
  269. package/scripts/code-quality.mjs +161 -0
  270. package/src/contracts/execution-controls.ts +26 -0
  271. package/src/contracts/selector.ts +17 -0
  272. package/src/contracts/target.ts +19 -0
  273. package/src/contracts/ui.ts +94 -0
  274. package/src/debugger/android.ts +3 -2
  275. package/src/debugger/diagnostics.ts +43 -0
  276. package/src/debugger/guard.ts +10 -5
  277. package/src/debugger/ios-worker.ts +49 -0
  278. package/src/debugger/ios.ts +45 -0
  279. package/{worker → src/debugger/java}/DebuggerWorker.java +9 -3
  280. package/src/debugger/lldb/breakpoints.py +51 -0
  281. package/src/debugger/lldb/inspection.py +109 -0
  282. package/src/debugger/lldb/worker.py +203 -0
  283. package/src/debugger/protocol.ts +37 -0
  284. package/src/debugger/sessions.ts +326 -155
  285. package/src/debugger/worker.ts +57 -41
  286. package/src/interact/adjustment/adjustment-context.ts +83 -0
  287. package/src/interact/adjustment/adjustment-gestures.ts +48 -0
  288. package/src/interact/adjustment/adjustment-plan.ts +23 -0
  289. package/src/interact/adjustment/adjustment-resolution.ts +104 -0
  290. package/src/interact/adjustment/adjustment-result.ts +34 -0
  291. package/src/interact/adjustment/adjustment-semantic.ts +53 -0
  292. package/src/interact/adjustment/adjustment-verification.ts +60 -0
  293. package/src/interact/adjustment/adjustment.ts +49 -0
  294. package/src/interact/adjustment/control-geometry.ts +120 -0
  295. package/src/interact/adjustment/control-values.ts +141 -0
  296. package/src/interact/adjustment/slider-labels.ts +84 -0
  297. package/src/interact/assertions/screen-assertion.ts +39 -0
  298. package/src/interact/assertions/state-assertion-context.ts +48 -0
  299. package/src/interact/assertions/state-assertion.ts +64 -0
  300. package/src/interact/assertions/state-comparison.ts +168 -0
  301. package/src/interact/assertions/visibility-assertion.ts +41 -0
  302. package/src/interact/conditional-action.ts +18 -12
  303. package/src/interact/index.ts +89 -3494
  304. package/src/interact/input-service.ts +11 -0
  305. package/src/interact/interaction-trace.ts +36 -0
  306. package/src/interact/model.ts +109 -0
  307. package/src/interact/observation.ts +7 -0
  308. package/src/interact/targeting/candidate-ancestor.ts +49 -0
  309. package/src/interact/targeting/candidate-result.ts +24 -0
  310. package/src/interact/targeting/candidate-scoring.ts +42 -0
  311. package/src/interact/targeting/element-helpers.ts +243 -0
  312. package/src/interact/targeting/find-element.ts +47 -0
  313. package/src/interact/targeting/nearby-control.ts +46 -0
  314. package/src/interact/targeting/resolved-elements.ts +20 -0
  315. package/src/interact/targeting/tap-target.ts +22 -0
  316. package/src/interact/targeting/target-scope.ts +121 -0
  317. package/src/interact/waits/legacy-log-wait.ts +30 -0
  318. package/src/interact/waits/legacy-wait-observations.ts +55 -0
  319. package/src/interact/waits/legacy-wait-state.ts +48 -0
  320. package/src/interact/waits/legacy-wait.ts +34 -0
  321. package/src/interact/waits/screen-wait.ts +37 -0
  322. package/src/interact/waits/ui-change-helpers.ts +184 -0
  323. package/src/interact/waits/ui-change-state.ts +26 -0
  324. package/src/interact/waits/ui-change-wait.ts +59 -0
  325. package/src/interact/waits/ui-wait-condition.ts +52 -0
  326. package/src/interact/waits/ui-wait-state.ts +55 -0
  327. package/src/interact/waits/ui-wait.ts +84 -0
  328. package/src/manage/android.ts +56 -50
  329. package/src/manage/index.ts +63 -93
  330. package/src/manage/ios/build-types.ts +14 -0
  331. package/src/manage/ios/build.ts +230 -0
  332. package/src/manage/ios/project-discovery.ts +26 -0
  333. package/src/manage/ios/scheme.ts +16 -0
  334. package/src/manage/ios.ts +67 -298
  335. package/src/observe/android/context.ts +1 -0
  336. package/src/observe/android/current-screen.ts +63 -0
  337. package/src/observe/android/fingerprint.ts +13 -0
  338. package/src/observe/android/log-stream.ts +127 -0
  339. package/src/observe/android/logs.ts +110 -0
  340. package/src/observe/android/screenshot.ts +79 -0
  341. package/src/observe/android/ui-tree.ts +113 -0
  342. package/src/observe/android.ts +17 -531
  343. package/src/observe/evidence.ts +122 -70
  344. package/src/observe/index.ts +24 -286
  345. package/src/observe/ios/context.ts +22 -0
  346. package/src/observe/ios/fingerprint.ts +12 -0
  347. package/src/observe/ios/log-stream.ts +103 -0
  348. package/src/observe/ios/logs.ts +179 -0
  349. package/src/observe/ios/screenshot.ts +26 -0
  350. package/src/observe/ios/ui-elements.ts +266 -0
  351. package/src/observe/ios/ui-tree.ts +143 -0
  352. package/src/observe/ios-trace/artifact.ts +55 -0
  353. package/src/observe/ios-trace/capabilities.ts +74 -0
  354. package/src/observe/ios-trace/native.ts +72 -0
  355. package/src/observe/ios-trace/notification.ts +30 -0
  356. package/src/observe/ios-trace/process.ts +82 -0
  357. package/src/observe/ios-trace/sessions.ts +204 -0
  358. package/src/observe/ios.ts +18 -749
  359. package/src/observe/logs/model.ts +11 -0
  360. package/src/{network → observe/network}/index.ts +2 -2
  361. package/src/observe/readiness-service.ts +6 -4
  362. package/src/observe/readiness.ts +42 -21
  363. package/src/observe/screenshot-encoding.ts +32 -0
  364. package/src/observe/snapshot-collection.ts +69 -0
  365. package/src/observe/snapshot-logs.ts +26 -0
  366. package/src/observe/snapshot-metadata.ts +51 -31
  367. package/src/observe/snapshot-semantic.ts +153 -0
  368. package/src/observe/system-trace-error.ts +3 -0
  369. package/src/observe/system-trace-router.ts +61 -0
  370. package/src/observe/system-trace.ts +28 -19
  371. package/src/runtime/action-results.ts +376 -0
  372. package/src/runtime/clock.ts +9 -0
  373. package/src/server/common.ts +9 -359
  374. package/src/server/definitions/assert.ts +164 -0
  375. package/src/server/definitions/diagnostic.ts +120 -0
  376. package/src/server/definitions/input.ts +276 -0
  377. package/src/server/definitions/manage.ts +196 -0
  378. package/src/server/definitions/observe.ts +151 -0
  379. package/src/server/definitions/target.ts +140 -0
  380. package/src/server/definitions/wait.ts +107 -0
  381. package/src/server/handlers/action-execution.ts +164 -0
  382. package/src/server/handlers/correlation.ts +20 -0
  383. package/src/server/handlers/debugger-handlers.ts +58 -0
  384. package/src/server/handlers/debugger-response.ts +7 -0
  385. package/src/server/handlers/diagnostic.ts +86 -0
  386. package/src/server/handlers/handler-types.ts +7 -0
  387. package/src/server/handlers/input.ts +173 -0
  388. package/src/server/handlers/interact.ts +135 -0
  389. package/src/server/handlers/journey-assertions.ts +39 -0
  390. package/src/server/handlers/journey.ts +132 -0
  391. package/src/server/handlers/manage.ts +141 -0
  392. package/src/server/handlers/observe.ts +116 -0
  393. package/src/server/handlers/payload.ts +36 -0
  394. package/src/server/handlers/tool-handlers.ts +170 -0
  395. package/src/server/schemas/condition-schema.ts +2 -0
  396. package/src/server/schemas/debugger-schema.ts +46 -0
  397. package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +7 -22
  398. package/src/server/schemas/system-trace-schema.ts +23 -0
  399. package/src/server/tool-definitions.ts +67 -1143
  400. package/src/server-core.ts +2 -2
  401. package/src/server.ts +4 -4
  402. package/src/system/android-status.ts +124 -0
  403. package/src/system/android.ts +33 -21
  404. package/src/system/index.ts +21 -525
  405. package/src/system/ios-status.ts +147 -0
  406. package/src/system/ios.ts +14 -10
  407. package/src/system/model.ts +41 -0
  408. package/src/system/probes.ts +114 -0
  409. package/src/system/status-summary.ts +47 -0
  410. package/src/system/unsupported-status.ts +55 -0
  411. package/src/types.ts +16 -106
  412. package/src/utils/android/commands.ts +81 -0
  413. package/src/utils/android/gradle.ts +132 -0
  414. package/src/utils/android/logs.ts +103 -0
  415. package/src/utils/android/ui-tree.ts +239 -0
  416. package/src/utils/android/utils.ts +16 -556
  417. package/src/utils/cli/idb/check-idb.ts +4 -4
  418. package/src/utils/cli/idb/idb-helper.ts +6 -8
  419. package/src/utils/cli/idb/install-idb.ts +15 -9
  420. package/src/utils/cli/ios/preflight-ios.ts +39 -31
  421. package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
  422. package/src/utils/ios/utils.ts +60 -61
  423. package/src/utils/java.ts +62 -43
  424. package/src/utils/resolve-device.ts +9 -6
  425. package/src/utils/ui/index.ts +28 -28
  426. package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
  427. package/test/device/automated/debugger/coroutine-support.ts +140 -0
  428. package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
  429. package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
  430. package/test/device/automated/debugger/runtime.android.test.ts +19 -3
  431. package/test/device/automated/debugger/runtime.ios.test.ts +221 -0
  432. package/test/device/automated/interact/interaction.android.test.ts +56 -0
  433. package/test/device/automated/observe/ios-system-trace.smoke.ts +119 -0
  434. package/test/device/automated/observe/system-trace.smoke.ts +1 -1
  435. package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
  436. package/test/fixtures/contracts/tool-definitions.json +3696 -0
  437. package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
  438. package/test/fixtures/coroutine-app/README.md +48 -0
  439. package/test/fixtures/coroutine-app/build.ts +35 -0
  440. package/test/fixtures/debugger-app/MainActivity.java +21 -0
  441. package/test/fixtures/debugger-app/MainActivity.kt +32 -0
  442. package/test/fixtures/debugger-app/README.md +52 -0
  443. package/test/fixtures/debugger-app/build.ts +25 -15
  444. package/test/fixtures/ios-debugger-app/Main.swift +95 -0
  445. package/test/fixtures/ios-debugger-app/README.md +27 -0
  446. package/test/fixtures/ios-debugger-app/build.ts +21 -0
  447. package/test/fixtures/latency-journey-app/README.md +2 -0
  448. package/test/fixtures/readiness-app/README.md +2 -0
  449. package/test/helpers/ios-trace-shutdown.ts +29 -0
  450. package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
  451. package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
  452. package/test/unit/debugger/diagnostics.test.ts +80 -0
  453. package/test/unit/debugger/guards.test.ts +4 -0
  454. package/test/unit/debugger/handle-guards.test.ts +88 -0
  455. package/test/unit/debugger/ios-preparation.test.ts +57 -0
  456. package/test/unit/debugger/ios.test.ts +104 -0
  457. package/test/unit/debugger/lldb-worker.py +166 -0
  458. package/test/unit/debugger/lldb-worker.test.ts +10 -0
  459. package/test/unit/debugger/preparation.test.ts +32 -0
  460. package/test/unit/debugger/protocol.test.ts +15 -0
  461. package/test/unit/debugger/sessions.test.ts +1 -1
  462. package/test/unit/interact/adjustment_service.test.ts +42 -0
  463. package/test/unit/interact/adjustment_verification.test.ts +35 -0
  464. package/test/unit/interact/clock_dependencies.test.ts +63 -0
  465. package/test/unit/interact/element_helpers.test.ts +19 -0
  466. package/test/unit/interact/legacy_wait_service.test.ts +17 -0
  467. package/test/unit/interact/state_comparison.test.ts +20 -0
  468. package/test/unit/interact/target_scope.test.ts +35 -0
  469. package/test/unit/network/get_network_activity.test.ts +1 -1
  470. package/test/unit/observe/ios-trace-native.test.ts +90 -0
  471. package/test/unit/observe/ios-trace-sessions.test.ts +131 -0
  472. package/test/unit/observe/readiness.test.ts +1 -1
  473. package/test/unit/observe/screenshot-encoding.test.ts +30 -0
  474. package/test/unit/observe/snapshot-logs.test.ts +14 -0
  475. package/test/unit/observe/system-trace-router.test.ts +49 -0
  476. package/test/unit/quality/agent-plugin-export.test.ts +38 -0
  477. package/test/unit/quality/architecture.test.ts +60 -0
  478. package/test/unit/quality/code-quality.test.ts +58 -0
  479. package/test/unit/quality/interaction_boundaries.test.ts +24 -0
  480. package/test/unit/server/conditional_actions.test.ts +1 -1
  481. package/test/unit/server/conditional_schema.test.ts +2 -2
  482. package/test/unit/server/debugger-response.test.ts +63 -0
  483. package/test/unit/server/system-trace.test.ts +12 -3
  484. package/test/unit/server/tool_definitions_parity.test.ts +8 -0
  485. package/dist/server/debugger-handlers.js +0 -38
  486. package/dist/server/system-trace-schema.js +0 -17
  487. package/dist/server/tool-handlers.js +0 -1000
  488. package/docs/ROADMAP.md +0 -646
  489. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
  490. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
  491. package/docs/specs/baseline-spec-v0.md +0 -312
  492. package/docs/specs/mcp-tooling-spec-v1.md +0 -446
  493. package/src/server/debugger-handlers.ts +0 -29
  494. package/src/server/debugger-schema.ts +0 -37
  495. package/src/server/system-trace-schema.ts +0 -17
  496. package/src/server/tool-handlers.ts +0 -1008
  497. /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
  498. /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
  499. /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
  500. /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
@@ -1,180 +1,29 @@
1
- import { checkDebuggerGuard } from '../debugger/guard.js';
2
- import { currentReadinessWait, observeReadinessWait, isDeviceUnavailable } from '../observe/readiness.js';
3
- import { readinessDelay } from '../observe/readiness-service.js';
1
+ import { resolveTap } from './targeting/tap-target.js';
2
+ import { waitForUICore } from './waits/legacy-wait.js';
3
+ import { expectState } from './assertions/state-assertion.js';
4
+ import { waitForUIChange } from './waits/ui-change-wait.js';
5
+ import { waitForScreenChange } from './waits/screen-wait.js';
6
+ import { expectScreen } from './assertions/screen-assertion.js';
7
+ import { expectVisible } from './assertions/visibility-assertion.js';
8
+ import { findElement } from './targeting/find-element.js';
9
+ import { waitForUI } from './waits/ui-wait.js';
10
+ import { adjustControl } from './adjustment/adjustment.js';
11
+ import { ResolvedElements } from './targeting/resolved-elements.js';
12
+ import { findScopedElement, resolveUiChangeScope } from './targeting/target-scope.js';
13
+ import { _normalizeBounds, _computeElementId, _resolvedTargetFromElement } from './targeting/element-helpers.js';
14
+ import { _collectSubtreeIndices } from './waits/ui-change-helpers.js';
4
15
  import { isConditionBlocked } from './conditional-action.js';
5
- import { remainingBudget } from '../utils/operation-budget.js';
6
- import { createHash } from 'crypto';
7
16
  import { AndroidInteract } from './android.js';
8
17
  import { iOSInteract } from './ios.js';
9
18
  export { AndroidInteract, iOSInteract };
10
19
  import { resolveTargetDevice } from '../utils/resolve-device.js';
11
20
  import { ToolsObserve } from '../observe/index.js';
12
- import { computeSnapshotSignature } from '../observe/snapshot-metadata.js';
13
- import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js';
14
- function buildObservationTrace({ actionType, stage, success, attempts, metadata }) {
15
- const now = Date.now();
16
- const actionId = nextActionId(actionType, now);
17
- const steps = [
18
- createTraceStep({
19
- stage,
20
- timestamp: now,
21
- result: success ? 'success' : 'failure',
22
- attemptIndex: 0,
23
- metadata
24
- })
25
- ];
26
- return {
27
- action_id: actionId,
28
- steps,
29
- final_outcome: success ? 'success' : 'failure',
30
- attempts: Math.max(1, Math.floor(attempts || 1))
31
- };
32
- }
21
+ import { buildActionExecutionResult } from '../runtime/action-results.js';
33
22
  export class ToolsInteract {
34
23
  static _maxResolvedUiElements = 256;
35
- static _uiChangeKinds = ['hierarchy_diff', 'text_change', 'state_change'];
36
- static _sliderSearchLookahead = 8;
37
- static _sliderNegativeGapTolerancePx = 32;
38
- static _sliderPositiveGapLimitPx = 640;
39
- static _sliderTrackMinLengthPx = 220;
40
- static _sliderTrackMaxThicknessPx = 180;
41
- static _sliderTrackLengthRatio = 0.18;
42
- static _sliderTrackThicknessRatio = 0.08;
43
- static _sliderLabelWidthRatio = 1.5;
44
- static _resolvedUiElements = new Map();
45
- static _normalize(s) {
46
- if (s === null || s === undefined)
47
- return '';
48
- try {
49
- return String(s).toLowerCase().trim();
50
- }
51
- catch {
52
- return '';
53
- }
54
- }
55
- static _normalizeBounds(bounds) {
56
- if (!Array.isArray(bounds) || bounds.length < 4)
57
- return null;
58
- const normalized = bounds.slice(0, 4).map((value) => Number(value));
59
- if (normalized.some((value) => Number.isNaN(value)))
60
- return null;
61
- return normalized;
62
- }
63
- static _hash(value) {
64
- return createHash('sha256').update(JSON.stringify(value)).digest('hex');
65
- }
66
- static _matchesSelector(el, selector) {
67
- if (!selector)
68
- return false;
69
- const normalize = ToolsInteract._normalize;
70
- const containsFlag = !!selector.contains;
71
- const text = normalize(el.text ?? el.label ?? el.value ?? '');
72
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
73
- const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '');
74
- if (selector.text !== undefined && selector.text !== null) {
75
- const q = normalize(selector.text);
76
- if (containsFlag ? !text.includes(q) : text !== q)
77
- return false;
78
- }
79
- if (selector.resource_id !== undefined && selector.resource_id !== null) {
80
- const q = normalize(selector.resource_id);
81
- if (containsFlag ? !resourceId.includes(q) : resourceId !== q)
82
- return false;
83
- }
84
- if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
85
- const q = normalize(selector.accessibility_id);
86
- if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q)
87
- return false;
88
- }
89
- return true;
90
- }
91
- static _findFirstMatchingElement(elements, selector) {
92
- if (!selector)
93
- return null;
94
- for (let i = 0; i < elements.length; i++) {
95
- const el = elements[i];
96
- if (!el)
97
- continue;
98
- if (ToolsInteract._matchesSelector(el, selector))
99
- return { el, idx: i };
100
- }
101
- return null;
102
- }
103
- static _resolveParentIndex(elements, parentId) {
104
- if (parentId === undefined || parentId === null)
105
- return null;
106
- if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
107
- return parentId;
108
- }
109
- if (typeof parentId === 'string') {
110
- const normalized = ToolsInteract._normalize(parentId);
111
- if (!normalized)
112
- return null;
113
- if (/^\d+$/.test(normalized)) {
114
- const index = Number(normalized);
115
- if (index >= 0 && index < elements.length)
116
- return index;
117
- }
118
- const foundIndex = elements.findIndex((el) => {
119
- if (!el)
120
- return false;
121
- return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
122
- ToolsInteract._normalize(el.stable_id ?? '') === normalized;
123
- });
124
- return foundIndex >= 0 ? foundIndex : null;
125
- }
126
- return null;
127
- }
128
- static _isVisibleElement(el) {
129
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
130
- return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
131
- }
132
- static _isTapActionable(el, storedStableId, platform) {
133
- if (!ToolsInteract._isVisibleElement(el)) {
134
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
135
- }
136
- if (el.enabled === false) {
137
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
138
- }
139
- const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
140
- (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap')));
141
- if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
142
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
143
- }
144
- if (storedStableId) {
145
- if (!el.stable_id || el.stable_id !== storedStableId) {
146
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
147
- }
148
- }
149
- return { actionable: true };
150
- }
151
- static _isAdjustableActionable(el, storedStableId) {
152
- if (!ToolsInteract._isVisibleElement(el)) {
153
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
154
- }
155
- if (el.enabled === false) {
156
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
157
- }
158
- if (storedStableId) {
159
- if (!el.stable_id || el.stable_id !== storedStableId) {
160
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
161
- }
162
- }
163
- return { actionable: true };
164
- }
165
- static _computeElementId(platform, deviceId, el, index) {
166
- const identity = {
167
- platform,
168
- deviceId: deviceId || '',
169
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
170
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
171
- accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
172
- class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
173
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
174
- index
175
- };
176
- return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`;
177
- }
24
+ static _resolvedUiElements = new ResolvedElements(ToolsInteract._maxResolvedUiElements);
25
+ static _normalizeBounds = _normalizeBounds;
26
+ static _computeElementId = _computeElementId;
178
27
  static _buildResolvedElement(platform, deviceId, el, index) {
179
28
  const bounds = ToolsInteract._normalizeBounds(el.bounds);
180
29
  const elementId = ToolsInteract._computeElementId(platform, deviceId, el, index);
@@ -203,224 +52,14 @@ export class ToolsInteract {
203
52
  };
204
53
  }
205
54
  static _resolveUiChangeScope(tree, scope, target) {
206
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
207
- const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
208
- if (normalizedScope === 'screen') {
209
- return {
210
- elements,
211
- resolution: {
212
- scope: 'screen',
213
- target: null,
214
- resolved: true,
215
- resolvedIndex: null,
216
- resolvedStableId: null,
217
- reason: 'screen scope'
218
- }
219
- };
220
- }
221
- const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
222
- if (!requestedTarget) {
223
- return {
224
- elements: [],
225
- resolution: {
226
- scope: 'subtree',
227
- target: null,
228
- resolved: false,
229
- resolvedIndex: null,
230
- resolvedStableId: null,
231
- reason: 'subtree scope requires a target element id'
232
- },
233
- error: {
234
- code: 'INVALID_SCOPE',
235
- message: 'scope=subtree requires a target element_id'
236
- }
237
- };
238
- }
239
- const resolved = ToolsInteract._findScopedElement(tree, requestedTarget);
240
- if (!resolved) {
241
- return {
242
- elements: [],
243
- resolution: {
244
- scope: 'subtree',
245
- target: requestedTarget,
246
- resolved: false,
247
- resolvedIndex: null,
248
- resolvedStableId: null,
249
- reason: 'target element could not be resolved'
250
- },
251
- error: {
252
- code: 'ELEMENT_NOT_FOUND',
253
- message: `Target element ${requestedTarget} could not be resolved for subtree scope`
254
- }
255
- };
256
- }
257
- const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index);
258
- const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
259
- return {
260
- elements: scopedElements,
261
- resolution: {
262
- scope: 'subtree',
263
- target: requestedTarget,
264
- resolved: true,
265
- resolvedIndex: resolved.index,
266
- resolvedStableId: resolved.stableId,
267
- reason: resolved.reason
268
- }
269
- };
270
- }
271
- static _findScopedElement(tree, targetElementId) {
272
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
273
- const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
274
- const deviceId = tree?.device?.id ?? undefined;
275
- const normalizedTarget = ToolsInteract._normalize(targetElementId);
276
- for (let i = 0; i < elements.length; i++) {
277
- const el = elements[i];
278
- if (!el)
279
- continue;
280
- const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i);
281
- if (computedElementId === targetElementId) {
282
- return {
283
- index: i,
284
- stableId: el.stable_id ?? null,
285
- reason: 'element_id_match'
286
- };
287
- }
288
- }
289
- for (let i = 0; i < elements.length; i++) {
290
- const el = elements[i];
291
- if (!el)
292
- continue;
293
- if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
294
- return {
295
- index: i,
296
- stableId: el.stable_id,
297
- reason: 'stable_id_match'
298
- };
299
- }
300
- }
301
- const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId);
302
- if (storedContext?.stable_id) {
303
- const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id);
304
- for (let i = 0; i < elements.length; i++) {
305
- const el = elements[i];
306
- if (!el?.stable_id)
307
- continue;
308
- if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
309
- return {
310
- index: i,
311
- stableId: el.stable_id,
312
- reason: 'stored_stable_id_match'
313
- };
314
- }
315
- }
316
- }
317
- return null;
55
+ return resolveUiChangeScope(tree, scope, target, (snapshot, id) => ToolsInteract._findScopedElement(snapshot, id));
318
56
  }
319
- static _collectSubtreeIndices(elements, rootIndex) {
320
- if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length)
321
- return [];
322
- const visited = new Set();
323
- const stack = [rootIndex];
324
- const result = [];
325
- while (stack.length > 0) {
326
- const index = stack.pop();
327
- if (index === undefined || visited.has(index) || index < 0 || index >= elements.length)
328
- continue;
329
- visited.add(index);
330
- result.push(index);
331
- const element = elements[index];
332
- if (!element)
333
- continue;
334
- const directChildren = new Set();
335
- if (Array.isArray(element.children)) {
336
- for (const childIndex of element.children) {
337
- if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
338
- directChildren.add(childIndex);
339
- }
340
- }
341
- }
342
- for (let i = 0; i < elements.length; i++) {
343
- if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
344
- directChildren.add(i);
345
- }
346
- }
347
- for (const childIndex of directChildren) {
348
- if (!visited.has(childIndex))
349
- stack.push(childIndex);
350
- }
351
- }
352
- return result.sort((left, right) => left - right);
353
- }
354
- static _changeIdentityForElement(el, index) {
355
- const stableId = ToolsInteract._normalize(el.stable_id);
356
- if (stableId)
357
- return `stable:${stableId}`;
358
- return `fallback:${ToolsInteract._hash({
359
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
360
- contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
361
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
362
- type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
363
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
364
- index
365
- })}`;
366
- }
367
- static _summarizeUiChangeDelta(initialElements, currentElements) {
368
- const buildMap = (elements) => {
369
- const map = new Map();
370
- for (let i = 0; i < elements.length; i++) {
371
- const element = elements[i];
372
- if (!element)
373
- continue;
374
- const key = ToolsInteract._changeIdentityForElement(element, i);
375
- map.set(key, ToolsInteract._hash({
376
- text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
377
- contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
378
- resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
379
- type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
380
- bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
381
- state: element.state ?? null,
382
- visible: !!element.visible,
383
- enabled: !!element.enabled,
384
- clickable: !!element.clickable
385
- }));
386
- }
387
- return map;
388
- };
389
- const initialMap = buildMap(initialElements);
390
- const currentMap = buildMap(currentElements);
391
- let added = 0;
392
- let removed = 0;
393
- let mutated = 0;
394
- for (const [key, value] of currentMap.entries()) {
395
- if (!initialMap.has(key)) {
396
- added++;
397
- }
398
- else if (initialMap.get(key) !== value) {
399
- mutated++;
400
- }
401
- }
402
- for (const key of initialMap.keys()) {
403
- if (!currentMap.has(key))
404
- removed++;
405
- }
406
- return {
407
- total_elements: currentElements.length,
408
- added_elements: added,
409
- removed_elements: removed,
410
- mutated_elements: mutated
411
- };
57
+ static _findScopedElement(tree, id) {
58
+ return findScopedElement(tree, id, ToolsInteract._resolvedUiElements);
412
59
  }
60
+ static _collectSubtreeIndices = _collectSubtreeIndices;
413
61
  static _rememberResolvedElement(elementId, context) {
414
- if (ToolsInteract._resolvedUiElements.has(elementId)) {
415
- ToolsInteract._resolvedUiElements.delete(elementId);
416
- }
417
- ToolsInteract._resolvedUiElements.set(elementId, context);
418
- while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
419
- const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value;
420
- if (!oldestElementId)
421
- break;
422
- ToolsInteract._resolvedUiElements.delete(oldestElementId);
423
- }
62
+ ToolsInteract._resolvedUiElements.remember(elementId, context);
424
63
  }
425
64
  static async _captureFingerprint(platform, deviceId) {
426
65
  if (isConditionBlocked())
@@ -433,402 +72,7 @@ export class ToolsInteract {
433
72
  return null;
434
73
  }
435
74
  }
436
- static _buildUiChangeSignatures(tree) {
437
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
438
- const textPayload = [];
439
- const statePayload = [];
440
- for (const el of elements) {
441
- textPayload.push({
442
- text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
443
- contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
444
- resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
445
- });
446
- statePayload.push({
447
- checked: el?.state?.checked ?? null,
448
- selected: el?.state?.selected ?? null,
449
- focused: el?.state?.focused ?? null,
450
- expanded: el?.state?.expanded ?? null,
451
- enabled: el?.state?.enabled ?? null,
452
- text_value: el?.state?.text_value ?? null,
453
- value: el?.state?.value ?? null,
454
- raw_value: el?.state?.raw_value ?? null,
455
- value_range: el?.state?.value_range ?? null
456
- });
457
- }
458
- return {
459
- hierarchy: computeSnapshotSignature(tree),
460
- text: ToolsInteract._hash({
461
- screen: ToolsInteract._normalize(tree?.screen),
462
- elements: textPayload
463
- }),
464
- state: ToolsInteract._hash({
465
- screen: ToolsInteract._normalize(tree?.screen),
466
- elements: statePayload
467
- })
468
- };
469
- }
470
- static _matchesUiChange(expected, initial, current) {
471
- const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds;
472
- for (const changeKind of candidates) {
473
- if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy)
474
- return changeKind;
475
- if (changeKind === 'text_change' && initial.text !== current.text)
476
- return changeKind;
477
- if (changeKind === 'state_change' && initial.state !== current.state)
478
- return changeKind;
479
- }
480
- return null;
481
- }
482
- static _uiChangeSignaturesEqual(left, right) {
483
- return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state;
484
- }
485
- static _resolvedTargetFromElement(elementId, element, index) {
486
- return {
487
- elementId,
488
- text: element.text ?? null,
489
- resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
490
- accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
491
- class: element.type ?? element.class ?? null,
492
- bounds: ToolsInteract._normalizeBounds(element.bounds),
493
- index,
494
- state: element.state ?? null,
495
- stable_id: element.stable_id ?? null,
496
- role: element.role ?? null,
497
- test_tag: element.test_tag ?? null,
498
- selector: element.selector ?? null,
499
- semantic: element.semantic ?? null
500
- };
501
- }
502
- static _summarizeResolutionCandidate(candidate) {
503
- const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds);
504
- return {
505
- text: candidate.el.text ?? null,
506
- resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
507
- accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
508
- class: candidate.el.type ?? candidate.el.class ?? null,
509
- bounds: bounds
510
- ? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
511
- : null,
512
- clickable: !!candidate.el.clickable,
513
- enabled: !!candidate.el.enabled,
514
- score: candidate.score,
515
- reason: candidate.reason
516
- };
517
- }
518
- static _isAdjustableControl(el) {
519
- if (!el)
520
- return false;
521
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
522
- const role = ToolsInteract._normalize(el.role ?? '');
523
- return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
524
- }
525
- static _isSemanticActionable(el) {
526
- if (!el?.semantic)
527
- return false;
528
- if (el.semantic.adjustable)
529
- return true;
530
- return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0;
531
- }
532
- static _readNumericControlValue(el, property) {
533
- if (!el?.state)
534
- return null;
535
- const stateValue = el.state[property];
536
- if (typeof stateValue === 'number' && Number.isFinite(stateValue))
537
- return stateValue;
538
- const parsedStateValue = ToolsInteract._parseNumericControlValue(stateValue);
539
- if (parsedStateValue !== null)
540
- return parsedStateValue;
541
- if (property === 'value' || property === 'raw_value') {
542
- const fallback = el.state.raw_value ?? el.state.value;
543
- if (typeof fallback === 'number' && Number.isFinite(fallback))
544
- return fallback;
545
- return ToolsInteract._parseNumericControlValue(fallback);
546
- }
547
- return null;
548
- }
549
- static _parseNumericControlValue(value) {
550
- if (typeof value === 'number' && Number.isFinite(value))
551
- return value;
552
- if (typeof value !== 'string')
553
- return null;
554
- const trimmed = value.trim();
555
- if (!trimmed)
556
- return null;
557
- const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/);
558
- if (percent)
559
- return Number(percent[1]);
560
- const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i);
561
- if (fraction)
562
- return Number(fraction[1]);
563
- const plain = Number(trimmed);
564
- if (Number.isFinite(plain))
565
- return plain;
566
- const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/);
567
- if (trailing)
568
- return Number(trailing[1]);
569
- return null;
570
- }
571
- static _readNumericControlRange(el) {
572
- const min = typeof el?.state?.value_range?.min === 'number' ? el.state.value_range.min : null;
573
- const max = typeof el?.state?.value_range?.max === 'number' ? el.state.value_range.max : null;
574
- if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
575
- return { min, max };
576
- }
577
- const value = el?.state?.value ?? el?.state?.raw_value;
578
- if (typeof value === 'string') {
579
- const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i);
580
- if (fraction) {
581
- const parsedMax = Number(fraction[1]);
582
- if (Number.isFinite(parsedMax) && parsedMax > 0)
583
- return { min: 0, max: parsedMax };
584
- }
585
- }
586
- return null;
587
- }
588
- static _readControlStep(el) {
589
- const candidates = [
590
- el?.state?.step,
591
- el?.state?.value_range?.step
592
- ];
593
- for (const candidate of candidates) {
594
- if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0)
595
- return candidate;
596
- }
597
- return null;
598
- }
599
- static _quantizeTargetValue(targetValue, min, max, step) {
600
- if (!step)
601
- return targetValue;
602
- const quantized = min + (Math.round((targetValue - min) / step) * step);
603
- const bounded = Math.max(min, Math.min(max, quantized));
604
- const precision = ToolsInteract._decimalPrecisionForStep(step);
605
- return Number(bounded.toFixed(precision));
606
- }
607
- static _decimalPrecisionForStep(step) {
608
- const normalized = String(step).toLowerCase();
609
- const [coefficient, exponentPart] = normalized.split('e');
610
- const coefficientDecimals = coefficient.includes('.')
611
- ? coefficient.split('.')[1]?.length ?? 0
612
- : 0;
613
- const exponent = exponentPart !== undefined ? Number(exponentPart) : 0;
614
- const precision = exponent < 0
615
- ? coefficientDecimals + Math.abs(exponent)
616
- : Math.max(0, coefficientDecimals - exponent);
617
- return Math.max(0, Math.min(12, precision));
618
- }
619
- static _defaultAdjustTolerance(range, step) {
620
- if (step)
621
- return 0;
622
- if (!range)
623
- return 0;
624
- return Math.max(0, 0.01 * Math.abs(range.max - range.min));
625
- }
626
- static _runtimeActionSet(el) {
627
- const values = [
628
- ...(Array.isArray(el?.actions) ? el.actions : []),
629
- ...(Array.isArray(el?.accessibilityActions) ? el.accessibilityActions : []),
630
- ...(Array.isArray(el?.runtimeActions) ? el.runtimeActions : []),
631
- ...(Array.isArray(el?.platformActions) ? el.platformActions : [])
632
- ];
633
- return new Set(values.map((value) => ToolsInteract._normalize(value)).filter(Boolean));
634
- }
635
- static _hasDirectAdjustmentAction(el) {
636
- const actions = ToolsInteract._runtimeActionSet(el);
637
- return [
638
- 'action_set_progress',
639
- 'set_progress',
640
- 'setprogress',
641
- 'semanticsactions.setprogress',
642
- 'set_value',
643
- 'setvalue',
644
- 'direct_set_value'
645
- ].some((action) => actions.has(action));
646
- }
647
- static _adjustmentDirections(el) {
648
- const actions = ToolsInteract._runtimeActionSet(el);
649
- return {
650
- increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
651
- decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
652
- };
653
- }
654
- static _textFromElement(el) {
655
- return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim();
656
- }
657
- static _parseLeadingNumber(text) {
658
- const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/);
659
- if (!match)
660
- return null;
661
- const value = Number(match[1]);
662
- return Number.isFinite(value) ? value : null;
663
- }
664
- static _inferLabeledSliderState(elements, track) {
665
- const trackBounds = ToolsInteract._normalizeBounds(track?.bounds);
666
- if (!track || !trackBounds)
667
- return null;
668
- const type = ToolsInteract._normalize(track.type ?? track.class ?? '');
669
- const role = ToolsInteract._normalize(track.role ?? '');
670
- const width = trackBounds[2] - trackBounds[0];
671
- const height = trackBounds[3] - trackBounds[1];
672
- const wideTrackLike = width >= 240 && height <= 220;
673
- const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role);
674
- if (!wideTrackLike && !knownTrackLike)
675
- return null;
676
- const trackTop = trackBounds[1];
677
- const trackBottom = trackBounds[3];
678
- const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2;
679
- const labels = elements
680
- .map((el, index) => {
681
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
682
- const text = ToolsInteract._textFromElement(el);
683
- const value = ToolsInteract._parseLeadingNumber(text);
684
- return bounds && text && value !== null
685
- ? {
686
- index,
687
- el,
688
- text,
689
- value,
690
- bounds,
691
- centerX: (bounds[0] + bounds[2]) / 2,
692
- centerY: (bounds[1] + bounds[3]) / 2,
693
- hasUnit: /[a-zA-Z%]/.test(text)
694
- }
695
- : null;
696
- })
697
- .filter((label) => !!label);
698
- const rangeLabels = labels
699
- .filter((label) => {
700
- const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260;
701
- const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120;
702
- return verticallyNearTrack && horizontallyNearTrack && label.hasUnit;
703
- })
704
- .sort((a, b) => a.centerX - b.centerX);
705
- if (rangeLabels.length < 2)
706
- return null;
707
- const minLabel = rangeLabels[0];
708
- const maxLabel = rangeLabels[rangeLabels.length - 1];
709
- if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value)
710
- return null;
711
- const currentCandidates = labels
712
- .filter((label) => {
713
- if (label.index === minLabel.index || label.index === maxLabel.index)
714
- return false;
715
- if (label.value < minLabel.value || label.value > maxLabel.value)
716
- return false;
717
- if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20)
718
- return false;
719
- return label.centerX >= trackCenterX - 80;
720
- })
721
- .sort((a, b) => {
722
- const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY);
723
- if (yDistance !== 0)
724
- return yDistance;
725
- return b.centerX - a.centerX;
726
- });
727
- const currentLabel = currentCandidates[0];
728
- if (!currentLabel)
729
- return null;
730
- const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop;
731
- const labelDerivedBounds = [
732
- Math.min(minLabel.bounds[0], trackBounds[0]),
733
- Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
734
- Math.max(maxLabel.bounds[2], trackBounds[2]),
735
- Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
736
- ];
737
- return {
738
- current: currentLabel.value,
739
- min: minLabel.value,
740
- max: maxLabel.value,
741
- value_range: { min: minLabel.value, max: maxLabel.value },
742
- touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
743
- source: 'nearby_labels'
744
- };
745
- }
746
- static _buildControlPoint(bounds, ratio, axis) {
747
- const clampedRatio = Math.max(0, Math.min(1, ratio));
748
- const [left, top, right, bottom] = bounds;
749
- const width = Math.max(1, right - left);
750
- const height = Math.max(1, bottom - top);
751
- const insetX = Math.max(8, Math.floor(width * 0.08));
752
- const insetY = Math.max(8, Math.floor(height * 0.08));
753
- if (axis === 'vertical') {
754
- const usableHeight = Math.max(1, height - (insetY * 2));
755
- return {
756
- x: Math.floor((left + right) / 2),
757
- y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
758
- };
759
- }
760
- const usableWidth = Math.max(1, width - (insetX * 2));
761
- return {
762
- x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
763
- y: Math.floor((top + bottom) / 2)
764
- };
765
- }
766
- static _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis) {
767
- const range = Math.max(1, max - min);
768
- const targetRatio = (targetValue - min) / range;
769
- const stepRatio = 1 / range;
770
- const centerBias = stepRatio / 2;
771
- const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue);
772
- const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0]);
773
- const edgeWindow = Math.max(3, Math.floor(range * 0.1));
774
- const isNearLowEdge = targetValue - min <= edgeWindow;
775
- const isNearHighEdge = max - targetValue <= edgeWindow;
776
- const directionBias = direction > 0
777
- ? -stepRatio * 0.15
778
- : direction < 0
779
- ? stepRatio * 0.65
780
- : 0;
781
- const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx));
782
- const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin);
783
- const edgeBias = isNearLowEdge
784
- ? endpointMargin
785
- : isNearHighEdge
786
- ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
787
- : 0;
788
- const safeRatio = Math.min(1 - (endpointMargin * 0.25), Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias));
789
- return ToolsInteract._buildControlPoint(bounds, safeRatio, axis);
790
- }
791
- static _buildAdjustmentProbePoints(bounds, targetValue, currentValue, min, max, axis) {
792
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis);
793
- const currentPoint = currentValue !== null
794
- ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
795
- : ToolsInteract._buildControlPoint(bounds, 0.5, axis);
796
- const [left, top, right, bottom] = bounds;
797
- const width = Math.max(1, right - left);
798
- const height = Math.max(1, bottom - top);
799
- const crossAxisBumps = axis === 'horizontal'
800
- ? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
801
- : [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))];
802
- const clampPoint = (point) => ({
803
- x: axis === 'horizontal'
804
- ? Math.max(left, Math.min(right, point.x))
805
- : Math.max(left, Math.min(right + Math.max(width, height), point.x)),
806
- y: axis === 'vertical'
807
- ? Math.max(top, Math.min(bottom, point.y))
808
- : Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
809
- });
810
- const probes = [targetPoint, currentPoint];
811
- for (const bump of crossAxisBumps) {
812
- if (axis === 'horizontal') {
813
- probes.push({ x: targetPoint.x, y: bottom + bump }, { x: currentPoint.x, y: bottom + bump });
814
- }
815
- else {
816
- probes.push({ x: right + bump, y: targetPoint.y }, { x: right + bump, y: currentPoint.y });
817
- }
818
- }
819
- return Array.from(new Map(probes
820
- .map(clampPoint)
821
- .map((point) => [`${point.x}:${point.y}`, point])).values());
822
- }
823
- static _controlAxis(el, bounds) {
824
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
825
- const role = ToolsInteract._normalize(el.role ?? '');
826
- if (/vertical/.test(type) || /vertical/.test(role))
827
- return 'vertical';
828
- if (/horizontal/.test(type) || /horizontal/.test(role))
829
- return 'horizontal';
830
- return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal';
831
- }
75
+ static _resolvedTargetFromElement = _resolvedTargetFromElement;
832
76
  static _actionFailure(actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter, sourceModule = 'interact') {
833
77
  return buildActionExecutionResult({
834
78
  actionType,
@@ -851,143 +95,43 @@ export class ToolsInteract {
851
95
  }
852
96
  return null;
853
97
  }
854
- static _resolveActionableAncestor(elements, chosen) {
855
- if (!chosen)
856
- return null;
857
- if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el))
858
- return chosen;
859
- let current = chosen;
860
- let safety = 0;
861
- while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
862
- const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId);
863
- if (parentIndex !== null && elements[parentIndex]) {
864
- current = { el: elements[parentIndex], idx: parentIndex };
865
- if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
866
- return current;
867
- }
868
- else {
869
- break;
870
- }
871
- safety++;
872
- }
873
- const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
874
- if (!childBounds)
875
- return null;
876
- const [cl, ct, cr, cb] = childBounds;
877
- let best = null;
878
- let bestArea = Infinity;
879
- for (let i = 0; i < elements.length; i++) {
880
- const el = elements[i];
881
- if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el)))
882
- continue;
883
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
884
- if (!bounds)
885
- continue;
886
- const [pl, pt, pr, pb] = bounds;
887
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
888
- const area = (pr - pl) * (pb - pt);
889
- if (area < bestArea) {
890
- bestArea = area;
891
- best = { el, idx: i };
892
- }
893
- }
894
- }
895
- return best;
896
- }
897
- static _resolveNearbyActionableControl(elements, chosen, screen) {
898
- if (!chosen)
899
- return null;
900
- const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
901
- if (!labelBounds)
902
- return null;
903
- const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds;
904
- const labelWidth = labelRight - labelLeft;
905
- const labelHeight = labelBottom - labelTop;
906
- const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0;
907
- const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0;
908
- const minTrackLengthPx = Math.max(ToolsInteract._sliderTrackMinLengthPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0);
909
- const maxTrackThicknessPx = Math.max(ToolsInteract._sliderTrackMaxThicknessPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0);
910
- let best = null;
911
- let bestScore = Infinity;
912
- for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
913
- const candidate = elements[i];
914
- if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false)
915
- continue;
916
- const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds);
917
- if (!candidateBounds)
918
- continue;
919
- const [left, top, right] = candidateBounds;
920
- const width = right - left;
921
- const height = candidateBounds[3] - top;
922
- const verticalGap = top - labelBottom;
923
- if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx)
924
- continue;
925
- const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left);
926
- if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx)
927
- continue;
928
- const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '');
929
- const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '');
930
- const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '');
931
- let score = verticalGap;
932
- const horizontalTrackLike = width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
933
- height <= maxTrackThicknessPx;
934
- const verticalTrackLike = height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
935
- width <= maxTrackThicknessPx;
936
- const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike;
937
- if (!candidateText && !candidateContent)
938
- score -= 18;
939
- if (trackLike)
940
- score -= 30;
941
- if (/view|layout|group|frame/i.test(candidateClass))
942
- score -= 10;
943
- if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio)
944
- score -= 8;
945
- if (candidateText || candidateContent)
946
- score += 20;
947
- if (score < bestScore) {
948
- bestScore = score;
949
- best = { el: candidate, idx: i, sliderLike: trackLike };
950
- }
951
- }
952
- return best;
953
- }
954
98
  static async getInteractionService(platform, deviceId) {
955
99
  const effectivePlatform = platform || 'android';
956
100
  const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
957
101
  const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract();
958
- return { interact: interact, resolved, platform: effectivePlatform };
102
+ return { interact, resolved, platform: effectivePlatform };
959
103
  }
960
104
  static async tapHandler({ platform, x, y, deviceId, timeoutMs }) {
961
105
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
962
106
  return await interact.tap(x, y, resolved.id, timeoutMs);
963
107
  }
108
+ static _getResolvedElement(elementId, platform, deviceId) {
109
+ const resolved = ToolsInteract._resolvedUiElements.get(elementId);
110
+ if (!resolved || (platform && resolved.platform !== platform) || (deviceId && resolved.deviceId !== deviceId))
111
+ return null;
112
+ return resolved;
113
+ }
114
+ static getResolvedElementTarget(elementId, platform, deviceId) {
115
+ const resolved = ToolsInteract._getResolvedElement(elementId, platform, deviceId);
116
+ return resolved ? { platform: resolved.platform, deviceId: resolved.deviceId } : null;
117
+ }
964
118
  static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }) {
965
119
  const actionType = 'tap_element';
966
120
  const selector = { elementId };
967
- const resolved = ToolsInteract._resolvedUiElements.get(elementId);
968
- if (!resolved || (platform && resolved.platform !== platform) || (deviceId && resolved.deviceId !== deviceId)) {
121
+ const resolved = ToolsInteract._getResolvedElement(elementId, platform, deviceId);
122
+ if (!resolved) {
969
123
  return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, null);
970
124
  }
971
125
  const fingerprintBefore = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null;
972
126
  const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
973
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform;
974
- const treeDeviceId = tree?.device?.id || resolved.deviceId;
975
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
976
- const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
977
- if (!currentMatch) {
978
- return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
979
- }
980
- const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
981
- const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform);
982
- if (!tapActionability.actionable) {
983
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
984
- }
985
- const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
986
- if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
987
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
988
- }
989
- const x = Math.floor((bounds[0] + bounds[2]) / 2);
990
- const y = Math.floor((bounds[1] + bounds[3]) / 2);
127
+ const tap = resolveTap(tree, resolved, {
128
+ current: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
129
+ target: (element, index) => ToolsInteract._resolvedTargetFromElement(resolved.elementId, element, index)
130
+ });
131
+ if ('failure' in tap)
132
+ return ToolsInteract._actionFailure(actionType, selector, tap.target, tap.failure, true, fingerprintBefore);
133
+ const resolvedTarget = tap.target;
134
+ const { x, y } = tap.point;
991
135
  const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId, timeoutMs });
992
136
  if (!tapResult.success) {
993
137
  const fingerprintAfterFailure = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null;
@@ -1005,763 +149,17 @@ export class ToolsInteract {
1005
149
  sourceModule: 'interact'
1006
150
  });
1007
151
  }
1008
- static async adjustControlHandler({ selector, element_id, property = 'value', targetValue, tolerance, maxAttempts, platform, deviceId }) {
1009
- const actionType = 'adjust_control';
1010
- const targetSelector = selector ?? (element_id ? { elementId: element_id } : null);
1011
- const requestedTolerance = typeof tolerance === 'number' && Number.isFinite(tolerance) ? Math.max(0, tolerance) : null;
1012
- const attemptsLimit = maxAttempts === undefined || maxAttempts === null
1013
- ? 5
1014
- : Math.max(1, Math.floor(Number(maxAttempts) || 1));
1015
- const sourcePlatform = platform || 'android';
1016
- let resolvedPlatform = sourcePlatform;
1017
- let resolvedDeviceId = deviceId;
1018
- const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null;
1019
- const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1020
- let semanticFallbackElement = null;
1021
- const traceSteps = [];
1022
- let traceAttemptIndex = 0;
1023
- const recordTraceStep = (stage, result, metadata) => {
1024
- traceSteps.push(createTraceStep({
1025
- stage,
1026
- timestamp: Date.now(),
1027
- result,
1028
- attemptIndex: traceAttemptIndex++,
1029
- metadata
1030
- }));
1031
- };
1032
- const buildFailure = (failureCode, reason, resolved, device, actualState, attempts, adjustmentMode = 'gesture', retryable = false, uiFingerprintAfter = null) => {
1033
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1034
- recordTraceStep('resolve', 'failure', {
1035
- reason,
1036
- failure_code: failureCode
1037
- });
1038
- }
1039
- if (!traceSteps.some((step) => step.stage === 'recover')) {
1040
- recordTraceStep('recover', retryable ? 'retry' : 'failure', {
1041
- reason,
1042
- failure_code: failureCode,
1043
- retry_allowed: retryable,
1044
- recovery_attempts: attempts,
1045
- retry_depth: attempts
1046
- });
1047
- }
1048
- const responseTolerance = requestedTolerance ?? 0;
1049
- const base = buildActionExecutionResult({
1050
- actionType,
1051
- sourceModule: 'interact',
1052
- device,
1053
- selector: targetSelector,
1054
- resolved,
1055
- success: false,
1056
- uiFingerprintBefore: fingerprintBefore,
1057
- uiFingerprintAfter,
1058
- failure: { failureCode, retryable },
1059
- details: {
1060
- target_value: targetValue,
1061
- tolerance: responseTolerance,
1062
- property,
1063
- attempts,
1064
- adjustment_mode: adjustmentMode,
1065
- actual_state: actualState,
1066
- converged: false,
1067
- within_tolerance: false,
1068
- reason
1069
- },
1070
- traceSteps
1071
- });
1072
- return {
1073
- ...base,
1074
- target_state: {
1075
- property,
1076
- target_value: targetValue,
1077
- tolerance: responseTolerance
1078
- },
1079
- actual_state: actualState,
1080
- within_tolerance: false,
1081
- converged: false,
1082
- attempts,
1083
- adjustment_mode: adjustmentMode
1084
- };
1085
- };
1086
- const resolveCurrentMatch = async () => {
1087
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
1088
- resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform;
1089
- resolvedDeviceId = tree?.device?.id || resolvedDeviceId;
1090
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
1091
- if (element_id) {
1092
- const stored = ToolsInteract._resolvedUiElements.get(element_id);
1093
- if (!stored) {
1094
- return null;
1095
- }
1096
- const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored);
1097
- if (!current) {
1098
- return null;
1099
- }
1100
- return {
1101
- tree,
1102
- device: tree?.device,
1103
- match: { el: current.el, idx: current.index },
1104
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index), current.el, current.index)
1105
- };
1106
- }
1107
- if (semanticFallbackElement) {
1108
- const fallbackBounds = ToolsInteract._normalizeBounds(Array.isArray(semanticFallbackElement.bounds)
1109
- ? semanticFallbackElement.bounds
1110
- : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
1111
- ? [
1112
- Number(semanticFallbackElement.bounds.left),
1113
- Number(semanticFallbackElement.bounds.top),
1114
- Number(semanticFallbackElement.bounds.right),
1115
- Number(semanticFallbackElement.bounds.bottom)
1116
- ]
1117
- : null);
1118
- let matchedIndex = -1;
1119
- if (fallbackBounds) {
1120
- matchedIndex = elements.findIndex((el) => {
1121
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
1122
- return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3];
1123
- });
1124
- }
1125
- if (matchedIndex === -1 && fallbackBounds) {
1126
- const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2);
1127
- const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2);
1128
- let bestDistance = Infinity;
1129
- for (let i = 0; i < elements.length; i++) {
1130
- const el = elements[i];
1131
- if (!ToolsInteract._isAdjustableControl(el))
1132
- continue;
1133
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
1134
- if (!bounds)
1135
- continue;
1136
- const centerX = Math.floor((bounds[0] + bounds[2]) / 2);
1137
- const centerY = Math.floor((bounds[1] + bounds[3]) / 2);
1138
- const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY);
1139
- if (distance < bestDistance) {
1140
- bestDistance = distance;
1141
- matchedIndex = i;
1142
- }
1143
- }
1144
- }
1145
- if (matchedIndex >= 0 && elements[matchedIndex]) {
1146
- const matched = { el: elements[matchedIndex], idx: matchedIndex };
1147
- return {
1148
- tree,
1149
- device: tree?.device,
1150
- match: matched,
1151
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
1152
- };
1153
- }
1154
- }
1155
- if (selector) {
1156
- const matched = ToolsInteract._findFirstMatchingElement(elements, selector);
1157
- if (!matched) {
1158
- return null;
1159
- }
1160
- return {
1161
- tree,
1162
- device: tree?.device,
1163
- match: matched,
1164
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
1165
- };
1166
- }
1167
- return null;
1168
- };
1169
- if ((!selector && !element_id) || (selector && element_id)) {
1170
- return buildFailure('ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', null, undefined, null, 0, 'gesture', false);
1171
- }
1172
- if (selector && !element_id) {
1173
- const waitResult = await ToolsInteract.waitForUIHandler({
1174
- selector,
1175
- condition: 'clickable',
1176
- timeout_ms: 5000,
1177
- poll_interval_ms: 300,
1178
- platform: resolvedPlatform,
1179
- deviceId: resolvedDeviceId
1180
- });
1181
- if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
1182
- const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? '';
1183
- if (!semanticQuery) {
1184
- return buildFailure(waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', waitResult?.error?.code === 'ELEMENT_NOT_FOUND');
1185
- }
1186
- const fallback = await ToolsInteract.findElementHandler({
1187
- query: semanticQuery,
1188
- exact: false,
1189
- timeoutMs: 3000,
1190
- platform: resolvedPlatform,
1191
- deviceId: resolvedDeviceId
1192
- });
1193
- if (!fallback.found || !fallback.element) {
1194
- return buildFailure('ELEMENT_NOT_FOUND', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', true);
1195
- }
1196
- semanticFallbackElement = fallback.element;
1197
- }
1198
- else {
1199
- element_id = waitResult.element.elementId;
1200
- semanticFallbackElement = null;
1201
- }
1202
- }
1203
- let lastObservedState = null;
1204
- let lastAdjustmentMode = 'gesture';
1205
- let resolvedTarget = null;
1206
- let currentDevice = undefined;
1207
- let attemptCount = 0;
1208
- let cachedResolvedMatch = null;
1209
- let coordinateTapsAwaitingFreshGesture = false;
1210
- adjustmentAttempts: for (let attempt = 0; attempt < attemptsLimit && attemptCount < attemptsLimit; attempt++) {
1211
- const resolved = cachedResolvedMatch
1212
- ? {
1213
- tree: null,
1214
- device: currentDevice,
1215
- match: cachedResolvedMatch,
1216
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx), cachedResolvedMatch.el, cachedResolvedMatch.idx)
1217
- }
1218
- : await resolveCurrentMatch();
1219
- if (!resolved || !resolved.match || !resolved.resolvedTarget) {
1220
- return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
1221
- }
1222
- currentDevice = resolved.device;
1223
- resolvedTarget = resolved.resolvedTarget;
1224
- const currentEl = resolved.match.el;
1225
- cachedResolvedMatch = resolved.match;
1226
- const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id);
1227
- if (!adjustableActionability.actionable) {
1228
- return buildFailure(adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', adjustableActionability.reason ?? 'adjustable control is not actionable', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
1229
- }
1230
- const elementsForInference = Array.isArray(resolved.tree?.elements)
1231
- ? resolved.tree.elements
1232
- : cachedResolvedMatch
1233
- ? [cachedResolvedMatch.el]
1234
- : [currentEl];
1235
- const inferredSliderState = ToolsInteract._inferLabeledSliderState(elementsForInference, currentEl);
1236
- const nativeBounds = ToolsInteract._normalizeBounds(currentEl.bounds);
1237
- const bounds = inferredSliderState?.touch_bounds ?? nativeBounds;
1238
- const valueRange = ToolsInteract._readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null;
1239
- const step = ToolsInteract._readControlStep(currentEl);
1240
- const normalizedTargetValue = valueRange
1241
- ? ToolsInteract._quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step)
1242
- : targetValue;
1243
- const effectiveTolerance = requestedTolerance ?? ToolsInteract._defaultAdjustTolerance(valueRange, step);
1244
- const currentValue = ToolsInteract._readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null;
1245
- const actualState = currentValue !== null
1246
- ? {
1247
- property,
1248
- value: currentValue,
1249
- raw_value: typeof currentEl.state?.raw_value === 'number'
1250
- ? currentEl.state.raw_value
1251
- : inferredSliderState?.current
1252
- }
1253
- : null;
1254
- lastObservedState = actualState;
1255
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1256
- recordTraceStep('resolve', 'success', {
1257
- resolved_target: resolvedTarget,
1258
- current_value: currentValue,
1259
- adjustment_mode: lastAdjustmentMode
1260
- });
1261
- }
1262
- if (property !== 'value' && property !== 'raw_value') {
1263
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1264
- }
1265
- const isAdjustableControl = ToolsInteract._isAdjustableControl(currentEl) || !!inferredSliderState;
1266
- if (!isAdjustableControl) {
1267
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1268
- }
1269
- if (currentValue === null) {
1270
- return buildFailure('UNKNOWN', 'current numeric value is unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1271
- }
1272
- if (valueRange && (targetValue < valueRange.min || targetValue > valueRange.max)) {
1273
- return buildFailure('CONTROL_CONVERGENCE_FAILED', `targetValue ${targetValue} is outside the control range ${valueRange.min}..${valueRange.max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1274
- }
1275
- if (currentValue !== null && Math.abs(currentValue - normalizedTargetValue) <= effectiveTolerance) {
1276
- recordTraceStep('verify', 'success', {
1277
- property,
1278
- target_value: normalizedTargetValue,
1279
- actual_state: actualState,
1280
- reason: 'control already within tolerance'
1281
- });
1282
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1283
- const base = buildActionExecutionResult({
1284
- actionType,
1285
- sourceModule: 'interact',
1286
- device: currentDevice,
1287
- selector: targetSelector,
1288
- resolved: resolvedTarget,
1289
- success: true,
1290
- uiFingerprintBefore: fingerprintBefore,
1291
- uiFingerprintAfter,
1292
- details: {
1293
- target_value: normalizedTargetValue,
1294
- tolerance: effectiveTolerance,
1295
- property,
1296
- attempts: attemptCount,
1297
- adjustment_mode: 'semantic',
1298
- actual_state: actualState,
1299
- converged: true,
1300
- within_tolerance: true,
1301
- reason: 'control already within tolerance'
1302
- },
1303
- traceSteps
1304
- });
1305
- return {
1306
- ...base,
1307
- target_state: {
1308
- property,
1309
- target_value: normalizedTargetValue,
1310
- tolerance: effectiveTolerance
1311
- },
1312
- actual_state: actualState,
1313
- within_tolerance: true,
1314
- converged: true,
1315
- attempts: attemptCount,
1316
- adjustment_mode: 'semantic'
1317
- };
1318
- }
1319
- if (!bounds) {
1320
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1321
- }
1322
- const runVerification = async () => {
1323
- if (inferredSliderState) {
1324
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
1325
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
1326
- const refreshed = ToolsInteract._inferLabeledSliderState(elements, currentEl);
1327
- const observedValue = refreshed?.current ?? null;
1328
- const observedState = observedValue !== null
1329
- ? { property, value: observedValue, raw_value: observedValue }
1330
- : actualState;
1331
- return {
1332
- verification: {
1333
- success: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance,
1334
- observed_state: observedState,
1335
- reason: observedValue !== null
1336
- ? 'visible labeled value read from nearby slider labels'
1337
- : 'visible labeled value unavailable'
1338
- },
1339
- observedState,
1340
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1341
- };
1342
- }
1343
- const verification = await ToolsInteract.expectStateHandler({
1344
- element_id: resolvedTarget?.elementId ?? element_id,
1345
- selector: selector ?? undefined,
1346
- property,
1347
- expected: normalizedTargetValue,
1348
- platform: resolvedPlatform,
1349
- deviceId: resolvedDeviceId
1350
- });
1351
- const observedValue = ToolsInteract._parseNumericControlValue(verification?.observed_state?.value) ??
1352
- ToolsInteract._parseNumericControlValue(verification?.observed_state?.raw_value);
1353
- const observedState = observedValue !== null
1354
- ? {
1355
- property,
1356
- value: observedValue,
1357
- raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
1358
- }
1359
- : actualState;
1360
- return {
1361
- verification,
1362
- observedState,
1363
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1364
- };
1365
- };
1366
- let actionDevice = currentDevice;
1367
- let observedState = actualState;
1368
- let verification = null;
1369
- let verificationResult = { verification: null, observedState: actualState, withinTolerance: false };
1370
- const directActionSupported = ToolsInteract._hasDirectAdjustmentAction(currentEl);
1371
- if (directActionSupported && attemptCount < attemptsLimit) {
1372
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId);
1373
- if (typeof interact.setAdjustableValue === 'function') {
1374
- lastAdjustmentMode = 'semantic';
1375
- recordTraceStep('execute', 'retry', {
1376
- attempt: attemptCount + 1,
1377
- mode: 'semantic',
1378
- strategy: 'direct_set_value',
1379
- target_value: normalizedTargetValue
1380
- });
1381
- const actionResult = await interact.setAdjustableValue({
1382
- deviceId: actionResolved.id,
1383
- target: resolvedTarget,
1384
- property,
1385
- value: normalizedTargetValue
1386
- });
1387
- attemptCount++;
1388
- actionDevice = actionResult?.device ?? actionDevice;
1389
- if (actionResult?.success !== false) {
1390
- verificationResult = await runVerification();
1391
- observedState = verificationResult.observedState;
1392
- lastObservedState = observedState;
1393
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1394
- attempt: attemptCount,
1395
- property,
1396
- target_value: normalizedTargetValue,
1397
- actual_state: observedState,
1398
- reason: verificationResult.verification?.reason ?? 'direct adjustment did not converge yet'
1399
- });
1400
- if (verificationResult.withinTolerance) {
1401
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1402
- const base = buildActionExecutionResult({
1403
- actionType,
1404
- sourceModule: 'interact',
1405
- device: actionDevice ?? currentDevice,
1406
- selector: targetSelector,
1407
- resolved: resolvedTarget,
1408
- success: true,
1409
- uiFingerprintBefore: fingerprintBefore,
1410
- uiFingerprintAfter,
1411
- details: {
1412
- target_value: normalizedTargetValue,
1413
- tolerance: effectiveTolerance,
1414
- property,
1415
- attempts: attemptCount,
1416
- adjustment_mode: lastAdjustmentMode,
1417
- actual_state: observedState,
1418
- converged: true,
1419
- within_tolerance: true,
1420
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1421
- },
1422
- traceSteps
1423
- });
1424
- return {
1425
- ...base,
1426
- target_state: {
1427
- property,
1428
- target_value: normalizedTargetValue,
1429
- tolerance: effectiveTolerance
1430
- },
1431
- actual_state: observedState,
1432
- within_tolerance: true,
1433
- converged: true,
1434
- attempts: attemptCount,
1435
- adjustment_mode: lastAdjustmentMode
1436
- };
1437
- }
1438
- cachedResolvedMatch = null;
1439
- if (attemptCount < attemptsLimit) {
1440
- continue;
1441
- }
1442
- }
1443
- }
1444
- }
1445
- const directions = ToolsInteract._adjustmentDirections(currentEl);
1446
- if ((directions.increment || directions.decrement) && attemptCount < attemptsLimit) {
1447
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId);
1448
- const adjustByDirection = typeof interact.adjustAccessibleValue === 'function'
1449
- ? interact.adjustAccessibleValue.bind(interact)
1450
- : null;
1451
- while (adjustByDirection && attemptCount < attemptsLimit) {
1452
- const direction = normalizedTargetValue > (observedState?.value ?? currentValue) ? 'increment' : 'decrement';
1453
- if ((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement))
1454
- break;
1455
- lastAdjustmentMode = 'semantic';
1456
- recordTraceStep('execute', 'retry', {
1457
- attempt: attemptCount + 1,
1458
- mode: 'semantic',
1459
- strategy: 'accessibility_increment_decrement',
1460
- direction
1461
- });
1462
- const actionResult = await adjustByDirection({
1463
- deviceId: actionResolved.id,
1464
- target: resolvedTarget,
1465
- property,
1466
- direction
1467
- });
1468
- attemptCount++;
1469
- actionDevice = actionResult?.device ?? actionDevice;
1470
- if (actionResult?.success === false)
1471
- break;
1472
- verificationResult = await runVerification();
1473
- observedState = verificationResult.observedState;
1474
- lastObservedState = observedState;
1475
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1476
- attempt: attemptCount,
1477
- property,
1478
- target_value: normalizedTargetValue,
1479
- actual_state: observedState,
1480
- reason: verificationResult.verification?.reason ?? 'incremental adjustment did not converge yet'
1481
- });
1482
- if (verificationResult.withinTolerance) {
1483
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1484
- const base = buildActionExecutionResult({
1485
- actionType,
1486
- sourceModule: 'interact',
1487
- device: actionDevice ?? currentDevice,
1488
- selector: targetSelector,
1489
- resolved: resolvedTarget,
1490
- success: true,
1491
- uiFingerprintBefore: fingerprintBefore,
1492
- uiFingerprintAfter,
1493
- details: {
1494
- target_value: normalizedTargetValue,
1495
- tolerance: effectiveTolerance,
1496
- property,
1497
- attempts: attemptCount,
1498
- adjustment_mode: lastAdjustmentMode,
1499
- actual_state: observedState,
1500
- converged: true,
1501
- within_tolerance: true,
1502
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1503
- },
1504
- traceSteps
1505
- });
1506
- return {
1507
- ...base,
1508
- target_state: {
1509
- property,
1510
- target_value: normalizedTargetValue,
1511
- tolerance: effectiveTolerance
1512
- },
1513
- actual_state: observedState,
1514
- within_tolerance: true,
1515
- converged: true,
1516
- attempts: attemptCount,
1517
- adjustment_mode: lastAdjustmentMode
1518
- };
1519
- }
1520
- cachedResolvedMatch = null;
1521
- if (attemptCount < attemptsLimit) {
1522
- continue adjustmentAttempts;
1523
- }
1524
- }
1525
- }
1526
- if (!bounds) {
1527
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1528
- }
1529
- if (!valueRange) {
1530
- return buildFailure('UNKNOWN', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1531
- }
1532
- const min = valueRange.min;
1533
- const max = valueRange.max;
1534
- const axis = ToolsInteract._controlAxis(currentEl, bounds);
1535
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, normalizedTargetValue, currentValue, min, max, axis);
1536
- const currentPoint = ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis);
1537
- if (!coordinateTapsAwaitingFreshGesture) {
1538
- const probePoints = inferredSliderState
1539
- ? [targetPoint]
1540
- : ToolsInteract._buildAdjustmentProbePoints(bounds, normalizedTargetValue, currentValue, min, max, axis);
1541
- for (let i = 0; i < probePoints.length && attemptCount < attemptsLimit; i++) {
1542
- const probePoint = probePoints[i];
1543
- lastAdjustmentMode = 'coordinate';
1544
- recordTraceStep('execute', 'retry', {
1545
- attempt: attemptCount + 1,
1546
- mode: 'coordinate',
1547
- point: probePoint
1548
- });
1549
- const actionResult = await ToolsInteract.tapHandler({
1550
- platform: resolvedPlatform,
1551
- x: probePoint.x,
1552
- y: probePoint.y,
1553
- deviceId: resolvedDeviceId
1554
- });
1555
- attemptCount++;
1556
- actionDevice = actionResult.device ?? actionDevice;
1557
- if (!actionResult.success) {
1558
- recordTraceStep('execute', 'retry', {
1559
- attempt: attemptCount,
1560
- mode: 'coordinate',
1561
- point: probePoint,
1562
- success: false
1563
- });
1564
- continue;
1565
- }
1566
- verificationResult = await runVerification();
1567
- observedState = verificationResult.observedState;
1568
- lastObservedState = observedState;
1569
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1570
- attempt: attemptCount,
1571
- property,
1572
- target_value: normalizedTargetValue,
1573
- actual_state: observedState,
1574
- reason: verificationResult.verification?.reason ?? 'control did not converge yet'
1575
- });
1576
- if (verificationResult.withinTolerance) {
1577
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1578
- const base = buildActionExecutionResult({
1579
- actionType,
1580
- sourceModule: 'interact',
1581
- device: actionDevice ?? currentDevice,
1582
- selector: targetSelector,
1583
- resolved: resolvedTarget,
1584
- success: true,
1585
- uiFingerprintBefore: fingerprintBefore,
1586
- uiFingerprintAfter,
1587
- details: {
1588
- target_value: normalizedTargetValue,
1589
- tolerance: effectiveTolerance,
1590
- property,
1591
- attempts: attemptCount,
1592
- adjustment_mode: lastAdjustmentMode,
1593
- actual_state: observedState,
1594
- converged: true,
1595
- within_tolerance: true,
1596
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1597
- },
1598
- traceSteps
1599
- });
1600
- return {
1601
- ...base,
1602
- target_state: {
1603
- property,
1604
- target_value: normalizedTargetValue,
1605
- tolerance: effectiveTolerance
1606
- },
1607
- actual_state: observedState,
1608
- within_tolerance: true,
1609
- converged: true,
1610
- attempts: attemptCount,
1611
- adjustment_mode: lastAdjustmentMode
1612
- };
1613
- }
1614
- coordinateTapsAwaitingFreshGesture = true;
1615
- cachedResolvedMatch = null;
1616
- if (attemptCount < attemptsLimit) {
1617
- continue adjustmentAttempts;
1618
- }
1619
- }
1620
- }
1621
- if (attemptCount < attemptsLimit) {
1622
- coordinateTapsAwaitingFreshGesture = false;
1623
- lastAdjustmentMode = 'gesture';
1624
- recordTraceStep('execute', 'retry', {
1625
- attempt: attemptCount + 1,
1626
- mode: 'gesture',
1627
- start: currentPoint,
1628
- end: targetPoint
1629
- });
1630
- const fallbackActionResult = await ToolsInteract.swipeHandler({
1631
- platform: resolvedPlatform,
1632
- x1: currentPoint.x,
1633
- y1: currentPoint.y,
1634
- x2: targetPoint.x,
1635
- y2: targetPoint.y,
1636
- duration: inferredSliderState ? 520 : 220,
1637
- deviceId: resolvedDeviceId
1638
- });
1639
- attemptCount++;
1640
- if (!fallbackActionResult.success) {
1641
- recordTraceStep('execute', 'failure', {
1642
- attempt: attemptCount,
1643
- mode: 'gesture',
1644
- start: currentPoint,
1645
- end: targetPoint,
1646
- success: false
1647
- });
1648
- return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false);
1649
- }
1650
- actionDevice = fallbackActionResult.device ?? actionDevice;
1651
- verificationResult = await runVerification();
1652
- observedState = verificationResult.observedState;
1653
- lastObservedState = observedState;
1654
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1655
- attempt: attemptCount,
1656
- property,
1657
- target_value: normalizedTargetValue,
1658
- actual_state: observedState,
1659
- reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
1660
- });
1661
- if (verificationResult.withinTolerance) {
1662
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1663
- const base = buildActionExecutionResult({
1664
- actionType,
1665
- sourceModule: 'interact',
1666
- device: actionDevice ?? currentDevice,
1667
- selector: targetSelector,
1668
- resolved: resolvedTarget,
1669
- success: true,
1670
- uiFingerprintBefore: fingerprintBefore,
1671
- uiFingerprintAfter,
1672
- details: {
1673
- target_value: normalizedTargetValue,
1674
- tolerance: effectiveTolerance,
1675
- property,
1676
- attempts: attemptCount,
1677
- adjustment_mode: lastAdjustmentMode,
1678
- actual_state: observedState,
1679
- converged: true,
1680
- within_tolerance: true,
1681
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1682
- },
1683
- traceSteps
1684
- });
1685
- return {
1686
- ...base,
1687
- target_state: {
1688
- property,
1689
- target_value: normalizedTargetValue,
1690
- tolerance: effectiveTolerance
1691
- },
1692
- actual_state: observedState,
1693
- within_tolerance: true,
1694
- converged: true,
1695
- attempts: attemptCount,
1696
- adjustment_mode: lastAdjustmentMode
1697
- };
1698
- }
1699
- }
1700
- verification = verificationResult.verification;
1701
- lastObservedState = observedState;
1702
- if (verificationResult.withinTolerance) {
1703
- recordTraceStep('verify', 'success', {
1704
- attempt: attemptCount,
1705
- property,
1706
- target_value: normalizedTargetValue,
1707
- actual_state: observedState,
1708
- reason: verification?.reason ?? 'control converged to target value'
1709
- });
1710
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1711
- const base = buildActionExecutionResult({
1712
- actionType,
1713
- sourceModule: 'interact',
1714
- device: actionDevice ?? currentDevice,
1715
- selector: targetSelector,
1716
- resolved: resolvedTarget,
1717
- success: true,
1718
- uiFingerprintBefore: fingerprintBefore,
1719
- uiFingerprintAfter,
1720
- details: {
1721
- target_value: normalizedTargetValue,
1722
- tolerance: effectiveTolerance,
1723
- property,
1724
- attempts: attemptCount,
1725
- adjustment_mode: lastAdjustmentMode,
1726
- actual_state: observedState,
1727
- converged: true,
1728
- within_tolerance: true,
1729
- reason: verification?.reason ?? 'control converged to target value'
1730
- },
1731
- traceSteps
1732
- });
1733
- return {
1734
- ...base,
1735
- target_state: {
1736
- property,
1737
- target_value: normalizedTargetValue,
1738
- tolerance: effectiveTolerance
1739
- },
1740
- actual_state: observedState,
1741
- within_tolerance: true,
1742
- converged: true,
1743
- attempts: attemptCount,
1744
- adjustment_mode: lastAdjustmentMode
1745
- };
1746
- }
1747
- cachedResolvedMatch = inferredSliderState
1748
- ? null
1749
- : {
1750
- el: {
1751
- ...currentEl,
1752
- state: {
1753
- ...(currentEl.state ?? null),
1754
- ...(observedState ? {
1755
- [observedState.property]: observedState.value,
1756
- raw_value: observedState.raw_value ?? observedState.value
1757
- } : {})
1758
- }
1759
- },
1760
- idx: resolved.match.idx
1761
- };
1762
- }
1763
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1764
- return buildFailure('CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter);
152
+ static async adjustControlHandler(input) {
153
+ return adjustControl(input, {
154
+ observe: (platform, deviceId) => ToolsObserve.getUITreeHandler({ platform, deviceId }),
155
+ fingerprint: (platform, deviceId) => ToolsInteract._captureFingerprint(platform, deviceId),
156
+ getHandle: id => ToolsInteract._resolvedUiElements.get(id),
157
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
158
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx),
159
+ wait: input => ToolsInteract.waitForUIHandler(input), find: input => ToolsInteract.findElementHandler(input),
160
+ expectState: input => ToolsInteract.expectStateHandler(input), service: (platform, deviceId) => ToolsInteract.getInteractionService(platform, deviceId),
161
+ tap: input => ToolsInteract.tapHandler(input), swipe: input => ToolsInteract.swipeHandler(input)
162
+ });
1765
163
  }
1766
164
  static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId, timeoutMs }) {
1767
165
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
@@ -1779,1446 +177,33 @@ export class ToolsInteract {
1779
177
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
1780
178
  return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id);
1781
179
  }
1782
- static async findElementHandler({ query, exact = false, timeoutMs = 3000, platform, deviceId }) {
1783
- // Try to use observe layer to fetch the current UI tree and perform a fast semantic search
1784
- const start = Date.now();
1785
- const deadline = start + timeoutMs;
1786
- const normalize = ToolsInteract._normalize;
1787
- const q = normalize(query);
1788
- if (!q)
1789
- return { found: false, error: 'Empty query' };
1790
- let best = null;
1791
- let bestTree = null;
1792
- let bestIterationCandidates = [];
1793
- let shouldStop = false;
1794
- const scoreElement = (el, idx) => {
1795
- if (!el || !el.visible)
1796
- return null;
1797
- const bounds = el.bounds || [0, 0, 0, 0];
1798
- if (!Array.isArray(bounds) || bounds.length < 4)
1799
- return null;
1800
- const [l, t, r, b] = bounds;
1801
- if (r <= l || b <= t)
1802
- return null;
1803
- // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
1804
- const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el));
1805
- const text = normalize(el.text ?? el.label ?? el.value ?? '');
1806
- const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
1807
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
1808
- const className = normalize(el.type ?? el.class ?? '');
1809
- const semanticRole = normalize(el.semantic?.semantic_role ?? '');
1810
- const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : [];
1811
- let score = 0;
1812
- let reason = 'best_scoring_candidate';
1813
- if (exact) {
1814
- if (text && text === q) {
1815
- score = 1.0;
1816
- reason = 'exact_text_match';
1817
- }
1818
- else if (content && content === q) {
1819
- score = 0.95;
1820
- reason = 'exact_content_desc_match';
1821
- }
1822
- else if (resourceId && resourceId === q) {
1823
- score = 0.92;
1824
- reason = 'exact_resource_id_match';
1825
- }
1826
- else if (className && className === q) {
1827
- score = 0.3;
1828
- reason = 'exact_class_match';
1829
- }
1830
- }
1831
- else {
1832
- if (text && text === q) {
1833
- score = 1.0;
1834
- reason = 'exact_text_match';
1835
- }
1836
- else if (content && content === q) {
1837
- score = 0.95;
1838
- reason = 'exact_content_desc_match';
1839
- }
1840
- else if (resourceId && resourceId === q) {
1841
- score = 0.92;
1842
- reason = 'exact_resource_id_match';
1843
- }
1844
- else if (text && text.includes(q)) {
1845
- score = 0.6;
1846
- reason = 'partial_text_match';
1847
- }
1848
- else if (content && content.includes(q)) {
1849
- score = 0.55;
1850
- reason = 'partial_content_desc_match';
1851
- }
1852
- else if (resourceId && resourceId.includes(q)) {
1853
- score = 0.7;
1854
- reason = 'partial_resource_id_match';
1855
- }
1856
- else if (className && className.includes(q)) {
1857
- score = 0.3;
1858
- reason = 'partial_class_match';
1859
- }
1860
- }
1861
- if (!exact) {
1862
- if (!score && semanticRole && semanticRole.includes(q)) {
1863
- score = 0.5;
1864
- reason = 'semantic_role_match';
1865
- }
1866
- if (semanticActions.some((action) => action.includes(q))) {
1867
- score = Math.max(score, score > 0 ? 0.65 : 0.6);
1868
- reason = 'semantic_action_match';
1869
- }
1870
- if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
1871
- score = 0.45;
1872
- reason = 'semantic_control_match';
1873
- }
1874
- }
1875
- else {
1876
- if (!score && semanticRole && semanticRole === q) {
1877
- score = 0.5;
1878
- reason = 'semantic_role_match';
1879
- }
1880
- if (semanticActions.some((action) => action === q)) {
1881
- score = Math.max(score, score > 0 ? 0.65 : 0.6);
1882
- reason = 'semantic_action_match';
1883
- }
1884
- }
1885
- if (score > 0 && interactable)
1886
- score += 0.05;
1887
- if (score <= 0)
1888
- return null;
1889
- return { el, idx, score, reason, interactable };
1890
- };
1891
- while (Date.now() <= deadline) {
1892
- try {
1893
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
1894
- if (tree && Array.isArray(tree.elements)) {
1895
- const elements = tree.elements;
1896
- const iterationCandidates = [];
1897
- let iterationImprovedBest = false;
1898
- for (let i = 0; i < elements.length; i++) {
1899
- const el = elements[i];
1900
- try {
1901
- const candidate = scoreElement(el, i);
1902
- if (!candidate)
1903
- continue;
1904
- iterationCandidates.push(candidate);
1905
- if (!best || candidate.score > best.score) {
1906
- best = candidate;
1907
- bestTree = tree;
1908
- iterationImprovedBest = true;
1909
- if (best.score >= 0.95) {
1910
- shouldStop = true;
1911
- break;
1912
- }
1913
- }
1914
- }
1915
- catch (e) {
1916
- checkDebuggerGuard();
1917
- console.error('Error scoring element:', e);
1918
- }
1919
- }
1920
- if (iterationImprovedBest) {
1921
- bestIterationCandidates = iterationCandidates.slice();
1922
- }
1923
- }
1924
- }
1925
- catch (e) {
1926
- checkDebuggerGuard();
1927
- console.error('Error fetching UI tree:', e);
1928
- }
1929
- if (shouldStop || Date.now() > deadline)
1930
- break;
1931
- await new Promise(r => setTimeout(r, 100));
1932
- }
1933
- if (!best)
1934
- return { found: false, error: 'Element not found' };
1935
- // If the best match is not interactable, try to resolve an actionable ancestor.
1936
- try {
1937
- const elements = (bestTree && Array.isArray(bestTree.elements)) ? bestTree.elements : [];
1938
- const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution : null;
1939
- let chosen = best;
1940
- const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null;
1941
- // Strategy 1: if parentId references an index, climb that chain
1942
- let resolvedAncestor = null;
1943
- if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
1944
- let cur = chosen;
1945
- let safety = 0;
1946
- while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
1947
- let pid = cur.el.parentId;
1948
- let idx = null;
1949
- if (typeof pid === 'number')
1950
- idx = pid;
1951
- else if (typeof pid === 'string' && /^\d+$/.test(pid))
1952
- idx = Number(pid);
1953
- // If parentId is not an index, try to find by matching resourceId or id field
1954
- if (idx !== null && elements[idx]) {
1955
- cur = { el: elements[idx], idx };
1956
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) {
1957
- resolvedAncestor = cur;
1958
- break;
1959
- }
1960
- }
1961
- else if (typeof pid === 'string') {
1962
- // fallback: search elements for matching resourceId or id
1963
- const foundIndex = elements.findIndex((el) => (el.resourceId === pid || el.id === pid));
1964
- const found = foundIndex >= 0 ? elements[foundIndex] : null;
1965
- if (found) {
1966
- cur = { el: found, idx: foundIndex };
1967
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) {
1968
- resolvedAncestor = cur;
1969
- break;
1970
- }
1971
- // otherwise continue climbing if this found element has its own parentId
1972
- }
1973
- else {
1974
- break;
1975
- }
1976
- }
1977
- else {
1978
- break;
1979
- }
1980
- safety++;
1981
- }
1982
- }
1983
- // Strategy 2: fallback - find a clickable element whose bounds fully contain the child's bounds
1984
- if (!resolvedAncestor && childBounds) {
1985
- const [cl, ct, cr, cb] = childBounds;
1986
- // find candidates that are clickable and contain the child bounds
1987
- const candidates = elements
1988
- .map((el, idx) => ({ el, idx }))
1989
- .filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds.length >= 4);
1990
- let bestCandidate = null;
1991
- let bestCandidateArea = Infinity;
1992
- for (const c of candidates) {
1993
- const bounds = c.el.bounds;
1994
- const [pl, pt, pr, pb] = bounds;
1995
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
1996
- const area = (pr - pl) * (pb - pt);
1997
- if (area < bestCandidateArea) {
1998
- bestCandidateArea = area;
1999
- bestCandidate = c;
2000
- }
2001
- }
2002
- }
2003
- if (bestCandidate)
2004
- resolvedAncestor = bestCandidate;
2005
- }
2006
- if (resolvedAncestor) {
2007
- best = {
2008
- el: resolvedAncestor.el,
2009
- idx: resolvedAncestor.idx,
2010
- score: Math.min(1, best.score + 0.02),
2011
- reason: 'clickable_parent_preferred',
2012
- interactable: true
2013
- };
2014
- }
2015
- if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
2016
- const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen);
2017
- if (nearbyActionable) {
2018
- best = {
2019
- el: nearbyActionable.el,
2020
- idx: nearbyActionable.idx,
2021
- score: Math.min(1, best.score + 0.02),
2022
- reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
2023
- interactable: true
2024
- };
2025
- }
2026
- }
2027
- }
2028
- catch (e) {
2029
- checkDebuggerGuard();
2030
- console.error('Error resolving ancestor:', e);
2031
- }
2032
- if (!best)
2033
- return { found: false, error: 'Element not found' };
2034
- const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null;
2035
- const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null;
2036
- const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el));
2037
- const alternateCandidates = uniqueRanked
2038
- .filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
2039
- .slice(0, 3)
2040
- .map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate));
2041
- const outEl = {
2042
- text: best.el.text ?? null,
2043
- resourceId: best.el.resourceId ?? null,
2044
- contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
2045
- class: best.el.type ?? best.el.class ?? null,
2046
- bounds: boundsObj,
2047
- clickable: !!best.el.clickable,
2048
- enabled: !!best.el.enabled,
2049
- stable_id: best.el.stable_id ?? null,
2050
- role: best.el.role ?? null,
2051
- test_tag: best.el.test_tag ?? null,
2052
- selector: best.el.selector ?? null,
2053
- semantic: best.el.semantic ?? null,
2054
- tapCoordinates,
2055
- telemetry: {
2056
- matchedIndex: best.idx ?? null,
2057
- matchedInteractable: !!best.interactable,
2058
- sliderLike: best.reason === 'slider_track_preferred'
2059
- }
2060
- };
2061
- if (best.reason === 'slider_track_preferred') {
2062
- const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left);
2063
- const interactionHint = {
2064
- kind: 'slider',
2065
- axis: isVertical ? 'vertical' : 'horizontal',
2066
- trackBounds: boundsObj
2067
- };
2068
- outEl.interactionHint = interactionHint;
2069
- }
2070
- const scoreVal = Math.min(1, Number(best.score.toFixed(3)));
2071
- const resolution = {
2072
- confidence: scoreVal,
2073
- reason: best.reason,
2074
- fallback_available: alternateCandidates.length > 0,
2075
- matched_count: uniqueRanked.length,
2076
- alternates: alternateCandidates
2077
- };
2078
- return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution };
2079
- }
2080
- static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false, resolveTapTarget = false }) {
2081
- const readinessTracker = currentReadinessWait();
2082
- const overallStart = Date.now();
2083
- const requestedIndex = typeof match?.index === 'number' ? match.index : null;
2084
- const requested = {
2085
- selector: selector ?? {},
2086
- condition,
2087
- match: requestedIndex === null ? null : { index: requestedIndex }
2088
- };
2089
- // Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
2090
- const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
2091
- const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
2092
- const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
2093
- if (!hasText && !hasResId && !hasAccId) {
2094
- return {
2095
- status: 'timeout',
2096
- error: {
2097
- code: 'INVALID_SELECTOR',
2098
- message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
2099
- },
2100
- metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
2101
- requested,
2102
- observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
2103
- };
2104
- }
2105
- // Validate condition
2106
- if (!['exists', 'not_exists', 'visible', 'clickable'].includes(condition)) {
2107
- return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } };
2108
- }
2109
- // Platform check
2110
- if (platform && !['android', 'ios'].includes(platform)) {
2111
- return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } };
2112
- }
2113
- const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000));
2114
- const maxAttempts = (retry && retry.max_attempts) ? Math.max(1, retry.max_attempts) : 1;
2115
- const backoff = (retry && retry.backoff_ms) ? Math.max(0, retry.backoff_ms) : 0;
2116
- let attempts = 0;
2117
- let totalPollCount = 0;
2118
- let lastMatchedCount = 0;
2119
- let lastMatchedElement = null;
2120
- let lastConditionSatisfied = false;
2121
- let matchedAt = null;
2122
- let stableMatchCount = 0;
2123
- const stableObservationCount = singleObservation ? 1 : 2;
2124
- const snapshotStaleThresholdMs = 500;
2125
- // Precompute normalized selector values and helpers (constant across polls)
2126
- const normalize = ToolsInteract._normalize;
2127
- const containsFlag = !!selector?.contains;
2128
- const selText = normalize(selector?.text);
2129
- const selRid = normalize(selector?.resource_id);
2130
- const selAid = normalize(selector?.accessibility_id);
2131
- try {
2132
- while (attempts < maxAttempts) {
2133
- attempts++;
2134
- const attemptStart = Date.now();
2135
- const deadline = attemptStart + (timeout_ms || 0);
2136
- while (Date.now() <= deadline || (singleAttempt && totalPollCount === 0)) {
2137
- remainingBudget();
2138
- totalPollCount++;
2139
- try {
2140
- const acquisitionStarted = Date.now();
2141
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
2142
- observeReadinessWait(tree, acquisitionStarted);
2143
- // A failed acquisition cannot establish that the requested tap target is absent.
2144
- if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error)))
2145
- throw new Error('ACTION_TIMEOUT');
2146
- const elements = (tree && Array.isArray(tree.elements)) ? tree.elements : [];
2147
- const matches = [];
2148
- for (let i = 0; i < elements.length; i++) {
2149
- const el = elements[i];
2150
- let ok = true;
2151
- // text
2152
- if (selector.text !== undefined && selector.text !== null) {
2153
- const val = normalize(el.text || el.label || el.value || '');
2154
- if (containsFlag) {
2155
- if (!val.includes(selText))
2156
- ok = false;
2157
- }
2158
- else {
2159
- if (val !== selText)
2160
- ok = false;
2161
- }
2162
- }
2163
- // resource_id
2164
- if (ok && selector.resource_id !== undefined && selector.resource_id !== null) {
2165
- const rid = normalize(el.resourceId || el.resourceID || el.id || '');
2166
- if (containsFlag) {
2167
- if (!rid.includes(selRid))
2168
- ok = false;
2169
- }
2170
- else {
2171
- if (rid !== selRid)
2172
- ok = false;
2173
- }
2174
- }
2175
- // accessibility_id
2176
- if (ok && selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
2177
- const aid = normalize(el.contentDescription || el.contentDesc || el.accessibilityLabel || el.label || '');
2178
- if (containsFlag) {
2179
- if (!aid.includes(selAid))
2180
- ok = false;
2181
- }
2182
- else {
2183
- if (aid !== selAid)
2184
- ok = false;
2185
- }
2186
- }
2187
- if (ok)
2188
- matches.push({ el, idx: i });
2189
- }
2190
- // Evaluate condition
2191
- const matchedCount = matches.length;
2192
- if (rejectAmbiguous && matchedCount > 1 && match?.index === undefined)
2193
- return { status: 'timeout', error: { code: 'AMBIGUOUS_TARGET' } };
2194
- const pickIndex = (typeof match?.index === 'number') ? match.index : undefined;
2195
- let chosen = null;
2196
- if (matches.length > 0) {
2197
- if (pickIndex !== undefined) {
2198
- // If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
2199
- if (pickIndex >= 0 && pickIndex < matches.length)
2200
- chosen = matches[pickIndex];
2201
- else
2202
- chosen = null;
2203
- }
2204
- else {
2205
- chosen = matches[0];
2206
- }
2207
- }
2208
- else {
2209
- chosen = null;
2210
- }
2211
- let conditionMet = false;
2212
- let matchedElement = chosen;
2213
- if (condition === 'exists') {
2214
- // when an index is specified, existence requires that specific index be present
2215
- conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1);
2216
- }
2217
- else if (condition === 'not_exists') {
2218
- // when an index is specified, not_exists is true if that index is absent
2219
- conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0);
2220
- }
2221
- else if (condition === 'visible') {
2222
- if (chosen) {
2223
- const b = chosen.el.bounds;
2224
- const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
2225
- conditionMet = visibleFlag;
2226
- }
2227
- else
2228
- conditionMet = false;
2229
- }
2230
- else if (condition === 'clickable') {
2231
- matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
2232
- ? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen) || chosen) : null;
2233
- if (matchedElement) {
2234
- const b = matchedElement.el.bounds;
2235
- const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
2236
- const enabled = !!matchedElement.el.enabled;
2237
- const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable;
2238
- conditionMet = visibleFlag && enabled && clickable;
2239
- }
2240
- else
2241
- conditionMet = false;
2242
- }
2243
- // Waiting for a label to exist/be visible is distinct from resolving its
2244
- // executable control. Reuse the clickable-wait ancestor policy without
2245
- // promoting hidden or disabled labels into an unrelated enabled target.
2246
- if (resolveTapTarget && conditionMet && matchedElement &&
2247
- ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
2248
- matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement;
2249
- }
2250
- const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
2251
- const resolvedDeviceId = tree?.device?.id || deviceId;
2252
- lastMatchedCount = matchedCount;
2253
- lastConditionSatisfied = conditionMet;
2254
- lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null;
2255
- const now = Date.now();
2256
- const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null;
2257
- const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs;
2258
- if (conditionMet && snapshotFresh) {
2259
- if (matchedAt === null)
2260
- matchedAt = now;
2261
- stableMatchCount++;
2262
- if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2263
- const latency_ms = now - overallStart;
2264
- const outEl = lastMatchedElement;
2265
- return {
2266
- status: 'success',
2267
- ...(singleObservation ? { _tree: tree } : {}),
2268
- matched: matchedCount,
2269
- element: outEl,
2270
- metrics: { latency_ms, poll_count: totalPollCount, attempts },
2271
- requested,
2272
- observed: {
2273
- matched_count: matchedCount,
2274
- condition_satisfied: true,
2275
- selected_index: outEl?.index ?? null,
2276
- last_matched_element: outEl
2277
- }
2278
- };
2279
- }
2280
- }
2281
- else {
2282
- stableMatchCount = 0;
2283
- matchedAt = null;
2284
- }
2285
- }
2286
- catch (e) {
2287
- checkDebuggerGuard();
2288
- // Deadline expiry does not invalidate the last successfully observed state.
2289
- if (e instanceof Error && e.message === 'ACTION_TIMEOUT')
2290
- throw e;
2291
- if (readinessTracker && !String(e).includes('READINESS_'))
2292
- readinessTracker.unavailable();
2293
- if (readinessTracker && isDeviceUnavailable(e))
2294
- throw e;
2295
- // Non-fatal per-poll error; record and continue
2296
- if (!readinessTracker)
2297
- console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e));
2298
- }
2299
- if (singleAttempt)
2300
- break;
2301
- // Sleep until next poll
2302
- await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)));
2303
- }
2304
- // Attempt timed out; if more attempts allowed, backoff then retry
2305
- if (attempts < maxAttempts) {
2306
- if (backoff > 0)
2307
- await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)));
2308
- continue;
2309
- }
2310
- // Final failure for this call
2311
- const elapsed = Date.now() - overallStart;
2312
- const observed = {
2313
- matched_count: lastMatchedCount,
2314
- condition_satisfied: lastConditionSatisfied,
2315
- selected_index: lastMatchedElement?.index ?? null,
2316
- last_matched_element: lastMatchedElement
2317
- };
2318
- const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
2319
- ? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
2320
- : ` observed ${lastMatchedCount} match(es)`;
2321
- return {
2322
- status: 'timeout',
2323
- error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
2324
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2325
- requested,
2326
- observed
2327
- };
2328
- }
2329
- }
2330
- catch (err) {
2331
- checkDebuggerGuard();
2332
- if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT')
2333
- throw err;
2334
- if (readinessTracker && isDeviceUnavailable(err))
2335
- throw err;
2336
- const elapsed = Date.now() - overallStart;
2337
- return {
2338
- status: 'timeout',
2339
- error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
2340
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2341
- requested,
2342
- observed: {
2343
- matched_count: lastMatchedCount,
2344
- condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
2345
- selected_index: lastMatchedElement?.index ?? null,
2346
- last_matched_element: lastMatchedElement
2347
- }
2348
- };
2349
- }
180
+ static async findElementHandler(input) {
181
+ return findElement(input, target => ToolsObserve.getUITreeHandler(target));
2350
182
  }
2351
- // Helper: normalize various log objects into plain message strings for comparison
2352
- static _logsToMessages(logsArr) {
2353
- if (!Array.isArray(logsArr))
2354
- return [];
2355
- return logsArr.map((l) => {
2356
- if (typeof l === 'string')
2357
- return l;
2358
- if (l && (l.message || l.msg))
2359
- return l.message || l.msg;
2360
- try {
2361
- return JSON.stringify(l);
2362
- }
2363
- catch {
2364
- return String(l);
2365
- }
2366
- });
183
+ static async waitForUIHandler(input) {
184
+ return waitForUI(input, { observe: target => ToolsObserve.getUITreeHandler(target),
185
+ resolve: (platform, deviceId, element, index) => ToolsInteract._buildResolvedElement(platform, deviceId, element, index) });
2367
186
  }
2368
- static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }) {
2369
- const start = Date.now();
2370
- let lastFingerprint = null;
2371
- let lastActivity = null;
2372
- while (Date.now() - start < timeoutMs) {
2373
- try {
2374
- const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
2375
- const fp = res?.fingerprint ?? null;
2376
- lastActivity = res?.activity ?? lastActivity;
2377
- if (fp === null || fp === undefined) {
2378
- lastFingerprint = null;
2379
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
2380
- continue;
2381
- }
2382
- lastFingerprint = fp;
2383
- if (fp !== previousFingerprint) {
2384
- // Stability confirmation
2385
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
2386
- try {
2387
- const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
2388
- const confirmFp = confirmRes?.fingerprint ?? null;
2389
- lastActivity = confirmRes?.activity ?? lastActivity;
2390
- if (confirmFp === fp) {
2391
- return {
2392
- success: true,
2393
- previousFingerprint,
2394
- newFingerprint: fp,
2395
- elapsedMs: Date.now() - start,
2396
- observed_screen: {
2397
- fingerprint: fp,
2398
- activity: lastActivity
2399
- }
2400
- };
2401
- }
2402
- lastFingerprint = confirmFp;
2403
- continue;
2404
- }
2405
- catch (e) {
2406
- checkDebuggerGuard();
2407
- console.error('Error confirming fingerprint:', e);
2408
- continue;
2409
- }
2410
- }
2411
- }
2412
- catch (e) {
2413
- checkDebuggerGuard();
2414
- console.error('Error getting screen fingerprint:', e);
2415
- }
2416
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
2417
- }
2418
- return {
2419
- success: false,
2420
- reason: 'timeout',
2421
- previousFingerprint,
2422
- lastFingerprint,
2423
- elapsedMs: Date.now() - start,
2424
- observed_screen: {
2425
- fingerprint: lastFingerprint,
2426
- activity: lastActivity
2427
- }
2428
- };
187
+ static async waitForScreenChangeHandler(input) {
188
+ return waitForScreenChange(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target) });
2429
189
  }
2430
- static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 300, expected_change, scope = 'screen', target = null }) {
2431
- const readinessTracker = currentReadinessWait();
2432
- const start = Date.now();
2433
- const pollIntervalMs = 300;
2434
- const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300);
2435
- let baseline = null;
2436
- let baselineScope = null;
2437
- let lastObservedRevision = null;
2438
- let lastLoadingState = null;
2439
- let lastSnapshotFreshnessMs = null;
2440
- let candidateSignatures = null;
2441
- let candidateObservedChange = null;
2442
- let candidateSinceMs = null;
2443
- let lastChangeSummary = null;
2444
- let lastScopeResolution = {
2445
- scope: scope === 'subtree' ? 'subtree' : 'screen',
2446
- target: target && typeof target === 'string' ? target : null,
2447
- resolved: scope !== 'subtree',
2448
- resolvedIndex: null,
2449
- resolvedStableId: null,
2450
- reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
2451
- };
2452
- while (Date.now() - start < timeout_ms) {
2453
- try {
2454
- if (readinessTracker)
2455
- remainingBudget();
2456
- const acquisitionStarted = Date.now();
2457
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
2458
- observeReadinessWait(tree, acquisitionStarted);
2459
- const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target);
2460
- if (scopedTree.error) {
2461
- lastScopeResolution = scopedTree.resolution;
2462
- return {
2463
- success: false,
2464
- observed_change: null,
2465
- snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
2466
- snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
2467
- timeout: true,
2468
- elapsed_ms: Date.now() - start,
2469
- expected_change,
2470
- loading_state: tree?.loading_state ?? lastLoadingState ?? null,
2471
- scope: scopedTree.resolution.scope,
2472
- target: scopedTree.resolution.target,
2473
- stability_state: 'transient',
2474
- change_summary: lastChangeSummary,
2475
- reason: scopedTree.error.message,
2476
- error: scopedTree.error
2477
- };
2478
- }
2479
- const scopedElements = scopedTree.elements;
2480
- const scopedSignatureTree = {
2481
- ...tree,
2482
- elements: scopedElements
2483
- };
2484
- const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree);
2485
- lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
2486
- lastLoadingState = tree?.loading_state ?? lastLoadingState;
2487
- lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs;
2488
- lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary;
2489
- lastScopeResolution = scopedTree.resolution;
2490
- baselineScope = baselineScope ?? scopedTree;
2491
- if (!baseline) {
2492
- baseline = signatures;
2493
- baselineScope = scopedTree;
2494
- }
2495
- else {
2496
- const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
2497
- if (observedChange) {
2498
- if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
2499
- candidateSignatures = signatures;
2500
- candidateObservedChange = observedChange;
2501
- candidateSinceMs = Date.now();
2502
- }
2503
- const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs;
2504
- if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2505
- return {
2506
- success: true,
2507
- observed_change: candidateObservedChange ?? observedChange,
2508
- snapshot_revision: lastObservedRevision ?? undefined,
2509
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2510
- timeout: false,
2511
- elapsed_ms: Date.now() - start,
2512
- expected_change,
2513
- loading_state: lastLoadingState ?? null,
2514
- scope: lastScopeResolution.scope,
2515
- target: lastScopeResolution.target,
2516
- stability_state: 'stable',
2517
- change_summary: lastChangeSummary,
2518
- reason: 'UI change observed'
2519
- };
2520
- }
2521
- }
2522
- else {
2523
- candidateSignatures = null;
2524
- candidateObservedChange = null;
2525
- candidateSinceMs = null;
2526
- }
2527
- }
2528
- }
2529
- catch (error) {
2530
- checkDebuggerGuard();
2531
- if (readinessTracker && !String(error).includes('READINESS_'))
2532
- readinessTracker.unavailable();
2533
- if (readinessTracker && isDeviceUnavailable(error))
2534
- throw error;
2535
- if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) {
2536
- readinessTracker.deadline();
2537
- break;
2538
- }
2539
- // Default waits retain best-effort polling.
2540
- }
2541
- await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)));
2542
- }
2543
- return {
2544
- success: false,
2545
- observed_change: null,
2546
- snapshot_revision: lastObservedRevision ?? undefined,
2547
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2548
- timeout: true,
2549
- elapsed_ms: Date.now() - start,
2550
- expected_change,
2551
- loading_state: lastLoadingState ?? null,
2552
- scope: lastScopeResolution.scope,
2553
- target: lastScopeResolution.target,
2554
- stability_state: 'transient',
2555
- change_summary: lastChangeSummary,
2556
- reason: 'timeout'
2557
- };
190
+ static async waitForUIChangeHandler(input) {
191
+ return waitForUIChange(input, { observe: target => ToolsObserve.getUITreeHandler(target), scope: (tree, scope, target) => ToolsInteract._resolveUiChangeScope(tree, scope, target) });
2558
192
  }
2559
- static async expectScreenHandler({ platform, fingerprint, screen, deviceId }) {
2560
- const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
2561
- const observedScreen = {
2562
- fingerprint: observedFingerprint?.fingerprint ?? null,
2563
- screen: observedFingerprint?.activity ?? null
2564
- };
2565
- let observedScreenLabel = observedScreen.screen;
2566
- if (!fingerprint && screen && platform !== 'ios') {
2567
- try {
2568
- const current = await ToolsObserve.getCurrentScreenHandler({ deviceId });
2569
- observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel;
2570
- }
2571
- catch {
2572
- // Keep fingerprint-derived activity when current-screen lookup is unavailable.
2573
- }
2574
- }
2575
- const expectedScreen = {
2576
- fingerprint: fingerprint ?? null,
2577
- screen: screen ?? null
2578
- };
2579
- let success = false;
2580
- let basis = 'none';
2581
- let reason = 'No fingerprint or screen expectation provided';
2582
- if (fingerprint) {
2583
- basis = 'fingerprint';
2584
- success = observedScreen.fingerprint === fingerprint;
2585
- reason = success
2586
- ? `observed fingerprint matches expected fingerprint ${fingerprint}`
2587
- : `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`;
2588
- }
2589
- else if (screen) {
2590
- basis = 'screen';
2591
- const candidates = new Set();
2592
- if (observedScreen.screen)
2593
- candidates.add(observedScreen.screen);
2594
- if (observedScreenLabel)
2595
- candidates.add(observedScreenLabel);
2596
- success = candidates.has(screen);
2597
- reason = success
2598
- ? `observed screen matches expected screen ${screen}`
2599
- : `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`;
2600
- }
2601
- return {
2602
- success,
2603
- observed_screen: {
2604
- fingerprint: observedScreen.fingerprint,
2605
- screen: observedScreenLabel
2606
- },
2607
- expected_screen: expectedScreen,
2608
- confidence: success ? 1 : 0,
2609
- comparison: {
2610
- basis,
2611
- matched: success,
2612
- reason
2613
- },
2614
- trace: buildObservationTrace({
2615
- actionType: 'expect_screen',
2616
- stage: 'verify',
2617
- success,
2618
- attempts: 1,
2619
- metadata: {
2620
- expected_screen: expectedScreen,
2621
- observed_screen: {
2622
- fingerprint: observedScreen.fingerprint,
2623
- screen: observedScreenLabel
2624
- },
2625
- comparison: {
2626
- basis,
2627
- matched: success
2628
- }
2629
- }
2630
- })
2631
- };
193
+ static async expectScreenHandler(input) {
194
+ return expectScreen(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), current: target => ToolsObserve.getCurrentScreenHandler(target) });
2632
195
  }
2633
- static async expectElementVisibleHandler({ selector, element_id, timeout_ms = 5000, poll_interval_ms = 300, platform, deviceId }) {
2634
- const result = await ToolsInteract.waitForUIHandler({
2635
- selector,
2636
- condition: 'visible',
2637
- timeout_ms,
2638
- poll_interval_ms,
2639
- platform,
2640
- deviceId
2641
- });
2642
- if (result?.status === 'success' && result?.element) {
2643
- return {
2644
- success: true,
2645
- selector,
2646
- element_id: result.element.elementId ?? element_id ?? null,
2647
- expected_condition: 'visible',
2648
- element: {
2649
- elementId: result.element.elementId ?? null,
2650
- text: result.element.text ?? null,
2651
- resource_id: result.element.resource_id ?? null,
2652
- accessibility_id: result.element.accessibility_id ?? null,
2653
- class: result.element.class ?? null,
2654
- bounds: result.element.bounds ?? null,
2655
- index: typeof result.element.index === 'number' ? result.element.index : null,
2656
- state: result.element.state ?? null,
2657
- stable_id: result.element.stable_id ?? null,
2658
- role: result.element.role ?? null,
2659
- test_tag: result.element.test_tag ?? null,
2660
- selector: result.element.selector ?? null,
2661
- semantic: result.element.semantic ?? null
2662
- },
2663
- observed: {
2664
- status: result.status,
2665
- matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
2666
- condition_satisfied: true,
2667
- selected_index: typeof result.element.index === 'number' ? result.element.index : null,
2668
- last_matched_element: {
2669
- elementId: result.element.elementId ?? null,
2670
- text: result.element.text ?? null,
2671
- resource_id: result.element.resource_id ?? null,
2672
- accessibility_id: result.element.accessibility_id ?? null,
2673
- class: result.element.class ?? null,
2674
- bounds: result.element.bounds ?? null,
2675
- index: typeof result.element.index === 'number' ? result.element.index : null,
2676
- state: result.element.state ?? null,
2677
- stable_id: result.element.stable_id ?? null,
2678
- role: result.element.role ?? null,
2679
- test_tag: result.element.test_tag ?? null,
2680
- selector: result.element.selector ?? null,
2681
- semantic: result.element.semantic ?? null
2682
- }
2683
- },
2684
- reason: 'selector is visible',
2685
- trace: buildObservationTrace({
2686
- actionType: 'expect_element_visible',
2687
- stage: 'verify',
2688
- success: true,
2689
- attempts: 1,
2690
- metadata: {
2691
- selector,
2692
- element_id: result.element.elementId ?? element_id ?? null,
2693
- status: result.status
2694
- }
2695
- })
2696
- };
2697
- }
2698
- const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT';
2699
- return {
2700
- success: false,
2701
- selector,
2702
- element_id: element_id ?? null,
2703
- expected_condition: 'visible',
2704
- observed: {
2705
- status: result?.status,
2706
- matched_count: result?.observed?.matched_count,
2707
- condition_satisfied: result?.observed?.condition_satisfied ?? false,
2708
- selected_index: result?.observed?.selected_index ?? null,
2709
- last_matched_element: result?.observed?.last_matched_element ?? null
2710
- },
2711
- reason: result?.error?.message ?? 'selector is not visible',
2712
- failure_code: errorCode,
2713
- retryable: errorCode === 'TIMEOUT',
2714
- trace: buildObservationTrace({
2715
- actionType: 'expect_element_visible',
2716
- stage: 'verify',
2717
- success: false,
2718
- attempts: 1,
2719
- metadata: {
2720
- selector,
2721
- element_id: element_id ?? null,
2722
- status: result?.status ?? null,
2723
- reason: result?.error?.message ?? 'selector is not visible'
2724
- }
2725
- })
2726
- };
196
+ static async expectElementVisibleHandler(input) {
197
+ return expectVisible(input, request => ToolsInteract.waitForUIHandler(request));
2727
198
  }
2728
- static async expectStateHandler({ selector, element_id, property, expected, platform, deviceId, stabilization_window_ms = 1000, stable_observation_count = 2, snapshot_stale_threshold_ms = 500, poll_interval_ms = 150, exact_budget = false }) {
2729
- const compareBoolean = (value) => typeof value === 'boolean' ? value : null;
2730
- const compareString = (value) => typeof value === 'string' ? value : null;
2731
- const compareNumber = (value) => typeof value === 'number' && Number.isFinite(value) ? value : null;
2732
- const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
2733
- const start = Date.now();
2734
- const deadline = start + (exact_budget ? stabilization_window_ms : Math.max(500, stabilization_window_ms));
2735
- const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2));
2736
- const pollDelay = exact_budget ? Math.max(50, Math.min(poll_interval_ms || 150, 1000)) : Math.max(100, Math.min(poll_interval_ms || 150, 200));
2737
- const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800));
2738
- let attempts = 0;
2739
- let stableCount = 0;
2740
- let lastReason = 'element not found';
2741
- let lastFailureCode = 'ELEMENT_NOT_FOUND';
2742
- let lastObservedElement = null;
2743
- let lastObservedValue = null;
2744
- let lastRawValue = null;
2745
- let lastResolvedElementId = element_id ?? null;
2746
- const traceSteps = [];
2747
- let traceAttemptIndex = 0;
2748
- let resolveRecorded = false;
2749
- const recordTraceStep = (stage, result, metadata) => {
2750
- traceSteps.push(createTraceStep({
2751
- stage,
2752
- timestamp: Date.now(),
2753
- result,
2754
- attemptIndex: traceAttemptIndex++,
2755
- metadata
2756
- }));
2757
- };
2758
- const buildStateTrace = (outcome) => ({
2759
- action_id: nextActionId('expect_state', Date.now()),
2760
- steps: traceSteps,
2761
- final_outcome: outcome,
2762
- attempts
2763
- });
2764
- while (Date.now() <= deadline) {
2765
- attempts++;
2766
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
2767
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
2768
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
2769
- const treeDeviceId = tree?.device?.id || deviceId;
2770
- const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null;
2771
- let matched = null;
2772
- if (element_id) {
2773
- const resolved = ToolsInteract._resolvedUiElements.get(element_id);
2774
- if (resolved) {
2775
- const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
2776
- if (current)
2777
- matched = { el: current.el, idx: current.index };
2778
- }
2779
- }
2780
- if (!matched && selector) {
2781
- matched = ToolsInteract._findFirstMatchingElement(elements, selector);
2782
- }
2783
- if (!matched) {
2784
- lastReason = 'element not found';
2785
- lastFailureCode = 'ELEMENT_NOT_FOUND';
2786
- stableCount = 0;
2787
- recordTraceStep('resolve', 'retry', {
2788
- selector: selector ?? null,
2789
- element_id: lastResolvedElementId,
2790
- matched: false,
2791
- reason: lastReason,
2792
- attempt: attempts
2793
- });
2794
- recordTraceStep('stabilize', 'retry', {
2795
- stabilization_attempts: attempts,
2796
- stable_observation_count: stableCount,
2797
- snapshot_freshness_ms: treeAgeMs,
2798
- reason: lastReason
2799
- });
2800
- await sleep(pollDelay);
2801
- continue;
2802
- }
2803
- const resolvedElement = ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx), matched.el, matched.idx);
2804
- lastResolvedElementId = resolvedElement.elementId;
2805
- lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null };
2806
- if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
2807
- lastReason = 'stale snapshot';
2808
- lastFailureCode = 'UNKNOWN';
2809
- stableCount = 0;
2810
- recordTraceStep('resolve', 'retry', {
2811
- selector: selector ?? null,
2812
- element_id: lastResolvedElementId,
2813
- matched: true,
2814
- reason: lastReason,
2815
- attempt: attempts
2816
- });
2817
- recordTraceStep('stabilize', 'retry', {
2818
- stabilization_attempts: attempts,
2819
- stable_observation_count: stableCount,
2820
- snapshot_freshness_ms: treeAgeMs,
2821
- reason: lastReason
2822
- });
2823
- await sleep(pollDelay);
2824
- continue;
2825
- }
2826
- if (!resolveRecorded) {
2827
- recordTraceStep('resolve', 'success', {
2828
- selector: selector ?? null,
2829
- element_id: lastResolvedElementId,
2830
- matched: true,
2831
- reason: 'element resolved'
2832
- });
2833
- resolveRecorded = true;
2834
- }
2835
- const observedState = matched.el.state ?? null;
2836
- const actual = observedState?.[property] ?? null;
2837
- let success = false;
2838
- let reason = '';
2839
- let rawValue = null;
2840
- let observedValue = actual;
2841
- switch (property) {
2842
- case 'checked':
2843
- case 'focused':
2844
- case 'expanded':
2845
- case 'enabled': {
2846
- const expectedBool = compareBoolean(expected);
2847
- const actualBool = compareBoolean(actual);
2848
- if (expectedBool === null) {
2849
- reason = `expected ${property} must be boolean`;
2850
- }
2851
- else if (actualBool === null) {
2852
- reason = `${property} state unavailable`;
2853
- }
2854
- else {
2855
- rawValue = actualBool;
2856
- success = actualBool === expectedBool;
2857
- reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`;
2858
- }
2859
- observedValue = actualBool;
2860
- break;
2861
- }
2862
- case 'value':
2863
- case 'raw_value': {
2864
- const expectedNumber = compareNumber(expected);
2865
- const actualNumber = compareNumber(actual);
2866
- if (expectedNumber !== null && actualNumber !== null) {
2867
- success = actualNumber === expectedNumber;
2868
- rawValue = actualNumber;
2869
- observedValue = actualNumber;
2870
- reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`;
2871
- break;
2872
- }
2873
- const expectedString = typeof expected === 'string' ? expected : null;
2874
- const actualString = compareString(actual);
2875
- if (expectedString !== null && actualString !== null) {
2876
- success = actualString === expectedString;
2877
- rawValue = actualString;
2878
- observedValue = actualString;
2879
- reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`;
2880
- }
2881
- else {
2882
- reason = 'value state unavailable';
2883
- }
2884
- break;
2885
- }
2886
- case 'selected': {
2887
- const expectedBool = typeof expected === 'boolean' ? expected : null;
2888
- const expectedString = typeof expected === 'string'
2889
- ? expected
2890
- : expected && typeof expected === 'object'
2891
- ? String(expected.id ?? expected.label ?? '')
2892
- : null;
2893
- if (!observedState || observedState.selected === undefined || observedState.selected === null) {
2894
- reason = 'selected state unavailable';
2895
- break;
2896
- }
2897
- if (expectedBool !== null) {
2898
- const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null;
2899
- if (actualBool === null) {
2900
- reason = 'selected state is not boolean';
2901
- break;
2902
- }
2903
- rawValue = actualBool;
2904
- observedValue = actualBool;
2905
- success = actualBool === expectedBool;
2906
- reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`;
2907
- break;
2908
- }
2909
- const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
2910
- ? String(observedState.selected.id ?? observedState.selected.label ?? '')
2911
- : String(observedState.selected);
2912
- const actualString = actualSelected.trim();
2913
- if (!expectedString) {
2914
- reason = 'expected selected must be boolean, string, or object with id/label';
2915
- break;
2916
- }
2917
- rawValue = actualString;
2918
- observedValue = actualString;
2919
- success = actualString === expectedString;
2920
- reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`;
2921
- break;
2922
- }
2923
- case 'text_value': {
2924
- const expectedString = typeof expected === 'string' ? expected : null;
2925
- const actualString = compareString(actual);
2926
- if (!expectedString) {
2927
- reason = 'expected text_value must be string';
2928
- }
2929
- else if (!actualString) {
2930
- reason = 'text_value state unavailable';
2931
- }
2932
- else {
2933
- success = actualString === expectedString;
2934
- rawValue = actualString;
2935
- observedValue = actualString;
2936
- reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`;
2937
- }
2938
- break;
2939
- }
2940
- default: {
2941
- if (actual !== null && actual !== undefined) {
2942
- success = actual === expected;
2943
- observedValue = actual;
2944
- rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null;
2945
- reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`;
2946
- }
2947
- else {
2948
- reason = `unsupported or unavailable state property: ${property}`;
2949
- }
2950
- }
2951
- }
2952
- if (success) {
2953
- stableCount++;
2954
- recordTraceStep('stabilize', 'success', {
2955
- stabilization_attempts: attempts,
2956
- stable_observation_count: stableCount,
2957
- snapshot_freshness_ms: treeAgeMs,
2958
- reason
2959
- });
2960
- recordTraceStep('verify', 'success', {
2961
- property,
2962
- expected,
2963
- observed_value: observedValue,
2964
- reason
2965
- });
2966
- if (stableCount >= stableTarget) {
2967
- return {
2968
- success: true,
2969
- selector,
2970
- element_id: lastResolvedElementId,
2971
- expected_state: { property, expected },
2972
- element: lastObservedElement,
2973
- observed_state: {
2974
- property,
2975
- value: observedValue,
2976
- ...(rawValue !== null ? { raw_value: rawValue } : {})
2977
- },
2978
- reason,
2979
- stabilization_attempts: attempts,
2980
- stabilization_window_ms: Date.now() - start,
2981
- stable_observation_count: stableCount,
2982
- snapshot_freshness_ms: treeAgeMs ?? undefined,
2983
- trace: buildStateTrace('success')
2984
- };
2985
- }
2986
- }
2987
- else {
2988
- stableCount = 0;
2989
- lastReason = reason || lastReason;
2990
- lastFailureCode = 'UNKNOWN';
2991
- recordTraceStep('stabilize', 'retry', {
2992
- stabilization_attempts: attempts,
2993
- stable_observation_count: stableCount,
2994
- snapshot_freshness_ms: treeAgeMs,
2995
- reason: lastReason
2996
- });
2997
- recordTraceStep('verify', 'retry', {
2998
- property,
2999
- expected,
3000
- observed_value: observedValue,
3001
- reason: lastReason
3002
- });
3003
- }
3004
- if (!success) {
3005
- lastObservedValue = observedValue;
3006
- lastRawValue = rawValue;
3007
- }
3008
- await sleep(pollDelay);
3009
- }
3010
- return {
3011
- success: false,
3012
- selector,
3013
- element_id: lastResolvedElementId,
3014
- expected_state: { property, expected },
3015
- element: lastObservedElement,
3016
- observed_state: {
3017
- property,
3018
- value: lastObservedValue,
3019
- ...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
3020
- },
3021
- reason: lastReason,
3022
- failure_code: lastFailureCode,
3023
- retryable: true,
3024
- trace: buildStateTrace('failure')
3025
- };
199
+ static async expectStateHandler(input) {
200
+ return expectState(input, { observe: target => ToolsObserve.getUITreeHandler(target), getHandle: id => ToolsInteract._resolvedUiElements.get(id),
201
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
202
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx) });
3026
203
  }
3027
- static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
3028
- const start = Date.now();
3029
- const deadline = start + (timeoutMs || 0);
3030
- const q = (query === null || query === undefined) ? '' : String(query);
3031
- // Clamp polling interval to 250-500ms for consistent behavior
3032
- const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500));
3033
- // Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
3034
- let initialFingerprint = null;
3035
- let baselineLastLine = null;
3036
- try {
3037
- const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
3038
- const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
3039
- const withTimeout = (p, ms) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))]);
3040
- const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)]);
3041
- if (fpRes && typeof fpRes === 'object')
3042
- initialFingerprint = fpRes.fingerprint ?? null;
3043
- if (gl) {
3044
- const logsArr = Array.isArray(gl.logs) ? gl.logs : [];
3045
- // Normalize to last message string for baseline comparison
3046
- const msgs = ToolsInteract._logsToMessages(logsArr);
3047
- baselineLastLine = msgs.length ? msgs[msgs.length - 1] : null;
3048
- }
3049
- }
3050
- catch (err) {
3051
- checkDebuggerGuard();
3052
- try {
3053
- console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err));
3054
- }
3055
- catch { }
3056
- }
3057
- // Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
3058
- let lastChangeAt = Date.now();
3059
- let prevFingerprint = initialFingerprint;
3060
- const sleep = (ms) => new Promise(r => setTimeout(r, ms));
3061
- // Optional initial observation delay requested by caller
3062
- if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
3063
- try {
3064
- console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`);
3065
- }
3066
- catch { }
3067
- await sleep(observationDelayMs);
3068
- }
3069
- // Telemetry
3070
- let pollCount = 0;
3071
- let matchedAt = null;
3072
- let lastObservedState = null;
3073
- let stableDuration = 0;
3074
- let matchSource = null;
3075
- while (Date.now() <= deadline) {
3076
- pollCount++;
3077
- const now = Date.now();
3078
- // Evaluate condition per type
3079
- if (type === 'ui') {
3080
- try {
3081
- // Prefer using the public findElementHandler which tests can override. This avoids relying
3082
- // on resolveObserve/getUITree for unit tests which may not have devices available.
3083
- try {
3084
- const findRes = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId });
3085
- const isPresent = !!(findRes && findRes.found);
3086
- const conditionTrue = (match === 'present') ? isPresent : !isPresent;
3087
- if (conditionTrue) {
3088
- if (matchedAt === null)
3089
- matchedAt = Date.now();
3090
- stableDuration = Date.now() - matchedAt;
3091
- lastObservedState = true;
3092
- if (stableDuration >= stability_ms) {
3093
- matchSource = 'ui-find';
3094
- const element = isPresent ? findRes.element : null;
3095
- const now2 = Date.now();
3096
- return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null };
3097
- }
3098
- }
3099
- else {
3100
- matchedAt = null;
3101
- stableDuration = 0;
3102
- lastObservedState = false;
3103
- }
3104
- }
3105
- catch (err) {
3106
- checkDebuggerGuard();
3107
- console.error('waitForUI(ui) find error:', err);
3108
- }
3109
- }
3110
- catch (err) {
3111
- checkDebuggerGuard();
3112
- console.error('waitForUI(ui) outer error:', err);
3113
- }
3114
- }
3115
- else if (type === 'log') {
3116
- try {
3117
- // Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
3118
- const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 });
3119
- const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : [];
3120
- for (const ent of entries) {
3121
- const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : '';
3122
- if (q && String(msg).includes(q)) {
3123
- const now2 = Date.now();
3124
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true };
3125
- }
3126
- }
3127
- const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
3128
- const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : [];
3129
- // Normalize to messages for comparison
3130
- const msgs = ToolsInteract._logsToMessages(logsArr);
3131
- let startIndex = 0;
3132
- if (baselineLastLine) {
3133
- const idx = msgs.lastIndexOf(baselineLastLine);
3134
- startIndex = idx >= 0 ? idx + 1 : 0;
3135
- }
3136
- for (let i = startIndex; i < msgs.length; i++) {
3137
- const line = msgs[i];
3138
- if (q && String(line).includes(q)) {
3139
- const now2 = Date.now();
3140
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true };
3141
- }
3142
- }
3143
- }
3144
- catch (err) {
3145
- checkDebuggerGuard();
3146
- console.error('waitForUI(log) error:', err);
3147
- }
3148
- }
3149
- else if (type === 'screen') {
3150
- try {
3151
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
3152
- const fp = fpRes?.fingerprint ?? null;
3153
- if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
3154
- // when screen changed, require stability_ms where fingerprint remains the same
3155
- if (matchedAt === null)
3156
- matchedAt = now;
3157
- const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }))?.fingerprint ?? null;
3158
- if (confirmFp === fp) {
3159
- stableDuration = Date.now() - matchedAt;
3160
- lastObservedState = true;
3161
- if (stableDuration >= stability_ms) {
3162
- const now2 = Date.now();
3163
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null };
3164
- }
3165
- }
3166
- else {
3167
- matchedAt = null;
3168
- stableDuration = 0;
3169
- lastObservedState = false;
3170
- }
3171
- }
3172
- }
3173
- catch (err) {
3174
- checkDebuggerGuard();
3175
- console.error('waitForUI(screen) error:', err);
3176
- }
3177
- }
3178
- else if (type === 'idle') {
3179
- try {
3180
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
3181
- const fp = fpRes?.fingerprint ?? null;
3182
- if (fp !== prevFingerprint) {
3183
- prevFingerprint = fp;
3184
- lastChangeAt = Date.now();
3185
- matchedAt = null;
3186
- stableDuration = 0;
3187
- lastObservedState = false;
3188
- }
3189
- else {
3190
- const idleMs = Date.now() - lastChangeAt;
3191
- lastObservedState = true;
3192
- if (idleMs >= stability_ms) {
3193
- const now2 = Date.now();
3194
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null };
3195
- }
3196
- }
3197
- }
3198
- catch (err) {
3199
- checkDebuggerGuard();
3200
- console.error('waitForUI(idle) error:', err);
3201
- }
3202
- }
3203
- // Respect poll interval and avoid tight loop
3204
- await sleep(pollInterval);
3205
- }
3206
- // On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
3207
- let snapshot = null;
3208
- if (includeSnapshotOnFailure) {
3209
- try {
3210
- // Use dynamic import to avoid circular-initialization issues where the ToolsObserve
3211
- // binding captured earlier may not reflect test-time overrides. Importing at call
3212
- // time ensures the latest exported ToolsObserve object is used.
3213
- const Obs = await import('../observe/index.js');
3214
- snapshot = await Obs.ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId });
3215
- }
3216
- catch (err) {
3217
- checkDebuggerGuard();
3218
- snapshot = { error: err instanceof Error ? err.message : String(err) };
3219
- }
3220
- }
3221
- const elapsed = Date.now() - start;
3222
- return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null };
204
+ static async waitForUICore(input) {
205
+ return waitForUICore(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), logs: target => ToolsObserve.getLogsHandler(target),
206
+ stream: target => ToolsObserve.readLogStreamHandler(target), find: input => ToolsInteract.findElementHandler(input),
207
+ snapshot: async (input) => { const { ToolsObserve } = await import('../observe/index.js'); return ToolsObserve.captureDebugSnapshotHandler(input); } });
3223
208
  }
3224
209
  }