mobile-debug-mcp 0.34.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (500) hide show
  1. package/.agents/skills/mcp-builder/SKILL.md +44 -64
  2. package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
  3. package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
  4. package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
  5. package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
  6. package/.agents/skills/test-authoring/SKILL.md +7 -6
  7. package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
  8. package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
  9. package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
  10. package/.cursor-plugin/marketplace.json +1 -1
  11. package/.github/copilot-instructions.md +14 -36
  12. package/.github/workflows/ci.yml +4 -4
  13. package/AGENTS.md +12 -3
  14. package/CONTRIBUTING.md +79 -0
  15. package/README.md +218 -129
  16. package/agent-plugin/.antigravity-plugin/README.md +48 -0
  17. package/agent-plugin/.antigravity-plugin/plugin.json +18 -0
  18. package/agent-plugin/.claude-plugin/plugin.json +2 -2
  19. package/agent-plugin/.codex-plugin/plugin.json +1 -1
  20. package/agent-plugin/.cursor-plugin/plugin.json +2 -2
  21. package/agent-plugin/.mcp.json +1 -1
  22. package/agent-plugin/README.md +50 -11
  23. package/agent-plugin/cursor/.cursor-plugin/plugin.json +23 -0
  24. package/agent-plugin/cursor/LICENSE +201 -0
  25. package/agent-plugin/cursor/README.md +9 -0
  26. package/agent-plugin/cursor/skills/mobile-debug-mcp/SKILL.md +253 -0
  27. package/agent-plugin/mcp.json +1 -1
  28. package/agent-plugin/mcp_config.json +13 -0
  29. package/agent-plugin/plugin.json +1 -1
  30. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +62 -17
  31. package/dist/contracts/execution-controls.js +24 -0
  32. package/dist/contracts/selector.js +8 -0
  33. package/dist/contracts/target.js +15 -0
  34. package/dist/contracts/ui.js +1 -0
  35. package/dist/debugger/android.js +2 -1
  36. package/dist/debugger/diagnostics.js +54 -0
  37. package/dist/debugger/guard.js +4 -4
  38. package/dist/debugger/ios-worker.js +62 -0
  39. package/dist/debugger/ios.js +57 -0
  40. package/dist/debugger/protocol.js +33 -0
  41. package/dist/debugger/sessions.js +133 -74
  42. package/dist/debugger/worker.js +59 -46
  43. package/dist/interact/adjustment/adjustment-context.js +41 -0
  44. package/dist/interact/adjustment/adjustment-gestures.js +55 -0
  45. package/dist/interact/adjustment/adjustment-plan.js +18 -0
  46. package/dist/interact/adjustment/adjustment-resolution.js +127 -0
  47. package/dist/interact/adjustment/adjustment-result.js +19 -0
  48. package/dist/interact/adjustment/adjustment-semantic.js +64 -0
  49. package/dist/interact/adjustment/adjustment-verification.js +35 -0
  50. package/dist/interact/adjustment/adjustment.js +52 -0
  51. package/dist/interact/adjustment/control-geometry.js +87 -0
  52. package/dist/interact/adjustment/control-values.js +136 -0
  53. package/dist/interact/adjustment/slider-labels.js +84 -0
  54. package/dist/interact/assertions/screen-assertion.js +37 -0
  55. package/dist/interact/assertions/state-assertion-context.js +35 -0
  56. package/dist/interact/assertions/state-assertion.js +76 -0
  57. package/dist/interact/assertions/state-comparison.js +141 -0
  58. package/dist/interact/assertions/visibility-assertion.js +34 -0
  59. package/dist/interact/conditional-action.js +12 -7
  60. package/dist/interact/index.js +77 -3092
  61. package/dist/interact/input-service.js +1 -0
  62. package/dist/interact/interaction-trace.js +21 -0
  63. package/dist/interact/model.js +1 -0
  64. package/dist/interact/observation.js +1 -0
  65. package/dist/interact/targeting/candidate-ancestor.js +58 -0
  66. package/dist/interact/targeting/candidate-result.js +23 -0
  67. package/dist/interact/targeting/candidate-scoring.js +50 -0
  68. package/dist/interact/targeting/element-helpers.js +216 -0
  69. package/dist/interact/targeting/find-element.js +65 -0
  70. package/dist/interact/targeting/nearby-control.js +56 -0
  71. package/dist/interact/targeting/resolved-elements.js +21 -0
  72. package/dist/interact/targeting/tap-target.js +17 -0
  73. package/dist/interact/targeting/target-scope.js +104 -0
  74. package/dist/interact/waits/legacy-log-wait.js +38 -0
  75. package/dist/interact/waits/legacy-wait-observations.js +103 -0
  76. package/dist/interact/waits/legacy-wait-state.js +34 -0
  77. package/dist/interact/waits/legacy-wait.js +48 -0
  78. package/dist/interact/waits/screen-wait.js +49 -0
  79. package/dist/interact/waits/ui-change-helpers.js +157 -0
  80. package/dist/interact/waits/ui-change-state.js +18 -0
  81. package/dist/interact/waits/ui-change-wait.js +77 -0
  82. package/dist/interact/waits/ui-wait-condition.js +56 -0
  83. package/dist/interact/waits/ui-wait-state.js +38 -0
  84. package/dist/interact/waits/ui-wait.js +106 -0
  85. package/dist/manage/android.js +55 -51
  86. package/dist/manage/index.js +38 -75
  87. package/dist/manage/ios/build-types.js +1 -0
  88. package/dist/manage/ios/build.js +210 -0
  89. package/dist/manage/ios/project-discovery.js +29 -0
  90. package/dist/manage/ios/scheme.js +18 -0
  91. package/dist/manage/ios.js +44 -256
  92. package/dist/observe/android/context.js +1 -0
  93. package/dist/observe/android/current-screen.js +55 -0
  94. package/dist/observe/android/fingerprint.js +13 -0
  95. package/dist/observe/android/log-stream.js +132 -0
  96. package/dist/observe/android/logs.js +107 -0
  97. package/dist/observe/android/screenshot.js +70 -0
  98. package/dist/observe/android/ui-tree.js +107 -0
  99. package/dist/observe/android.js +16 -520
  100. package/dist/observe/evidence.js +114 -75
  101. package/dist/observe/index.js +16 -263
  102. package/dist/observe/ios/context.js +16 -0
  103. package/dist/observe/ios/fingerprint.js +12 -0
  104. package/dist/observe/ios/log-stream.js +105 -0
  105. package/dist/observe/ios/logs.js +168 -0
  106. package/dist/observe/ios/screenshot.js +22 -0
  107. package/dist/observe/ios/ui-elements.js +241 -0
  108. package/dist/observe/ios/ui-tree.js +137 -0
  109. package/dist/observe/ios-trace/artifact.js +63 -0
  110. package/dist/observe/ios-trace/capabilities.js +86 -0
  111. package/dist/observe/ios-trace/native.js +93 -0
  112. package/dist/observe/ios-trace/notification.js +35 -0
  113. package/dist/observe/ios-trace/process.js +98 -0
  114. package/dist/observe/ios-trace/sessions.js +244 -0
  115. package/dist/observe/ios.js +17 -715
  116. package/dist/observe/logs/model.js +1 -0
  117. package/dist/{network → observe/network}/index.js +2 -2
  118. package/dist/observe/readiness-service.js +5 -3
  119. package/dist/observe/readiness.js +33 -14
  120. package/dist/observe/screenshot-encoding.js +41 -0
  121. package/dist/observe/snapshot-collection.js +79 -0
  122. package/dist/observe/snapshot-logs.js +30 -0
  123. package/dist/observe/snapshot-metadata.js +43 -31
  124. package/dist/observe/snapshot-semantic.js +115 -0
  125. package/dist/observe/system-trace-error.js +7 -0
  126. package/dist/observe/system-trace-router.js +81 -0
  127. package/dist/observe/system-trace.js +26 -21
  128. package/dist/runtime/action-results.js +270 -0
  129. package/dist/runtime/clock.js +4 -0
  130. package/dist/server/common.js +1 -256
  131. package/dist/server/definitions/assert.js +162 -0
  132. package/dist/server/definitions/diagnostic.js +118 -0
  133. package/dist/server/definitions/input.js +274 -0
  134. package/dist/server/definitions/manage.js +194 -0
  135. package/dist/server/definitions/observe.js +149 -0
  136. package/dist/server/definitions/target.js +138 -0
  137. package/dist/server/definitions/wait.js +105 -0
  138. package/dist/server/handlers/action-execution.js +163 -0
  139. package/dist/server/handlers/correlation.js +23 -0
  140. package/dist/server/handlers/debugger-handlers.js +67 -0
  141. package/dist/server/handlers/debugger-response.js +9 -0
  142. package/dist/server/handlers/diagnostic.js +77 -0
  143. package/dist/server/handlers/handler-types.js +1 -0
  144. package/dist/server/handlers/input.js +154 -0
  145. package/dist/server/handlers/interact.js +113 -0
  146. package/dist/server/handlers/journey-assertions.js +35 -0
  147. package/dist/server/handlers/journey.js +147 -0
  148. package/dist/server/handlers/manage.js +119 -0
  149. package/dist/server/handlers/observe.js +95 -0
  150. package/dist/server/handlers/payload.js +4 -0
  151. package/dist/server/handlers/tool-handlers.js +170 -0
  152. package/dist/server/schemas/condition-schema.js +2 -0
  153. package/dist/server/{debugger-schema.js → schemas/debugger-schema.js} +20 -12
  154. package/dist/server/{journey-schema.js → schemas/journey-schema.js} +7 -22
  155. package/dist/server/schemas/system-trace-schema.js +22 -0
  156. package/dist/server/tool-definitions.js +58 -1135
  157. package/dist/server-core.js +2 -2
  158. package/dist/server.js +4 -4
  159. package/dist/system/android-status.js +101 -0
  160. package/dist/system/android.js +37 -27
  161. package/dist/system/index.js +13 -439
  162. package/dist/system/ios-status.js +132 -0
  163. package/dist/system/ios.js +11 -8
  164. package/dist/system/model.js +1 -0
  165. package/dist/system/probes.js +105 -0
  166. package/dist/system/status-summary.js +43 -0
  167. package/dist/system/unsupported-status.js +52 -0
  168. package/dist/utils/android/commands.js +89 -0
  169. package/dist/utils/android/gradle.js +139 -0
  170. package/dist/utils/android/logs.js +103 -0
  171. package/dist/utils/android/ui-tree.js +233 -0
  172. package/dist/utils/android/utils.js +6 -564
  173. package/dist/utils/cli/idb/idb-helper.js +4 -6
  174. package/dist/utils/cli/idb/install-idb.js +13 -9
  175. package/dist/utils/cli/ios/preflight-ios.js +42 -38
  176. package/dist/utils/ios/utils.js +48 -46
  177. package/dist/utils/java.js +65 -49
  178. package/dist/utils/resolve-device.js +4 -2
  179. package/dist/utils/ui/index.js +26 -26
  180. package/docs/CHANGELOG.md +67 -1
  181. package/docs/README.md +38 -0
  182. package/docs/agent-plugin-installation-verification.md +39 -0
  183. package/docs/benchmarks/README.md +18 -0
  184. package/docs/benchmarks/interaction-latency/README.md +14 -0
  185. package/docs/benchmarks/interaction-latency/final.md +4 -1
  186. package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
  187. package/docs/benchmarks/ui-tree-performance/README.md +24 -16
  188. package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
  189. package/docs/code-structure.md +120 -0
  190. package/docs/specs/001-state-verification/spec.md +16 -14
  191. package/docs/specs/002-richer-element-identity/spec.md +12 -10
  192. package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
  193. package/docs/specs/004-action-verification-routing/spec.md +3 -3
  194. package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
  195. package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
  196. package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
  197. package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
  198. package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
  199. package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
  200. package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
  201. package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
  202. package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
  203. package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
  204. package/docs/specs/014-actionability-resolution/spec.md +20 -18
  205. package/docs/specs/015-adjustable-control-support/spec.md +17 -15
  206. package/docs/specs/016-environment-tools/spec.md +20 -20
  207. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
  208. package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
  209. package/docs/specs/020-conditional-actions/intent.md +5 -3
  210. package/docs/specs/020-conditional-actions/plan.md +8 -9
  211. package/docs/specs/020-conditional-actions/spec.md +4 -2
  212. package/docs/specs/021-app-readiness-signals/intent.md +4 -2
  213. package/docs/specs/021-app-readiness-signals/plan.md +7 -6
  214. package/docs/specs/021-app-readiness-signals/spec.md +3 -1
  215. package/docs/specs/022-correlated-evidence/intent.md +4 -4
  216. package/docs/specs/022-correlated-evidence/plan.md +10 -11
  217. package/docs/specs/022-correlated-evidence/spec.md +3 -1
  218. package/docs/specs/023-android-system-traces/intent.md +3 -2
  219. package/docs/specs/023-android-system-traces/plan.md +5 -7
  220. package/docs/specs/023-android-system-traces/spec.md +3 -1
  221. package/docs/specs/024-android-heap-dump-capture/intent.md +3 -1
  222. package/docs/specs/025-android-runtime-debugger/intent.md +4 -3
  223. package/docs/specs/025-android-runtime-debugger/plan.md +3 -1
  224. package/docs/specs/025-android-runtime-debugger/spec.md +4 -3
  225. package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
  226. package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
  227. package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
  228. package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
  229. package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
  230. package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
  231. package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
  232. package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
  233. package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
  234. package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
  235. package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
  236. package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
  237. package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
  238. package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
  239. package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
  240. package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
  241. package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
  242. package/docs/specs/030-android-coroutine-validation/report.md +240 -0
  243. package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
  244. package/docs/specs/031-ios-system-traces/intent.md +38 -0
  245. package/docs/specs/031-ios-system-traces/investigation.md +92 -0
  246. package/docs/specs/031-ios-system-traces/late-run-failure-evidence.json +32 -0
  247. package/docs/specs/031-ios-system-traces/plan.md +138 -0
  248. package/docs/specs/031-ios-system-traces/pr102-monitor-evidence.json +22 -0
  249. package/docs/specs/031-ios-system-traces/simulator-evidence.json +47 -0
  250. package/docs/specs/031-ios-system-traces/spec.md +135 -0
  251. package/docs/specs/032-ios-runtime-debugger/evidence.json +176 -0
  252. package/docs/specs/032-ios-runtime-debugger/intent.md +38 -0
  253. package/docs/specs/032-ios-runtime-debugger/plan.md +95 -0
  254. package/docs/specs/032-ios-runtime-debugger/report.md +205 -0
  255. package/docs/specs/032-ios-runtime-debugger/spec.md +153 -0
  256. package/docs/tools/TOOLS.md +31 -18
  257. package/docs/tools/{evidence.md → correlated-evidence.md} +9 -1
  258. package/docs/tools/debugger.md +120 -4
  259. package/docs/tools/interact.md +42 -20
  260. package/docs/tools/manage.md +25 -44
  261. package/docs/tools/observe.md +55 -23
  262. package/docs/tools/system-traces.md +89 -14
  263. package/docs/tools/system.md +12 -5
  264. package/package.json +12 -8
  265. package/scripts/agent-plugin.mjs +55 -5
  266. package/scripts/architecture-graph.mjs +69 -0
  267. package/scripts/check-architecture.mjs +47 -0
  268. package/scripts/code-quality-baseline.json +1 -0
  269. package/scripts/code-quality.mjs +161 -0
  270. package/src/contracts/execution-controls.ts +26 -0
  271. package/src/contracts/selector.ts +17 -0
  272. package/src/contracts/target.ts +19 -0
  273. package/src/contracts/ui.ts +94 -0
  274. package/src/debugger/android.ts +3 -2
  275. package/src/debugger/diagnostics.ts +43 -0
  276. package/src/debugger/guard.ts +10 -5
  277. package/src/debugger/ios-worker.ts +49 -0
  278. package/src/debugger/ios.ts +45 -0
  279. package/{worker → src/debugger/java}/DebuggerWorker.java +9 -3
  280. package/src/debugger/lldb/breakpoints.py +51 -0
  281. package/src/debugger/lldb/inspection.py +109 -0
  282. package/src/debugger/lldb/worker.py +203 -0
  283. package/src/debugger/protocol.ts +37 -0
  284. package/src/debugger/sessions.ts +326 -155
  285. package/src/debugger/worker.ts +57 -41
  286. package/src/interact/adjustment/adjustment-context.ts +83 -0
  287. package/src/interact/adjustment/adjustment-gestures.ts +48 -0
  288. package/src/interact/adjustment/adjustment-plan.ts +23 -0
  289. package/src/interact/adjustment/adjustment-resolution.ts +104 -0
  290. package/src/interact/adjustment/adjustment-result.ts +34 -0
  291. package/src/interact/adjustment/adjustment-semantic.ts +53 -0
  292. package/src/interact/adjustment/adjustment-verification.ts +60 -0
  293. package/src/interact/adjustment/adjustment.ts +49 -0
  294. package/src/interact/adjustment/control-geometry.ts +120 -0
  295. package/src/interact/adjustment/control-values.ts +141 -0
  296. package/src/interact/adjustment/slider-labels.ts +84 -0
  297. package/src/interact/assertions/screen-assertion.ts +39 -0
  298. package/src/interact/assertions/state-assertion-context.ts +48 -0
  299. package/src/interact/assertions/state-assertion.ts +64 -0
  300. package/src/interact/assertions/state-comparison.ts +168 -0
  301. package/src/interact/assertions/visibility-assertion.ts +41 -0
  302. package/src/interact/conditional-action.ts +18 -12
  303. package/src/interact/index.ts +89 -3494
  304. package/src/interact/input-service.ts +11 -0
  305. package/src/interact/interaction-trace.ts +36 -0
  306. package/src/interact/model.ts +109 -0
  307. package/src/interact/observation.ts +7 -0
  308. package/src/interact/targeting/candidate-ancestor.ts +49 -0
  309. package/src/interact/targeting/candidate-result.ts +24 -0
  310. package/src/interact/targeting/candidate-scoring.ts +42 -0
  311. package/src/interact/targeting/element-helpers.ts +243 -0
  312. package/src/interact/targeting/find-element.ts +47 -0
  313. package/src/interact/targeting/nearby-control.ts +46 -0
  314. package/src/interact/targeting/resolved-elements.ts +20 -0
  315. package/src/interact/targeting/tap-target.ts +22 -0
  316. package/src/interact/targeting/target-scope.ts +121 -0
  317. package/src/interact/waits/legacy-log-wait.ts +30 -0
  318. package/src/interact/waits/legacy-wait-observations.ts +55 -0
  319. package/src/interact/waits/legacy-wait-state.ts +48 -0
  320. package/src/interact/waits/legacy-wait.ts +34 -0
  321. package/src/interact/waits/screen-wait.ts +37 -0
  322. package/src/interact/waits/ui-change-helpers.ts +184 -0
  323. package/src/interact/waits/ui-change-state.ts +26 -0
  324. package/src/interact/waits/ui-change-wait.ts +59 -0
  325. package/src/interact/waits/ui-wait-condition.ts +52 -0
  326. package/src/interact/waits/ui-wait-state.ts +55 -0
  327. package/src/interact/waits/ui-wait.ts +84 -0
  328. package/src/manage/android.ts +56 -50
  329. package/src/manage/index.ts +63 -93
  330. package/src/manage/ios/build-types.ts +14 -0
  331. package/src/manage/ios/build.ts +230 -0
  332. package/src/manage/ios/project-discovery.ts +26 -0
  333. package/src/manage/ios/scheme.ts +16 -0
  334. package/src/manage/ios.ts +67 -298
  335. package/src/observe/android/context.ts +1 -0
  336. package/src/observe/android/current-screen.ts +63 -0
  337. package/src/observe/android/fingerprint.ts +13 -0
  338. package/src/observe/android/log-stream.ts +127 -0
  339. package/src/observe/android/logs.ts +110 -0
  340. package/src/observe/android/screenshot.ts +79 -0
  341. package/src/observe/android/ui-tree.ts +113 -0
  342. package/src/observe/android.ts +17 -531
  343. package/src/observe/evidence.ts +122 -70
  344. package/src/observe/index.ts +24 -286
  345. package/src/observe/ios/context.ts +22 -0
  346. package/src/observe/ios/fingerprint.ts +12 -0
  347. package/src/observe/ios/log-stream.ts +103 -0
  348. package/src/observe/ios/logs.ts +179 -0
  349. package/src/observe/ios/screenshot.ts +26 -0
  350. package/src/observe/ios/ui-elements.ts +266 -0
  351. package/src/observe/ios/ui-tree.ts +143 -0
  352. package/src/observe/ios-trace/artifact.ts +55 -0
  353. package/src/observe/ios-trace/capabilities.ts +74 -0
  354. package/src/observe/ios-trace/native.ts +72 -0
  355. package/src/observe/ios-trace/notification.ts +30 -0
  356. package/src/observe/ios-trace/process.ts +82 -0
  357. package/src/observe/ios-trace/sessions.ts +204 -0
  358. package/src/observe/ios.ts +18 -749
  359. package/src/observe/logs/model.ts +11 -0
  360. package/src/{network → observe/network}/index.ts +2 -2
  361. package/src/observe/readiness-service.ts +6 -4
  362. package/src/observe/readiness.ts +42 -21
  363. package/src/observe/screenshot-encoding.ts +32 -0
  364. package/src/observe/snapshot-collection.ts +69 -0
  365. package/src/observe/snapshot-logs.ts +26 -0
  366. package/src/observe/snapshot-metadata.ts +51 -31
  367. package/src/observe/snapshot-semantic.ts +153 -0
  368. package/src/observe/system-trace-error.ts +3 -0
  369. package/src/observe/system-trace-router.ts +61 -0
  370. package/src/observe/system-trace.ts +28 -19
  371. package/src/runtime/action-results.ts +376 -0
  372. package/src/runtime/clock.ts +9 -0
  373. package/src/server/common.ts +9 -359
  374. package/src/server/definitions/assert.ts +164 -0
  375. package/src/server/definitions/diagnostic.ts +120 -0
  376. package/src/server/definitions/input.ts +276 -0
  377. package/src/server/definitions/manage.ts +196 -0
  378. package/src/server/definitions/observe.ts +151 -0
  379. package/src/server/definitions/target.ts +140 -0
  380. package/src/server/definitions/wait.ts +107 -0
  381. package/src/server/handlers/action-execution.ts +164 -0
  382. package/src/server/handlers/correlation.ts +20 -0
  383. package/src/server/handlers/debugger-handlers.ts +58 -0
  384. package/src/server/handlers/debugger-response.ts +7 -0
  385. package/src/server/handlers/diagnostic.ts +86 -0
  386. package/src/server/handlers/handler-types.ts +7 -0
  387. package/src/server/handlers/input.ts +173 -0
  388. package/src/server/handlers/interact.ts +135 -0
  389. package/src/server/handlers/journey-assertions.ts +39 -0
  390. package/src/server/handlers/journey.ts +132 -0
  391. package/src/server/handlers/manage.ts +141 -0
  392. package/src/server/handlers/observe.ts +116 -0
  393. package/src/server/handlers/payload.ts +36 -0
  394. package/src/server/handlers/tool-handlers.ts +170 -0
  395. package/src/server/schemas/condition-schema.ts +2 -0
  396. package/src/server/schemas/debugger-schema.ts +46 -0
  397. package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +7 -22
  398. package/src/server/schemas/system-trace-schema.ts +23 -0
  399. package/src/server/tool-definitions.ts +67 -1143
  400. package/src/server-core.ts +2 -2
  401. package/src/server.ts +4 -4
  402. package/src/system/android-status.ts +124 -0
  403. package/src/system/android.ts +33 -21
  404. package/src/system/index.ts +21 -525
  405. package/src/system/ios-status.ts +147 -0
  406. package/src/system/ios.ts +14 -10
  407. package/src/system/model.ts +41 -0
  408. package/src/system/probes.ts +114 -0
  409. package/src/system/status-summary.ts +47 -0
  410. package/src/system/unsupported-status.ts +55 -0
  411. package/src/types.ts +16 -106
  412. package/src/utils/android/commands.ts +81 -0
  413. package/src/utils/android/gradle.ts +132 -0
  414. package/src/utils/android/logs.ts +103 -0
  415. package/src/utils/android/ui-tree.ts +239 -0
  416. package/src/utils/android/utils.ts +16 -556
  417. package/src/utils/cli/idb/check-idb.ts +4 -4
  418. package/src/utils/cli/idb/idb-helper.ts +6 -8
  419. package/src/utils/cli/idb/install-idb.ts +15 -9
  420. package/src/utils/cli/ios/preflight-ios.ts +39 -31
  421. package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
  422. package/src/utils/ios/utils.ts +60 -61
  423. package/src/utils/java.ts +62 -43
  424. package/src/utils/resolve-device.ts +9 -6
  425. package/src/utils/ui/index.ts +28 -28
  426. package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
  427. package/test/device/automated/debugger/coroutine-support.ts +140 -0
  428. package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
  429. package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
  430. package/test/device/automated/debugger/runtime.android.test.ts +19 -3
  431. package/test/device/automated/debugger/runtime.ios.test.ts +221 -0
  432. package/test/device/automated/interact/interaction.android.test.ts +56 -0
  433. package/test/device/automated/observe/ios-system-trace.smoke.ts +119 -0
  434. package/test/device/automated/observe/system-trace.smoke.ts +1 -1
  435. package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
  436. package/test/fixtures/contracts/tool-definitions.json +3696 -0
  437. package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
  438. package/test/fixtures/coroutine-app/README.md +48 -0
  439. package/test/fixtures/coroutine-app/build.ts +35 -0
  440. package/test/fixtures/debugger-app/MainActivity.java +21 -0
  441. package/test/fixtures/debugger-app/MainActivity.kt +32 -0
  442. package/test/fixtures/debugger-app/README.md +52 -0
  443. package/test/fixtures/debugger-app/build.ts +25 -15
  444. package/test/fixtures/ios-debugger-app/Main.swift +95 -0
  445. package/test/fixtures/ios-debugger-app/README.md +27 -0
  446. package/test/fixtures/ios-debugger-app/build.ts +21 -0
  447. package/test/fixtures/latency-journey-app/README.md +2 -0
  448. package/test/fixtures/readiness-app/README.md +2 -0
  449. package/test/helpers/ios-trace-shutdown.ts +29 -0
  450. package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
  451. package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
  452. package/test/unit/debugger/diagnostics.test.ts +80 -0
  453. package/test/unit/debugger/guards.test.ts +4 -0
  454. package/test/unit/debugger/handle-guards.test.ts +88 -0
  455. package/test/unit/debugger/ios-preparation.test.ts +57 -0
  456. package/test/unit/debugger/ios.test.ts +104 -0
  457. package/test/unit/debugger/lldb-worker.py +166 -0
  458. package/test/unit/debugger/lldb-worker.test.ts +10 -0
  459. package/test/unit/debugger/preparation.test.ts +32 -0
  460. package/test/unit/debugger/protocol.test.ts +15 -0
  461. package/test/unit/debugger/sessions.test.ts +1 -1
  462. package/test/unit/interact/adjustment_service.test.ts +42 -0
  463. package/test/unit/interact/adjustment_verification.test.ts +35 -0
  464. package/test/unit/interact/clock_dependencies.test.ts +63 -0
  465. package/test/unit/interact/element_helpers.test.ts +19 -0
  466. package/test/unit/interact/legacy_wait_service.test.ts +17 -0
  467. package/test/unit/interact/state_comparison.test.ts +20 -0
  468. package/test/unit/interact/target_scope.test.ts +35 -0
  469. package/test/unit/network/get_network_activity.test.ts +1 -1
  470. package/test/unit/observe/ios-trace-native.test.ts +90 -0
  471. package/test/unit/observe/ios-trace-sessions.test.ts +131 -0
  472. package/test/unit/observe/readiness.test.ts +1 -1
  473. package/test/unit/observe/screenshot-encoding.test.ts +30 -0
  474. package/test/unit/observe/snapshot-logs.test.ts +14 -0
  475. package/test/unit/observe/system-trace-router.test.ts +49 -0
  476. package/test/unit/quality/agent-plugin-export.test.ts +38 -0
  477. package/test/unit/quality/architecture.test.ts +60 -0
  478. package/test/unit/quality/code-quality.test.ts +58 -0
  479. package/test/unit/quality/interaction_boundaries.test.ts +24 -0
  480. package/test/unit/server/conditional_actions.test.ts +1 -1
  481. package/test/unit/server/conditional_schema.test.ts +2 -2
  482. package/test/unit/server/debugger-response.test.ts +63 -0
  483. package/test/unit/server/system-trace.test.ts +12 -3
  484. package/test/unit/server/tool_definitions_parity.test.ts +8 -0
  485. package/dist/server/debugger-handlers.js +0 -38
  486. package/dist/server/system-trace-schema.js +0 -17
  487. package/dist/server/tool-handlers.js +0 -1000
  488. package/docs/ROADMAP.md +0 -646
  489. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
  490. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
  491. package/docs/specs/baseline-spec-v0.md +0 -312
  492. package/docs/specs/mcp-tooling-spec-v1.md +0 -446
  493. package/src/server/debugger-handlers.ts +0 -29
  494. package/src/server/debugger-schema.ts +0 -37
  495. package/src/server/system-trace-schema.ts +0 -17
  496. package/src/server/tool-handlers.ts +0 -1008
  497. /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
  498. /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
  499. /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
  500. /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
@@ -1,19 +1,34 @@
1
- import { checkDebuggerGuard } from '../debugger/guard.js'
2
- import { currentReadinessWait, observeReadinessWait, isDeviceUnavailable } from '../observe/readiness.js'
3
- import { readinessDelay } from '../observe/readiness-service.js'
1
+ import type { UiTree } from './observation.js'
2
+ import { resolveTap } from './targeting/tap-target.js'
3
+ import { waitForUICore } from './waits/legacy-wait.js'
4
+ import type { LegacyWaitInput } from './waits/legacy-wait-state.js'
5
+ import { expectState } from './assertions/state-assertion.js'
6
+ import type { StateInput } from './assertions/state-assertion-context.js'
7
+ import { waitForUIChange } from './waits/ui-change-wait.js'
8
+ import type { ChangeInput } from './waits/ui-change-state.js'
9
+ import { waitForScreenChange, type ScreenWaitInput } from './waits/screen-wait.js'
10
+ import { expectScreen, type ScreenExpectation } from './assertions/screen-assertion.js'
11
+ import { expectVisible, type VisibilityInput } from './assertions/visibility-assertion.js'
12
+ import { findElement, type FindInput } from './targeting/find-element.js'
13
+ import { waitForUI } from './waits/ui-wait.js'
14
+ import type { UIWaitInput } from './waits/ui-wait-state.js'
15
+ import { adjustControl } from './adjustment/adjustment.js'
16
+ import type { AdjustmentInput } from './adjustment/adjustment-context.js'
17
+ import type { InputService } from './input-service.js'
18
+ import { ResolvedElements } from './targeting/resolved-elements.js'
19
+ import { findScopedElement, resolveUiChangeScope, type ScopeTree } from './targeting/target-scope.js'
20
+ import type { ScreenFingerprintResponse, UiElement, ResolvedUiElementContext } from './model.js'
21
+ import { _normalizeBounds, _computeElementId, _resolvedTargetFromElement } from './targeting/element-helpers.js'
22
+ import { _collectSubtreeIndices } from './waits/ui-change-helpers.js'
4
23
  import { isConditionBlocked } from './conditional-action.js'
5
- import { remainingBudget } from '../utils/operation-budget.js'
6
- import { createHash } from 'crypto'
7
24
  import { AndroidInteract } from './android.js';
8
25
  import { iOSInteract } from './ios.js';
9
26
  export { AndroidInteract, iOSInteract };
10
27
 
11
28
  import { resolveTargetDevice } from '../utils/resolve-device.js'
12
29
  import { ToolsObserve } from '../observe/index.js'
13
- import { computeSnapshotSignature } from '../observe/snapshot-metadata.js'
14
- import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js'
30
+ import { buildActionExecutionResult } from '../runtime/action-results.js'
15
31
  import type {
16
- ActionTrace,
17
32
  ActionFailureCode,
18
33
  ActionTargetResolved,
19
34
  AdjustControlResponse,
@@ -22,318 +37,17 @@ import type {
22
37
  ExpectStateResponse,
23
38
  ExpectScreenResponse,
24
39
  WaitForUIChangeResponse,
25
- UIElementSemanticMetadata,
26
- UIElementState,
27
- TraceStep,
28
40
  TapElementResponse
29
41
  } from '../types.js'
30
42
 
31
- interface ScreenFingerprintResponse { fingerprint: string | null }
32
-
33
- interface UiElement {
34
- text?: string | null
35
- label?: string | null
36
- value?: string | null
37
- contentDescription?: string | null
38
- contentDesc?: string | null
39
- accessibilityLabel?: string | null
40
- resourceId?: string | null
41
- resourceID?: string | null
42
- id?: string | null
43
- type?: string | null
44
- class?: string | null
45
- bounds?: number[] | null
46
- clickable?: boolean
47
- enabled?: boolean
48
- focusable?: boolean
49
- visible?: boolean
50
- parentId?: number | string | null
51
- children?: number[]
52
- _index?: number
53
- _interactable?: boolean
54
- _sliderLike?: boolean
55
- state?: UIElementState | null
56
- stable_id?: string | null
57
- role?: string | null
58
- test_tag?: string | null
59
- selector?: { value: string | null, confidence: { score: number, reason: string } | null } | null
60
- semantic?: UIElementSemanticMetadata | null
61
- actions?: string[] | null
62
- accessibilityActions?: string[] | null
63
- runtimeActions?: string[] | null
64
- platformActions?: string[] | null
65
- }
66
-
67
- interface ResolvedUiElementContext {
68
- elementId: string
69
- platform: 'android' | 'ios'
70
- deviceId?: string
71
- bounds: [number, number, number, number] | null
72
- index: number
73
- stable_id?: string | null
74
- }
75
-
76
- interface UiResolution {
77
- width?: number
78
- height?: number
79
- }
80
-
81
- interface UiChangeSignatureSet {
82
- hierarchy: string | null
83
- text: string | null
84
- state: string | null
85
- }
86
-
87
- interface UiChangeScopeResolution {
88
- scope: 'screen' | 'subtree'
89
- target: string | null
90
- resolved: boolean
91
- resolvedIndex: number | null
92
- resolvedStableId: string | null
93
- reason: string
94
- }
95
-
96
- interface UiChangeScopeResult {
97
- elements: UiElement[]
98
- resolution: UiChangeScopeResolution
99
- error?: {
100
- code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
101
- message: string
102
- }
103
- }
104
-
105
- interface RankedResolutionCandidate {
106
- el: UiElement
107
- idx: number
108
- score: number
109
- reason: string
110
- interactable: boolean
111
- }
112
-
113
- interface InferredLabeledSliderState {
114
- current: number
115
- min: number
116
- max: number
117
- value_range: { min: number; max: number }
118
- touch_bounds: [number, number, number, number]
119
- source: 'nearby_labels'
120
- }
121
-
122
- function buildObservationTrace({
123
- actionType,
124
- stage,
125
- success,
126
- attempts,
127
- metadata
128
- }: {
129
- actionType: string
130
- stage: 'verify' | 'stabilize' | 'resolve'
131
- success: boolean
132
- attempts: number
133
- metadata?: Record<string, unknown>
134
- }): ActionTrace {
135
- const now = Date.now()
136
- const actionId = nextActionId(actionType, now)
137
- const steps: TraceStep[] = [
138
- createTraceStep({
139
- stage,
140
- timestamp: now,
141
- result: success ? 'success' : 'failure',
142
- attemptIndex: 0,
143
- metadata
144
- })
145
- ]
146
-
147
- return {
148
- action_id: actionId,
149
- steps,
150
- final_outcome: success ? 'success' : 'failure',
151
- attempts: Math.max(1, Math.floor(attempts || 1))
152
- }
153
- }
154
-
155
- interface FindElementResolutionSummary {
156
- confidence: number
157
- reason: string
158
- fallback_available: boolean
159
- matched_count: number
160
- alternates: Array<{
161
- text: string | null
162
- resource_id: string | null
163
- accessibility_id: string | null
164
- class: string | null
165
- bounds: { left: number; top: number; right: number; bottom: number } | null
166
- clickable: boolean
167
- enabled: boolean
168
- score: number
169
- reason: string
170
- }>
171
- }
172
-
173
-
174
43
  export class ToolsInteract {
175
44
  private static readonly _maxResolvedUiElements = 256
176
- private static readonly _uiChangeKinds: Array<'hierarchy_diff' | 'text_change' | 'state_change'> = ['hierarchy_diff', 'text_change', 'state_change']
177
- private static readonly _sliderSearchLookahead = 8
178
- private static readonly _sliderNegativeGapTolerancePx = 32
179
- private static readonly _sliderPositiveGapLimitPx = 640
180
- private static readonly _sliderTrackMinLengthPx = 220
181
- private static readonly _sliderTrackMaxThicknessPx = 180
182
- private static readonly _sliderTrackLengthRatio = 0.18
183
- private static readonly _sliderTrackThicknessRatio = 0.08
184
- private static readonly _sliderLabelWidthRatio = 1.5
185
- private static _resolvedUiElements = new Map<string, ResolvedUiElementContext>()
186
-
187
- private static _normalize(s: any): string {
188
- if (s === null || s === undefined) return ''
189
- try { return String(s).toLowerCase().trim() } catch { return '' }
190
- }
191
-
192
- private static _normalizeBounds(bounds: any): [number, number, number, number] | null {
193
- if (!Array.isArray(bounds) || bounds.length < 4) return null
194
- const normalized = bounds.slice(0, 4).map((value: any) => Number(value))
195
- if (normalized.some((value: number) => Number.isNaN(value))) return null
196
- return normalized as [number, number, number, number]
197
- }
198
-
199
- private static _hash(value: unknown): string {
200
- return createHash('sha256').update(JSON.stringify(value)).digest('hex')
201
- }
202
-
203
- private static _matchesSelector(el: UiElement, selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }): boolean {
204
- if (!selector) return false
205
- const normalize = ToolsInteract._normalize
206
- const containsFlag = !!selector.contains
207
- const text = normalize(el.text ?? el.label ?? el.value ?? '')
208
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
209
- const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '')
210
-
211
- if (selector.text !== undefined && selector.text !== null) {
212
- const q = normalize(selector.text)
213
- if (containsFlag ? !text.includes(q) : text !== q) return false
214
- }
215
-
216
- if (selector.resource_id !== undefined && selector.resource_id !== null) {
217
- const q = normalize(selector.resource_id)
218
- if (containsFlag ? !resourceId.includes(q) : resourceId !== q) return false
219
- }
220
-
221
- if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
222
- const q = normalize(selector.accessibility_id)
223
- if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q) return false
224
- }
225
-
226
- return true
227
- }
228
-
229
- private static _findFirstMatchingElement(
230
- elements: UiElement[],
231
- selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }
232
- ): { el: UiElement, idx: number } | null {
233
- if (!selector) return null
234
- for (let i = 0; i < elements.length; i++) {
235
- const el = elements[i]
236
- if (!el) continue
237
- if (ToolsInteract._matchesSelector(el, selector)) return { el, idx: i }
238
- }
239
- return null
240
- }
241
-
242
- private static _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
243
- if (parentId === undefined || parentId === null) return null
244
-
245
- if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
246
- return parentId
247
- }
248
-
249
- if (typeof parentId === 'string') {
250
- const normalized = ToolsInteract._normalize(parentId)
251
- if (!normalized) return null
252
-
253
- if (/^\d+$/.test(normalized)) {
254
- const index = Number(normalized)
255
- if (index >= 0 && index < elements.length) return index
256
- }
257
-
258
- const foundIndex = elements.findIndex((el) => {
259
- if (!el) return false
260
- return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
261
- ToolsInteract._normalize(el.stable_id ?? '') === normalized
262
- })
263
-
264
- return foundIndex >= 0 ? foundIndex : null
265
- }
266
-
267
- return null
268
- }
269
-
270
- private static _isVisibleElement(el: UiElement): boolean {
271
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
272
- return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
273
- }
274
45
 
275
- private static _isTapActionable(
276
- el: UiElement,
277
- storedStableId?: string | null,
278
- platform?: 'android' | 'ios'
279
- ): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
280
- if (!ToolsInteract._isVisibleElement(el)) {
281
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
282
- }
283
-
284
- if (el.enabled === false) {
285
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
286
- }
287
-
288
- const semanticTapActionable = !!el.semantic && (
289
- el.semantic.is_clickable ||
290
- (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap'))
291
- )
292
-
293
- if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
294
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
295
- }
296
-
297
- if (storedStableId) {
298
- if (!el.stable_id || el.stable_id !== storedStableId) {
299
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
300
- }
301
- }
302
-
303
- return { actionable: true }
304
- }
305
-
306
- private static _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
307
- if (!ToolsInteract._isVisibleElement(el)) {
308
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
309
- }
310
-
311
- if (el.enabled === false) {
312
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
313
- }
314
-
315
- if (storedStableId) {
316
- if (!el.stable_id || el.stable_id !== storedStableId) {
317
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
318
- }
319
- }
46
+ private static _resolvedUiElements = new ResolvedElements(ToolsInteract._maxResolvedUiElements)
320
47
 
321
- return { actionable: true }
322
- }
48
+ private static _normalizeBounds = _normalizeBounds
323
49
 
324
- private static _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
325
- const identity = {
326
- platform,
327
- deviceId: deviceId || '',
328
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
329
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
330
- accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
331
- class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
332
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
333
- index
334
- }
335
- return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`
336
- }
50
+ private static _computeElementId = _computeElementId
337
51
 
338
52
  private static _buildResolvedElement(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number) {
339
53
  const bounds = ToolsInteract._normalizeBounds(el.bounds)
@@ -365,247 +79,18 @@ export class ToolsInteract {
365
79
  }
366
80
  }
367
81
 
368
- private static _resolveUiChangeScope(
369
- tree: any,
370
- scope: 'screen' | 'subtree' | undefined,
371
- target: string | null | undefined
372
- ): UiChangeScopeResult {
373
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
374
- const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen'
375
-
376
- if (normalizedScope === 'screen') {
377
- return {
378
- elements,
379
- resolution: {
380
- scope: 'screen',
381
- target: null,
382
- resolved: true,
383
- resolvedIndex: null,
384
- resolvedStableId: null,
385
- reason: 'screen scope'
386
- }
387
- }
388
- }
389
-
390
- const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null
391
- if (!requestedTarget) {
392
- return {
393
- elements: [],
394
- resolution: {
395
- scope: 'subtree',
396
- target: null,
397
- resolved: false,
398
- resolvedIndex: null,
399
- resolvedStableId: null,
400
- reason: 'subtree scope requires a target element id'
401
- },
402
- error: {
403
- code: 'INVALID_SCOPE',
404
- message: 'scope=subtree requires a target element_id'
405
- }
406
- }
407
- }
408
-
409
- const resolved = ToolsInteract._findScopedElement(tree, requestedTarget)
410
- if (!resolved) {
411
- return {
412
- elements: [],
413
- resolution: {
414
- scope: 'subtree',
415
- target: requestedTarget,
416
- resolved: false,
417
- resolvedIndex: null,
418
- resolvedStableId: null,
419
- reason: 'target element could not be resolved'
420
- },
421
- error: {
422
- code: 'ELEMENT_NOT_FOUND',
423
- message: `Target element ${requestedTarget} could not be resolved for subtree scope`
424
- }
425
- }
426
- }
427
-
428
- const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index)
429
- const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element): element is UiElement => !!element)
430
-
431
- return {
432
- elements: scopedElements,
433
- resolution: {
434
- scope: 'subtree',
435
- target: requestedTarget,
436
- resolved: true,
437
- resolvedIndex: resolved.index,
438
- resolvedStableId: resolved.stableId,
439
- reason: resolved.reason
440
- }
441
- }
442
- }
443
-
444
- private static _findScopedElement(tree: any, targetElementId: string): { index: number, stableId: string | null, reason: string } | null {
445
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
446
- const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android'
447
- const deviceId = tree?.device?.id ?? undefined
448
- const normalizedTarget = ToolsInteract._normalize(targetElementId)
449
-
450
- for (let i = 0; i < elements.length; i++) {
451
- const el = elements[i]
452
- if (!el) continue
453
-
454
- const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i)
455
- if (computedElementId === targetElementId) {
456
- return {
457
- index: i,
458
- stableId: el.stable_id ?? null,
459
- reason: 'element_id_match'
460
- }
461
- }
462
- }
463
-
464
- for (let i = 0; i < elements.length; i++) {
465
- const el = elements[i]
466
- if (!el) continue
467
-
468
- if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
469
- return {
470
- index: i,
471
- stableId: el.stable_id,
472
- reason: 'stable_id_match'
473
- }
474
- }
475
- }
476
-
477
- const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId)
478
- if (storedContext?.stable_id) {
479
- const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id)
480
- for (let i = 0; i < elements.length; i++) {
481
- const el = elements[i]
482
- if (!el?.stable_id) continue
483
- if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
484
- return {
485
- index: i,
486
- stableId: el.stable_id,
487
- reason: 'stored_stable_id_match'
488
- }
489
- }
490
- }
491
- }
492
-
493
- return null
494
- }
495
-
496
- private static _collectSubtreeIndices(elements: UiElement[], rootIndex: number): number[] {
497
- if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length) return []
498
-
499
- const visited = new Set<number>()
500
- const stack = [rootIndex]
501
- const result: number[] = []
502
-
503
- while (stack.length > 0) {
504
- const index = stack.pop()
505
- if (index === undefined || visited.has(index) || index < 0 || index >= elements.length) continue
506
- visited.add(index)
507
- result.push(index)
508
-
509
- const element = elements[index]
510
- if (!element) continue
511
-
512
- const directChildren = new Set<number>()
513
- if (Array.isArray(element.children)) {
514
- for (const childIndex of element.children) {
515
- if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
516
- directChildren.add(childIndex)
517
- }
518
- }
519
- }
520
-
521
- for (let i = 0; i < elements.length; i++) {
522
- if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
523
- directChildren.add(i)
524
- }
525
- }
526
-
527
- for (const childIndex of directChildren) {
528
- if (!visited.has(childIndex)) stack.push(childIndex)
529
- }
530
- }
531
-
532
- return result.sort((left, right) => left - right)
82
+ private static _resolveUiChangeScope(tree: ScopeTree | null | undefined, scope: 'screen' | 'subtree' | undefined, target: string | null | undefined) {
83
+ return resolveUiChangeScope(tree, scope, target, (snapshot, id) => ToolsInteract._findScopedElement(snapshot, id))
533
84
  }
534
85
 
535
- private static _changeIdentityForElement(el: UiElement, index: number): string {
536
- const stableId = ToolsInteract._normalize(el.stable_id)
537
- if (stableId) return `stable:${stableId}`
538
-
539
- return `fallback:${ToolsInteract._hash({
540
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
541
- contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
542
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
543
- type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
544
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
545
- index
546
- })}`
86
+ private static _findScopedElement(tree: ScopeTree | null | undefined, id: string) {
87
+ return findScopedElement(tree, id, ToolsInteract._resolvedUiElements)
547
88
  }
548
89
 
549
- private static _summarizeUiChangeDelta(initialElements: UiElement[], currentElements: UiElement[]) {
550
- const buildMap = (elements: UiElement[]) => {
551
- const map = new Map<string, string>()
552
- for (let i = 0; i < elements.length; i++) {
553
- const element = elements[i]
554
- if (!element) continue
555
- const key = ToolsInteract._changeIdentityForElement(element, i)
556
- map.set(key, ToolsInteract._hash({
557
- text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
558
- contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
559
- resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
560
- type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
561
- bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
562
- state: element.state ?? null,
563
- visible: !!element.visible,
564
- enabled: !!element.enabled,
565
- clickable: !!element.clickable
566
- }))
567
- }
568
- return map
569
- }
570
-
571
- const initialMap = buildMap(initialElements)
572
- const currentMap = buildMap(currentElements)
573
- let added = 0
574
- let removed = 0
575
- let mutated = 0
576
-
577
- for (const [key, value] of currentMap.entries()) {
578
- if (!initialMap.has(key)) {
579
- added++
580
- } else if (initialMap.get(key) !== value) {
581
- mutated++
582
- }
583
- }
584
-
585
- for (const key of initialMap.keys()) {
586
- if (!currentMap.has(key)) removed++
587
- }
588
-
589
- return {
590
- total_elements: currentElements.length,
591
- added_elements: added,
592
- removed_elements: removed,
593
- mutated_elements: mutated
594
- }
595
- }
90
+ private static _collectSubtreeIndices = _collectSubtreeIndices
596
91
 
597
92
  private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
598
- if (ToolsInteract._resolvedUiElements.has(elementId)) {
599
- ToolsInteract._resolvedUiElements.delete(elementId)
600
- }
601
-
602
- ToolsInteract._resolvedUiElements.set(elementId, context)
603
-
604
- while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
605
- const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value
606
- if (!oldestElementId) break
607
- ToolsInteract._resolvedUiElements.delete(oldestElementId)
608
- }
93
+ ToolsInteract._resolvedUiElements.remember(elementId, context)
609
94
  }
610
95
 
611
96
  private static async _captureFingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null> {
@@ -618,454 +103,7 @@ export class ToolsInteract {
618
103
  }
619
104
  }
620
105
 
621
- private static _buildUiChangeSignatures(tree: any): UiChangeSignatureSet {
622
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
623
- const textPayload: Array<{ text: string, contentDescription: string, resourceId: string }> = []
624
- const statePayload: Array<{
625
- checked: boolean | null
626
- selected: boolean | string | { id: string; label?: string } | null
627
- focused: boolean | null
628
- expanded: boolean | null
629
- enabled: boolean | null
630
- text_value: string | null
631
- value: number | string | null
632
- raw_value: number | string | null
633
- value_range: UIElementState['value_range']
634
- }> = []
635
-
636
- for (const el of elements) {
637
- textPayload.push({
638
- text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
639
- contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
640
- resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
641
- })
642
-
643
- statePayload.push({
644
- checked: el?.state?.checked ?? null,
645
- selected: el?.state?.selected ?? null,
646
- focused: el?.state?.focused ?? null,
647
- expanded: el?.state?.expanded ?? null,
648
- enabled: el?.state?.enabled ?? null,
649
- text_value: el?.state?.text_value ?? null,
650
- value: el?.state?.value ?? null,
651
- raw_value: el?.state?.raw_value ?? null,
652
- value_range: el?.state?.value_range ?? null
653
- })
654
- }
655
-
656
- return {
657
- hierarchy: computeSnapshotSignature(tree),
658
- text: ToolsInteract._hash({
659
- screen: ToolsInteract._normalize(tree?.screen),
660
- elements: textPayload
661
- }),
662
- state: ToolsInteract._hash({
663
- screen: ToolsInteract._normalize(tree?.screen),
664
- elements: statePayload
665
- })
666
- }
667
- }
668
-
669
- private static _matchesUiChange(expected: 'hierarchy_diff' | 'text_change' | 'state_change' | undefined, initial: UiChangeSignatureSet, current: UiChangeSignatureSet): 'hierarchy_diff' | 'text_change' | 'state_change' | null {
670
- const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds
671
-
672
- for (const changeKind of candidates) {
673
- if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy) return changeKind
674
- if (changeKind === 'text_change' && initial.text !== current.text) return changeKind
675
- if (changeKind === 'state_change' && initial.state !== current.state) return changeKind
676
- }
677
-
678
- return null
679
- }
680
-
681
- private static _uiChangeSignaturesEqual(left: UiChangeSignatureSet, right: UiChangeSignatureSet): boolean {
682
- return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state
683
- }
684
-
685
- private static _resolvedTargetFromElement(
686
- elementId: string,
687
- element: UiElement,
688
- index: number
689
- ): ActionTargetResolved {
690
- return {
691
- elementId,
692
- text: element.text ?? null,
693
- resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
694
- accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
695
- class: element.type ?? element.class ?? null,
696
- bounds: ToolsInteract._normalizeBounds(element.bounds),
697
- index,
698
- state: element.state ?? null,
699
- stable_id: element.stable_id ?? null,
700
- role: element.role ?? null,
701
- test_tag: element.test_tag ?? null,
702
- selector: element.selector ?? null,
703
- semantic: element.semantic ?? null
704
- }
705
- }
706
-
707
- private static _summarizeResolutionCandidate(candidate: RankedResolutionCandidate): FindElementResolutionSummary['alternates'][number] {
708
- const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds)
709
- return {
710
- text: candidate.el.text ?? null,
711
- resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
712
- accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
713
- class: candidate.el.type ?? candidate.el.class ?? null,
714
- bounds: bounds
715
- ? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
716
- : null,
717
- clickable: !!candidate.el.clickable,
718
- enabled: !!candidate.el.enabled,
719
- score: candidate.score,
720
- reason: candidate.reason
721
- }
722
- }
723
-
724
- private static _isAdjustableControl(el: UiElement | null): boolean {
725
- if (!el) return false
726
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
727
- const role = ToolsInteract._normalize(el.role ?? '')
728
- return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
729
- }
730
-
731
- private static _isSemanticActionable(el: UiElement | null): boolean {
732
- if (!el?.semantic) return false
733
- if (el.semantic.adjustable) return true
734
- return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0
735
- }
736
-
737
- private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
738
- if (!el?.state) return null
739
- const stateValue = el.state[property as keyof UIElementState]
740
- if (typeof stateValue === 'number' && Number.isFinite(stateValue)) return stateValue
741
- const parsedStateValue = ToolsInteract._parseNumericControlValue(stateValue)
742
- if (parsedStateValue !== null) return parsedStateValue
743
- if (property === 'value' || property === 'raw_value') {
744
- const fallback = el.state.raw_value ?? el.state.value
745
- if (typeof fallback === 'number' && Number.isFinite(fallback)) return fallback
746
- return ToolsInteract._parseNumericControlValue(fallback)
747
- }
748
- return null
749
- }
750
-
751
- private static _parseNumericControlValue(value: unknown): number | null {
752
- if (typeof value === 'number' && Number.isFinite(value)) return value
753
- if (typeof value !== 'string') return null
754
- const trimmed = value.trim()
755
- if (!trimmed) return null
756
-
757
- const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/)
758
- if (percent) return Number(percent[1])
759
-
760
- const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i)
761
- if (fraction) return Number(fraction[1])
762
-
763
- const plain = Number(trimmed)
764
- if (Number.isFinite(plain)) return plain
765
-
766
- const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/)
767
- if (trailing) return Number(trailing[1])
768
-
769
- return null
770
- }
771
-
772
- private static _readNumericControlRange(el: UiElement | null): { min: number; max: number } | null {
773
- const min = typeof el?.state?.value_range?.min === 'number' ? el.state.value_range.min : null
774
- const max = typeof el?.state?.value_range?.max === 'number' ? el.state.value_range.max : null
775
- if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
776
- return { min, max }
777
- }
778
-
779
- const value = el?.state?.value ?? el?.state?.raw_value
780
- if (typeof value === 'string') {
781
- const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i)
782
- if (fraction) {
783
- const parsedMax = Number(fraction[1])
784
- if (Number.isFinite(parsedMax) && parsedMax > 0) return { min: 0, max: parsedMax }
785
- }
786
- }
787
-
788
- return null
789
- }
790
-
791
- private static _readControlStep(el: UiElement | null): number | null {
792
- const candidates = [
793
- el?.state?.step,
794
- el?.state?.value_range?.step
795
- ]
796
- for (const candidate of candidates) {
797
- if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0) return candidate
798
- }
799
- return null
800
- }
801
-
802
- private static _quantizeTargetValue(targetValue: number, min: number, max: number, step: number | null): number {
803
- if (!step) return targetValue
804
- const quantized = min + (Math.round((targetValue - min) / step) * step)
805
- const bounded = Math.max(min, Math.min(max, quantized))
806
- const precision = ToolsInteract._decimalPrecisionForStep(step)
807
- return Number(bounded.toFixed(precision))
808
- }
809
-
810
- private static _decimalPrecisionForStep(step: number): number {
811
- const normalized = String(step).toLowerCase()
812
- const [coefficient, exponentPart] = normalized.split('e')
813
- const coefficientDecimals = coefficient.includes('.')
814
- ? coefficient.split('.')[1]?.length ?? 0
815
- : 0
816
- const exponent = exponentPart !== undefined ? Number(exponentPart) : 0
817
- const precision = exponent < 0
818
- ? coefficientDecimals + Math.abs(exponent)
819
- : Math.max(0, coefficientDecimals - exponent)
820
- return Math.max(0, Math.min(12, precision))
821
- }
822
-
823
- private static _defaultAdjustTolerance(range: { min: number; max: number } | null, step: number | null): number {
824
- if (step) return 0
825
- if (!range) return 0
826
- return Math.max(0, 0.01 * Math.abs(range.max - range.min))
827
- }
828
-
829
- private static _runtimeActionSet(el: UiElement | null): Set<string> {
830
- const values = [
831
- ...(Array.isArray(el?.actions) ? el!.actions! : []),
832
- ...(Array.isArray(el?.accessibilityActions) ? el!.accessibilityActions! : []),
833
- ...(Array.isArray(el?.runtimeActions) ? el!.runtimeActions! : []),
834
- ...(Array.isArray(el?.platformActions) ? el!.platformActions! : [])
835
- ]
836
- return new Set(values.map((value) => ToolsInteract._normalize(value)).filter(Boolean))
837
- }
838
-
839
- private static _hasDirectAdjustmentAction(el: UiElement | null): boolean {
840
- const actions = ToolsInteract._runtimeActionSet(el)
841
- return [
842
- 'action_set_progress',
843
- 'set_progress',
844
- 'setprogress',
845
- 'semanticsactions.setprogress',
846
- 'set_value',
847
- 'setvalue',
848
- 'direct_set_value'
849
- ].some((action) => actions.has(action))
850
- }
851
-
852
- private static _adjustmentDirections(el: UiElement | null): { increment: boolean; decrement: boolean } {
853
- const actions = ToolsInteract._runtimeActionSet(el)
854
- return {
855
- increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
856
- decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
857
- }
858
- }
859
-
860
- private static _textFromElement(el: UiElement | null): string {
861
- return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim()
862
- }
863
-
864
- private static _parseLeadingNumber(text: string): number | null {
865
- const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/)
866
- if (!match) return null
867
- const value = Number(match[1])
868
- return Number.isFinite(value) ? value : null
869
- }
870
-
871
- private static _inferLabeledSliderState(elements: UiElement[], track: UiElement | null): InferredLabeledSliderState | null {
872
- const trackBounds = ToolsInteract._normalizeBounds(track?.bounds)
873
- if (!track || !trackBounds) return null
874
- const type = ToolsInteract._normalize(track.type ?? track.class ?? '')
875
- const role = ToolsInteract._normalize(track.role ?? '')
876
- const width = trackBounds[2] - trackBounds[0]
877
- const height = trackBounds[3] - trackBounds[1]
878
- const wideTrackLike = width >= 240 && height <= 220
879
- const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role)
880
- if (!wideTrackLike && !knownTrackLike) return null
881
-
882
- const trackTop = trackBounds[1]
883
- const trackBottom = trackBounds[3]
884
- const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2
885
- const labels = elements
886
- .map((el, index) => {
887
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
888
- const text = ToolsInteract._textFromElement(el)
889
- const value = ToolsInteract._parseLeadingNumber(text)
890
- return bounds && text && value !== null
891
- ? {
892
- index,
893
- el,
894
- text,
895
- value,
896
- bounds,
897
- centerX: (bounds[0] + bounds[2]) / 2,
898
- centerY: (bounds[1] + bounds[3]) / 2,
899
- hasUnit: /[a-zA-Z%]/.test(text)
900
- }
901
- : null
902
- })
903
- .filter((label): label is NonNullable<typeof label> => !!label)
904
-
905
- const rangeLabels = labels
906
- .filter((label) => {
907
- const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260
908
- const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120
909
- return verticallyNearTrack && horizontallyNearTrack && label.hasUnit
910
- })
911
- .sort((a, b) => a.centerX - b.centerX)
912
-
913
- if (rangeLabels.length < 2) return null
914
- const minLabel = rangeLabels[0]
915
- const maxLabel = rangeLabels[rangeLabels.length - 1]
916
- if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value) return null
917
-
918
- const currentCandidates = labels
919
- .filter((label) => {
920
- if (label.index === minLabel.index || label.index === maxLabel.index) return false
921
- if (label.value < minLabel.value || label.value > maxLabel.value) return false
922
- if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20) return false
923
- return label.centerX >= trackCenterX - 80
924
- })
925
- .sort((a, b) => {
926
- const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY)
927
- if (yDistance !== 0) return yDistance
928
- return b.centerX - a.centerX
929
- })
930
-
931
- const currentLabel = currentCandidates[0]
932
- if (!currentLabel) return null
933
-
934
- const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop
935
- const labelDerivedBounds: [number, number, number, number] = [
936
- Math.min(minLabel.bounds[0], trackBounds[0]),
937
- Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
938
- Math.max(maxLabel.bounds[2], trackBounds[2]),
939
- Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
940
- ]
941
-
942
- return {
943
- current: currentLabel.value,
944
- min: minLabel.value,
945
- max: maxLabel.value,
946
- value_range: { min: minLabel.value, max: maxLabel.value },
947
- touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
948
- source: 'nearby_labels'
949
- }
950
- }
951
-
952
- private static _buildControlPoint(bounds: [number, number, number, number], ratio: number, axis: 'horizontal' | 'vertical') {
953
- const clampedRatio = Math.max(0, Math.min(1, ratio))
954
- const [left, top, right, bottom] = bounds
955
- const width = Math.max(1, right - left)
956
- const height = Math.max(1, bottom - top)
957
- const insetX = Math.max(8, Math.floor(width * 0.08))
958
- const insetY = Math.max(8, Math.floor(height * 0.08))
959
- if (axis === 'vertical') {
960
- const usableHeight = Math.max(1, height - (insetY * 2))
961
- return {
962
- x: Math.floor((left + right) / 2),
963
- y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
964
- }
965
- }
966
- const usableWidth = Math.max(1, width - (insetX * 2))
967
- return {
968
- x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
969
- y: Math.floor((top + bottom) / 2)
970
- }
971
- }
972
-
973
- private static _buildConservativeControlPoint(
974
- bounds: [number, number, number, number],
975
- targetValue: number,
976
- currentValue: number | null,
977
- min: number,
978
- max: number,
979
- axis: 'horizontal' | 'vertical'
980
- ) {
981
- const range = Math.max(1, max - min)
982
- const targetRatio = (targetValue - min) / range
983
- const stepRatio = 1 / range
984
- const centerBias = stepRatio / 2
985
- const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue)
986
- const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0])
987
- const edgeWindow = Math.max(3, Math.floor(range * 0.1))
988
- const isNearLowEdge = targetValue - min <= edgeWindow
989
- const isNearHighEdge = max - targetValue <= edgeWindow
990
- const directionBias = direction > 0
991
- ? -stepRatio * 0.15
992
- : direction < 0
993
- ? stepRatio * 0.65
994
- : 0
995
- const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx))
996
- const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin)
997
- const edgeBias = isNearLowEdge
998
- ? endpointMargin
999
- : isNearHighEdge
1000
- ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
1001
- : 0
1002
- const safeRatio = Math.min(
1003
- 1 - (endpointMargin * 0.25),
1004
- Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias)
1005
- )
1006
- return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
1007
- }
1008
-
1009
- private static _buildAdjustmentProbePoints(
1010
- bounds: [number, number, number, number],
1011
- targetValue: number,
1012
- currentValue: number | null,
1013
- min: number,
1014
- max: number,
1015
- axis: 'horizontal' | 'vertical'
1016
- ) {
1017
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
1018
- const currentPoint = currentValue !== null
1019
- ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
1020
- : ToolsInteract._buildControlPoint(bounds, 0.5, axis)
1021
-
1022
- const [left, top, right, bottom] = bounds
1023
- const width = Math.max(1, right - left)
1024
- const height = Math.max(1, bottom - top)
1025
- const crossAxisBumps = axis === 'horizontal'
1026
- ? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
1027
- : [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))]
1028
-
1029
- const clampPoint = (point: { x: number, y: number }) => ({
1030
- x: axis === 'horizontal'
1031
- ? Math.max(left, Math.min(right, point.x))
1032
- : Math.max(left, Math.min(right + Math.max(width, height), point.x)),
1033
- y: axis === 'vertical'
1034
- ? Math.max(top, Math.min(bottom, point.y))
1035
- : Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
1036
- })
1037
-
1038
- const probes = [targetPoint, currentPoint]
1039
- for (const bump of crossAxisBumps) {
1040
- if (axis === 'horizontal') {
1041
- probes.push(
1042
- { x: targetPoint.x, y: bottom + bump },
1043
- { x: currentPoint.x, y: bottom + bump }
1044
- )
1045
- } else {
1046
- probes.push(
1047
- { x: right + bump, y: targetPoint.y },
1048
- { x: right + bump, y: currentPoint.y }
1049
- )
1050
- }
1051
- }
1052
-
1053
- return Array.from(
1054
- new Map(
1055
- probes
1056
- .map(clampPoint)
1057
- .map((point) => [`${point.x}:${point.y}`, point] as const)
1058
- ).values()
1059
- )
1060
- }
1061
-
1062
- private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
1063
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
1064
- const role = ToolsInteract._normalize(el.role ?? '')
1065
- if (/vertical/.test(type) || /vertical/.test(role)) return 'vertical'
1066
- if (/horizontal/.test(type) || /horizontal/.test(role)) return 'horizontal'
1067
- return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal'
1068
- }
106
+ private static _resolvedTargetFromElement = _resolvedTargetFromElement
1069
107
 
1070
108
  private static _actionFailure(
1071
109
  actionType: string,
@@ -1107,129 +145,11 @@ export class ToolsInteract {
1107
145
  return null
1108
146
  }
1109
147
 
1110
- private static _resolveActionableAncestor(elements: UiElement[], chosen: { el: UiElement, idx: number } | null): { el: UiElement, idx: number } | null {
1111
- if (!chosen) return null
1112
- if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el)) return chosen
1113
-
1114
- let current = chosen
1115
- let safety = 0
1116
-
1117
- while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
1118
- const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId)
1119
-
1120
- if (parentIndex !== null && elements[parentIndex]) {
1121
- current = { el: elements[parentIndex], idx: parentIndex }
1122
- if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
1123
- } else {
1124
- break
1125
- }
1126
-
1127
- safety++
1128
- }
1129
-
1130
- const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
1131
- if (!childBounds) return null
1132
- const [cl, ct, cr, cb] = childBounds
1133
-
1134
- let best: { el: UiElement, idx: number } | null = null
1135
- let bestArea = Infinity
1136
-
1137
- for (let i = 0; i < elements.length; i++) {
1138
- const el = elements[i]
1139
- if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el))) continue
1140
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
1141
- if (!bounds) continue
1142
- const [pl, pt, pr, pb] = bounds
1143
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
1144
- const area = (pr - pl) * (pb - pt)
1145
- if (area < bestArea) {
1146
- bestArea = area
1147
- best = { el, idx: i }
1148
- }
1149
- }
1150
- }
1151
-
1152
- return best
1153
- }
1154
-
1155
- private static _resolveNearbyActionableControl(
1156
- elements: UiElement[],
1157
- chosen: { el: UiElement, idx: number } | null,
1158
- screen?: UiResolution | null
1159
- ): { el: UiElement, idx: number, sliderLike?: boolean } | null {
1160
- if (!chosen) return null
1161
-
1162
- const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
1163
- if (!labelBounds) return null
1164
-
1165
- const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds
1166
- const labelWidth = labelRight - labelLeft
1167
- const labelHeight = labelBottom - labelTop
1168
- const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0
1169
- const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0
1170
- const minTrackLengthPx = Math.max(
1171
- ToolsInteract._sliderTrackMinLengthPx,
1172
- screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0,
1173
- screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0
1174
- )
1175
- const maxTrackThicknessPx = Math.max(
1176
- ToolsInteract._sliderTrackMaxThicknessPx,
1177
- screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0,
1178
- screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0
1179
- )
1180
-
1181
- let best: { el: UiElement, idx: number, sliderLike?: boolean } | null = null
1182
- let bestScore = Infinity
1183
-
1184
- for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
1185
- const candidate = elements[i]
1186
- if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false) continue
1187
-
1188
- const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds)
1189
- if (!candidateBounds) continue
1190
-
1191
- const [left, top, right] = candidateBounds
1192
- const width = right - left
1193
- const height = candidateBounds[3] - top
1194
- const verticalGap = top - labelBottom
1195
- if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx) continue
1196
-
1197
- const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left)
1198
- if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx) continue
1199
-
1200
- const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '')
1201
- const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '')
1202
- const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '')
1203
-
1204
- let score = verticalGap
1205
- const horizontalTrackLike =
1206
- width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
1207
- height <= maxTrackThicknessPx
1208
- const verticalTrackLike =
1209
- height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
1210
- width <= maxTrackThicknessPx
1211
- const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike
1212
- if (!candidateText && !candidateContent) score -= 18
1213
- if (trackLike) score -= 30
1214
- if (/view|layout|group|frame/i.test(candidateClass)) score -= 10
1215
- if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio) score -= 8
1216
- if (candidateText || candidateContent) score += 20
1217
-
1218
- if (score < bestScore) {
1219
- bestScore = score
1220
- best = { el: candidate, idx: i, sliderLike: trackLike }
1221
- }
1222
- }
1223
-
1224
- return best
1225
- }
1226
-
1227
-
1228
148
  private static async getInteractionService(platform?: 'android' | 'ios', deviceId?: string) {
1229
149
  const effectivePlatform = platform || 'android'
1230
150
  const resolved = await resolveTargetDevice({ platform: effectivePlatform as 'android' | 'ios', deviceId })
1231
- const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract()
1232
- return { interact: interact as any, resolved, platform: effectivePlatform }
151
+ const interact: InputService = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract()
152
+ return { interact, resolved, platform: effectivePlatform }
1233
153
  }
1234
154
 
1235
155
  static async tapHandler({ platform, x, y, deviceId, timeoutMs }: { platform?: 'android' | 'ios', x: number, y: number, deviceId?: string, timeoutMs?: number }) {
@@ -1237,47 +157,35 @@ export class ToolsInteract {
1237
157
  return await interact.tap(x, y, resolved.id, timeoutMs)
1238
158
  }
1239
159
 
1240
- static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }: { elementId: string, verificationMode?: 'none' | 'light' | 'full', timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string, freshTree?: any }): Promise<TapElementResponse> {
160
+ private static _getResolvedElement(elementId: string, platform?: 'android' | 'ios', deviceId?: string) {
161
+ const resolved = ToolsInteract._resolvedUiElements.get(elementId)
162
+ if (!resolved || (platform && resolved.platform !== platform) || (deviceId && resolved.deviceId !== deviceId)) return null
163
+ return resolved
164
+ }
165
+
166
+ static getResolvedElementTarget(elementId: string, platform?: 'android' | 'ios', deviceId?: string) {
167
+ const resolved = ToolsInteract._getResolvedElement(elementId, platform, deviceId)
168
+ return resolved ? { platform: resolved.platform, deviceId: resolved.deviceId } : null
169
+ }
170
+
171
+ static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }: { elementId: string, verificationMode?: 'none' | 'light' | 'full', timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string, freshTree?: UiTree }): Promise<TapElementResponse> {
1241
172
  const actionType = 'tap_element'
1242
173
  const selector = { elementId }
1243
- const resolved = ToolsInteract._resolvedUiElements.get(elementId)
1244
- if (!resolved || (platform && resolved.platform !== platform) || (deviceId && resolved.deviceId !== deviceId)) {
174
+ const resolved = ToolsInteract._getResolvedElement(elementId, platform, deviceId)
175
+ if (!resolved) {
1245
176
  return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, null)
1246
177
  }
1247
178
 
1248
179
  const fingerprintBefore = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null
1249
180
 
1250
- const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId }) as any
1251
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform
1252
- const treeDeviceId = tree?.device?.id || resolved.deviceId
1253
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1254
- const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
1255
-
1256
- if (!currentMatch) {
1257
- return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore)
1258
- }
1259
-
1260
- const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
1261
-
1262
- const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform)
1263
- if (!tapActionability.actionable) {
1264
- return ToolsInteract._actionFailure(
1265
- actionType,
1266
- selector,
1267
- resolvedTarget,
1268
- tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
1269
- true,
1270
- fingerprintBefore
1271
- )
1272
- }
1273
-
1274
- const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
1275
- if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
1276
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
1277
- }
1278
-
1279
- const x = Math.floor((bounds[0] + bounds[2]) / 2)
1280
- const y = Math.floor((bounds[1] + bounds[3]) / 2)
181
+ const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId })
182
+ const tap = resolveTap(tree, resolved, {
183
+ current: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
184
+ target: (element, index) => ToolsInteract._resolvedTargetFromElement(resolved.elementId, element, index)
185
+ })
186
+ if ('failure' in tap) return ToolsInteract._actionFailure(actionType, selector, tap.target, tap.failure, true, fingerprintBefore)
187
+ const resolvedTarget = tap.target
188
+ const { x, y } = tap.point
1281
189
  const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId, timeoutMs })
1282
190
 
1283
191
  if (!tapResult.success) {
@@ -1298,916 +206,17 @@ export class ToolsInteract {
1298
206
  })
1299
207
  }
1300
208
 
1301
- static async adjustControlHandler({
1302
- selector,
1303
- element_id,
1304
- property = 'value',
1305
- targetValue,
1306
- tolerance,
1307
- maxAttempts,
1308
- platform,
1309
- deviceId
1310
- }: {
1311
- selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
1312
- element_id?: string,
1313
- property?: string,
1314
- targetValue: number,
1315
- tolerance?: number,
1316
- maxAttempts?: number,
1317
- platform?: 'android' | 'ios',
1318
- deviceId?: string
1319
- }): Promise<AdjustControlResponse> {
1320
- const actionType = 'adjust_control'
1321
- const targetSelector = selector ?? (element_id ? { elementId: element_id } : null)
1322
- const requestedTolerance = typeof tolerance === 'number' && Number.isFinite(tolerance) ? Math.max(0, tolerance) : null
1323
- const attemptsLimit = maxAttempts === undefined || maxAttempts === null
1324
- ? 5
1325
- : Math.max(1, Math.floor(Number(maxAttempts) || 1))
1326
- const sourcePlatform: 'android' | 'ios' = platform || 'android'
1327
- let resolvedPlatform = sourcePlatform
1328
- let resolvedDeviceId = deviceId
1329
- const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null
1330
- const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1331
- let semanticFallbackElement: FindElementResponse['element'] | null = null
1332
- const traceSteps: TraceStep[] = []
1333
- let traceAttemptIndex = 0
1334
-
1335
- const recordTraceStep = (
1336
- stage: TraceStep['stage'],
1337
- result: TraceStep['result'],
1338
- metadata?: Record<string, unknown>
1339
- ) => {
1340
- traceSteps.push(createTraceStep({
1341
- stage,
1342
- timestamp: Date.now(),
1343
- result,
1344
- attemptIndex: traceAttemptIndex++,
1345
- metadata
1346
- }))
1347
- }
1348
-
1349
- const buildFailure = (
1350
- failureCode: ActionFailureCode,
1351
- reason: string,
1352
- resolved: ActionTargetResolved | null,
1353
- device: any,
1354
- actualState: { property: string; value: number | null; raw_value?: number | null } | null,
1355
- attempts: number,
1356
- adjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture',
1357
- retryable = false,
1358
- uiFingerprintAfter: string | null = null
1359
- ): AdjustControlResponse => {
1360
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1361
- recordTraceStep('resolve', 'failure',
1362
- {
1363
- reason,
1364
- failure_code: failureCode
1365
- })
1366
- }
1367
- if (!traceSteps.some((step) => step.stage === 'recover')) {
1368
- recordTraceStep('recover', retryable ? 'retry' : 'failure', {
1369
- reason,
1370
- failure_code: failureCode,
1371
- retry_allowed: retryable,
1372
- recovery_attempts: attempts,
1373
- retry_depth: attempts
1374
- })
1375
- }
1376
- const responseTolerance = requestedTolerance ?? 0
1377
- const base = buildActionExecutionResult({
1378
- actionType,
1379
- sourceModule: 'interact',
1380
- device,
1381
- selector: targetSelector,
1382
- resolved,
1383
- success: false,
1384
- uiFingerprintBefore: fingerprintBefore,
1385
- uiFingerprintAfter,
1386
- failure: { failureCode, retryable },
1387
- details: {
1388
- target_value: targetValue,
1389
- tolerance: responseTolerance,
1390
- property,
1391
- attempts,
1392
- adjustment_mode: adjustmentMode,
1393
- actual_state: actualState,
1394
- converged: false,
1395
- within_tolerance: false,
1396
- reason
1397
- },
1398
- traceSteps
1399
- }) as AdjustControlResponse
1400
-
1401
- return {
1402
- ...base,
1403
- target_state: {
1404
- property,
1405
- target_value: targetValue,
1406
- tolerance: responseTolerance
1407
- },
1408
- actual_state: actualState,
1409
- within_tolerance: false,
1410
- converged: false,
1411
- attempts,
1412
- adjustment_mode: adjustmentMode
1413
- }
1414
- }
1415
-
1416
- const resolveCurrentMatch = async (): Promise<{
1417
- tree: any
1418
- device: any
1419
- match: { el: UiElement, idx: number } | null
1420
- resolvedTarget: ActionTargetResolved | null
1421
- } | null> => {
1422
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
1423
- resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform
1424
- resolvedDeviceId = tree?.device?.id || resolvedDeviceId
1425
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1426
-
1427
- if (element_id) {
1428
- const stored = ToolsInteract._resolvedUiElements.get(element_id)
1429
- if (!stored) {
1430
- return null
1431
- }
1432
- const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored)
1433
- if (!current) {
1434
- return null
1435
- }
1436
- return {
1437
- tree,
1438
- device: tree?.device,
1439
- match: { el: current.el, idx: current.index },
1440
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1441
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index),
1442
- current.el,
1443
- current.index
1444
- )
1445
- }
1446
- }
1447
-
1448
- if (semanticFallbackElement) {
1449
- const fallbackBounds = ToolsInteract._normalizeBounds(
1450
- Array.isArray(semanticFallbackElement.bounds)
1451
- ? semanticFallbackElement.bounds
1452
- : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
1453
- ? [
1454
- Number((semanticFallbackElement.bounds as any).left),
1455
- Number((semanticFallbackElement.bounds as any).top),
1456
- Number((semanticFallbackElement.bounds as any).right),
1457
- Number((semanticFallbackElement.bounds as any).bottom)
1458
- ]
1459
- : null
1460
- )
1461
-
1462
- let matchedIndex = -1
1463
- if (fallbackBounds) {
1464
- matchedIndex = elements.findIndex((el) => {
1465
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
1466
- return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3]
1467
- })
1468
- }
1469
-
1470
- if (matchedIndex === -1 && fallbackBounds) {
1471
- const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2)
1472
- const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2)
1473
- let bestDistance = Infinity
1474
- for (let i = 0; i < elements.length; i++) {
1475
- const el = elements[i]
1476
- if (!ToolsInteract._isAdjustableControl(el)) continue
1477
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
1478
- if (!bounds) continue
1479
- const centerX = Math.floor((bounds[0] + bounds[2]) / 2)
1480
- const centerY = Math.floor((bounds[1] + bounds[3]) / 2)
1481
- const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY)
1482
- if (distance < bestDistance) {
1483
- bestDistance = distance
1484
- matchedIndex = i
1485
- }
1486
- }
1487
- }
1488
-
1489
- if (matchedIndex >= 0 && elements[matchedIndex]) {
1490
- const matched = { el: elements[matchedIndex], idx: matchedIndex }
1491
- return {
1492
- tree,
1493
- device: tree?.device,
1494
- match: matched,
1495
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1496
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
1497
- matched.el,
1498
- matched.idx
1499
- )
1500
- }
1501
- }
1502
- }
1503
-
1504
- if (selector) {
1505
- const matched = ToolsInteract._findFirstMatchingElement(elements, selector)
1506
- if (!matched) {
1507
- return null
1508
- }
1509
- return {
1510
- tree,
1511
- device: tree?.device,
1512
- match: matched,
1513
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1514
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
1515
- matched.el,
1516
- matched.idx
1517
- )
1518
- }
1519
- }
1520
-
1521
- return null
1522
- }
1523
-
1524
- if ((!selector && !element_id) || (selector && element_id)) {
1525
- return buildFailure('ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', null, undefined, null, 0, 'gesture', false)
1526
- }
1527
-
1528
- if (selector && !element_id) {
1529
- const waitResult = await ToolsInteract.waitForUIHandler({
1530
- selector,
1531
- condition: 'clickable',
1532
- timeout_ms: 5000,
1533
- poll_interval_ms: 300,
1534
- platform: resolvedPlatform,
1535
- deviceId: resolvedDeviceId
1536
- }) as any
1537
-
1538
- if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
1539
- const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
1540
- if (!semanticQuery) {
1541
- return buildFailure(
1542
- waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
1543
- waitResult?.error?.message ?? 'adjustable control not found',
1544
- null,
1545
- waitResult?.device,
1546
- null,
1547
- 0,
1548
- 'gesture',
1549
- waitResult?.error?.code === 'ELEMENT_NOT_FOUND'
1550
- )
1551
- }
1552
-
1553
- const fallback = await ToolsInteract.findElementHandler({
1554
- query: semanticQuery,
1555
- exact: false,
1556
- timeoutMs: 3000,
1557
- platform: resolvedPlatform,
1558
- deviceId: resolvedDeviceId
1559
- })
1560
-
1561
- if (!fallback.found || !fallback.element) {
1562
- return buildFailure(
1563
- 'ELEMENT_NOT_FOUND',
1564
- waitResult?.error?.message ?? 'adjustable control not found',
1565
- null,
1566
- waitResult?.device,
1567
- null,
1568
- 0,
1569
- 'gesture',
1570
- true
1571
- )
1572
- }
1573
-
1574
- semanticFallbackElement = fallback.element
1575
- } else {
1576
- element_id = waitResult.element.elementId
1577
- semanticFallbackElement = null
1578
- }
1579
- }
1580
-
1581
- let lastObservedState: { property: string; value: number | null; raw_value?: number | null } | null = null
1582
- let lastAdjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture'
1583
- let resolvedTarget: ActionTargetResolved | null = null
1584
- let currentDevice: any = undefined
1585
- let attemptCount = 0
1586
- let cachedResolvedMatch: { el: UiElement, idx: number } | null = null
1587
- let coordinateTapsAwaitingFreshGesture = false
1588
-
1589
- adjustmentAttempts:
1590
- for (let attempt = 0; attempt < attemptsLimit && attemptCount < attemptsLimit; attempt++) {
1591
- const resolved: {
1592
- tree: any
1593
- device: any
1594
- match: { el: UiElement, idx: number } | null
1595
- resolvedTarget: ActionTargetResolved | null
1596
- } | null = cachedResolvedMatch
1597
- ? {
1598
- tree: null,
1599
- device: currentDevice,
1600
- match: cachedResolvedMatch,
1601
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1602
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx),
1603
- cachedResolvedMatch.el,
1604
- cachedResolvedMatch.idx
1605
- )
1606
- }
1607
- : await resolveCurrentMatch()
1608
- if (!resolved || !resolved.match || !resolved.resolvedTarget) {
1609
- return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true)
1610
- }
1611
-
1612
- currentDevice = resolved.device
1613
- resolvedTarget = resolved.resolvedTarget
1614
- const currentEl: UiElement = resolved.match.el
1615
- cachedResolvedMatch = resolved.match
1616
-
1617
- const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id)
1618
- if (!adjustableActionability.actionable) {
1619
- return buildFailure(
1620
- adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
1621
- adjustableActionability.reason ?? 'adjustable control is not actionable',
1622
- resolvedTarget,
1623
- currentDevice,
1624
- lastObservedState,
1625
- attemptCount,
1626
- lastAdjustmentMode,
1627
- true
1628
- )
1629
- }
1630
-
1631
- const elementsForInference = Array.isArray(resolved.tree?.elements)
1632
- ? resolved.tree.elements as UiElement[]
1633
- : cachedResolvedMatch
1634
- ? [cachedResolvedMatch.el]
1635
- : [currentEl]
1636
- const inferredSliderState = ToolsInteract._inferLabeledSliderState(elementsForInference, currentEl)
1637
- const nativeBounds = ToolsInteract._normalizeBounds(currentEl.bounds)
1638
- const bounds = inferredSliderState?.touch_bounds ?? nativeBounds
1639
- const valueRange = ToolsInteract._readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null
1640
- const step = ToolsInteract._readControlStep(currentEl)
1641
- const normalizedTargetValue = valueRange
1642
- ? ToolsInteract._quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step)
1643
- : targetValue
1644
- const effectiveTolerance = requestedTolerance ?? ToolsInteract._defaultAdjustTolerance(valueRange, step)
1645
- const currentValue = ToolsInteract._readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null
1646
- const actualState = currentValue !== null
1647
- ? {
1648
- property,
1649
- value: currentValue,
1650
- raw_value: typeof currentEl.state?.raw_value === 'number'
1651
- ? currentEl.state.raw_value
1652
- : inferredSliderState?.current
1653
- }
1654
- : null
1655
-
1656
- lastObservedState = actualState
1657
-
1658
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1659
- recordTraceStep('resolve', 'success', {
1660
- resolved_target: resolvedTarget,
1661
- current_value: currentValue,
1662
- adjustment_mode: lastAdjustmentMode
1663
- })
1664
- }
1665
-
1666
- if (property !== 'value' && property !== 'raw_value') {
1667
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1668
- }
1669
-
1670
- const isAdjustableControl = ToolsInteract._isAdjustableControl(currentEl) || !!inferredSliderState
1671
- if (!isAdjustableControl) {
1672
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1673
- }
1674
-
1675
- if (currentValue === null) {
1676
- return buildFailure('UNKNOWN', 'current numeric value is unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1677
- }
1678
-
1679
- if (valueRange && (targetValue < valueRange.min || targetValue > valueRange.max)) {
1680
- return buildFailure('CONTROL_CONVERGENCE_FAILED', `targetValue ${targetValue} is outside the control range ${valueRange.min}..${valueRange.max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1681
- }
1682
-
1683
- if (currentValue !== null && Math.abs(currentValue - normalizedTargetValue) <= effectiveTolerance) {
1684
- recordTraceStep('verify', 'success', {
1685
- property,
1686
- target_value: normalizedTargetValue,
1687
- actual_state: actualState,
1688
- reason: 'control already within tolerance'
1689
- })
1690
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1691
- const base = buildActionExecutionResult({
1692
- actionType,
1693
- sourceModule: 'interact',
1694
- device: currentDevice,
1695
- selector: targetSelector,
1696
- resolved: resolvedTarget,
1697
- success: true,
1698
- uiFingerprintBefore: fingerprintBefore,
1699
- uiFingerprintAfter,
1700
- details: {
1701
- target_value: normalizedTargetValue,
1702
- tolerance: effectiveTolerance,
1703
- property,
1704
- attempts: attemptCount,
1705
- adjustment_mode: 'semantic',
1706
- actual_state: actualState,
1707
- converged: true,
1708
- within_tolerance: true,
1709
- reason: 'control already within tolerance'
1710
- },
1711
- traceSteps
1712
- }) as AdjustControlResponse
1713
-
1714
- return {
1715
- ...base,
1716
- target_state: {
1717
- property,
1718
- target_value: normalizedTargetValue,
1719
- tolerance: effectiveTolerance
1720
- },
1721
- actual_state: actualState,
1722
- within_tolerance: true,
1723
- converged: true,
1724
- attempts: attemptCount,
1725
- adjustment_mode: 'semantic'
1726
- }
1727
- }
1728
-
1729
- if (!bounds) {
1730
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1731
- }
1732
-
1733
- const runVerification = async (): Promise<{
1734
- verification: any
1735
- observedState: { property: string; value: number | null; raw_value?: number | null } | null
1736
- withinTolerance: boolean
1737
- }> => {
1738
- if (inferredSliderState) {
1739
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
1740
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1741
- const refreshed = ToolsInteract._inferLabeledSliderState(elements, currentEl)
1742
- const observedValue = refreshed?.current ?? null
1743
- const observedState = observedValue !== null
1744
- ? { property, value: observedValue, raw_value: observedValue }
1745
- : actualState
1746
-
1747
- return {
1748
- verification: {
1749
- success: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance,
1750
- observed_state: observedState,
1751
- reason: observedValue !== null
1752
- ? 'visible labeled value read from nearby slider labels'
1753
- : 'visible labeled value unavailable'
1754
- },
1755
- observedState,
1756
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1757
- }
1758
- }
1759
-
1760
- const verification = await ToolsInteract.expectStateHandler({
1761
- element_id: resolvedTarget?.elementId ?? element_id,
1762
- selector: selector ?? undefined,
1763
- property,
1764
- expected: normalizedTargetValue,
1765
- platform: resolvedPlatform,
1766
- deviceId: resolvedDeviceId
1767
- }) as any
1768
-
1769
- const observedValue = ToolsInteract._parseNumericControlValue(verification?.observed_state?.value) ??
1770
- ToolsInteract._parseNumericControlValue(verification?.observed_state?.raw_value)
1771
- const observedState = observedValue !== null
1772
- ? {
1773
- property,
1774
- value: observedValue,
1775
- raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
1776
- }
1777
- : actualState
1778
-
1779
- return {
1780
- verification,
1781
- observedState,
1782
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1783
- }
1784
- }
1785
-
1786
- let actionDevice: any = currentDevice
1787
- let observedState: { property: string; value: number | null; raw_value?: number | null } | null = actualState
1788
- let verification: any = null
1789
- let verificationResult: any = { verification: null, observedState: actualState, withinTolerance: false }
1790
-
1791
- const directActionSupported = ToolsInteract._hasDirectAdjustmentAction(currentEl)
1792
- if (directActionSupported && attemptCount < attemptsLimit) {
1793
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId)
1794
- if (typeof interact.setAdjustableValue === 'function') {
1795
- lastAdjustmentMode = 'semantic'
1796
- recordTraceStep('execute', 'retry', {
1797
- attempt: attemptCount + 1,
1798
- mode: 'semantic',
1799
- strategy: 'direct_set_value',
1800
- target_value: normalizedTargetValue
1801
- })
1802
- const actionResult = await interact.setAdjustableValue({
1803
- deviceId: actionResolved.id,
1804
- target: resolvedTarget,
1805
- property,
1806
- value: normalizedTargetValue
1807
- })
1808
- attemptCount++
1809
- actionDevice = actionResult?.device ?? actionDevice
1810
-
1811
- if (actionResult?.success !== false) {
1812
- verificationResult = await runVerification()
1813
- observedState = verificationResult.observedState
1814
- lastObservedState = observedState
1815
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1816
- attempt: attemptCount,
1817
- property,
1818
- target_value: normalizedTargetValue,
1819
- actual_state: observedState,
1820
- reason: verificationResult.verification?.reason ?? 'direct adjustment did not converge yet'
1821
- })
1822
-
1823
- if (verificationResult.withinTolerance) {
1824
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1825
- const base = buildActionExecutionResult({
1826
- actionType,
1827
- sourceModule: 'interact',
1828
- device: actionDevice ?? currentDevice,
1829
- selector: targetSelector,
1830
- resolved: resolvedTarget,
1831
- success: true,
1832
- uiFingerprintBefore: fingerprintBefore,
1833
- uiFingerprintAfter,
1834
- details: {
1835
- target_value: normalizedTargetValue,
1836
- tolerance: effectiveTolerance,
1837
- property,
1838
- attempts: attemptCount,
1839
- adjustment_mode: lastAdjustmentMode,
1840
- actual_state: observedState,
1841
- converged: true,
1842
- within_tolerance: true,
1843
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1844
- },
1845
- traceSteps
1846
- }) as AdjustControlResponse
1847
-
1848
- return {
1849
- ...base,
1850
- target_state: {
1851
- property,
1852
- target_value: normalizedTargetValue,
1853
- tolerance: effectiveTolerance
1854
- },
1855
- actual_state: observedState,
1856
- within_tolerance: true,
1857
- converged: true,
1858
- attempts: attemptCount,
1859
- adjustment_mode: lastAdjustmentMode
1860
- }
1861
- }
1862
-
1863
- cachedResolvedMatch = null
1864
- if (attemptCount < attemptsLimit) {
1865
- continue
1866
- }
1867
- }
1868
- }
1869
- }
1870
-
1871
- const directions = ToolsInteract._adjustmentDirections(currentEl)
1872
- if ((directions.increment || directions.decrement) && attemptCount < attemptsLimit) {
1873
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId)
1874
- const adjustByDirection = typeof interact.adjustAccessibleValue === 'function'
1875
- ? interact.adjustAccessibleValue.bind(interact)
1876
- : null
1877
- while (adjustByDirection && attemptCount < attemptsLimit) {
1878
- const direction = normalizedTargetValue > (observedState?.value ?? currentValue) ? 'increment' : 'decrement'
1879
- if ((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement)) break
1880
- lastAdjustmentMode = 'semantic'
1881
- recordTraceStep('execute', 'retry', {
1882
- attempt: attemptCount + 1,
1883
- mode: 'semantic',
1884
- strategy: 'accessibility_increment_decrement',
1885
- direction
1886
- })
1887
- const actionResult = await adjustByDirection({
1888
- deviceId: actionResolved.id,
1889
- target: resolvedTarget,
1890
- property,
1891
- direction
1892
- })
1893
- attemptCount++
1894
- actionDevice = actionResult?.device ?? actionDevice
1895
- if (actionResult?.success === false) break
1896
-
1897
- verificationResult = await runVerification()
1898
- observedState = verificationResult.observedState
1899
- lastObservedState = observedState
1900
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1901
- attempt: attemptCount,
1902
- property,
1903
- target_value: normalizedTargetValue,
1904
- actual_state: observedState,
1905
- reason: verificationResult.verification?.reason ?? 'incremental adjustment did not converge yet'
1906
- })
1907
-
1908
- if (verificationResult.withinTolerance) {
1909
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1910
- const base = buildActionExecutionResult({
1911
- actionType,
1912
- sourceModule: 'interact',
1913
- device: actionDevice ?? currentDevice,
1914
- selector: targetSelector,
1915
- resolved: resolvedTarget,
1916
- success: true,
1917
- uiFingerprintBefore: fingerprintBefore,
1918
- uiFingerprintAfter,
1919
- details: {
1920
- target_value: normalizedTargetValue,
1921
- tolerance: effectiveTolerance,
1922
- property,
1923
- attempts: attemptCount,
1924
- adjustment_mode: lastAdjustmentMode,
1925
- actual_state: observedState,
1926
- converged: true,
1927
- within_tolerance: true,
1928
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1929
- },
1930
- traceSteps
1931
- }) as AdjustControlResponse
1932
-
1933
- return {
1934
- ...base,
1935
- target_state: {
1936
- property,
1937
- target_value: normalizedTargetValue,
1938
- tolerance: effectiveTolerance
1939
- },
1940
- actual_state: observedState,
1941
- within_tolerance: true,
1942
- converged: true,
1943
- attempts: attemptCount,
1944
- adjustment_mode: lastAdjustmentMode
1945
- }
1946
- }
1947
-
1948
- cachedResolvedMatch = null
1949
- if (attemptCount < attemptsLimit) {
1950
- continue adjustmentAttempts
1951
- }
1952
- }
1953
- }
1954
-
1955
- if (!bounds) {
1956
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1957
- }
1958
-
1959
- if (!valueRange) {
1960
- return buildFailure('UNKNOWN', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1961
- }
1962
-
1963
- const min = valueRange.min
1964
- const max = valueRange.max
1965
- const axis = ToolsInteract._controlAxis(currentEl, bounds)
1966
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, normalizedTargetValue, currentValue, min, max, axis)
1967
- const currentPoint = ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
1968
- if (!coordinateTapsAwaitingFreshGesture) {
1969
- const probePoints = inferredSliderState
1970
- ? [targetPoint]
1971
- : ToolsInteract._buildAdjustmentProbePoints(bounds, normalizedTargetValue, currentValue, min, max, axis)
1972
-
1973
- for (let i = 0; i < probePoints.length && attemptCount < attemptsLimit; i++) {
1974
- const probePoint = probePoints[i]
1975
- lastAdjustmentMode = 'coordinate'
1976
- recordTraceStep('execute', 'retry', {
1977
- attempt: attemptCount + 1,
1978
- mode: 'coordinate',
1979
- point: probePoint
1980
- })
1981
- const actionResult = await ToolsInteract.tapHandler({
1982
- platform: resolvedPlatform,
1983
- x: probePoint.x,
1984
- y: probePoint.y,
1985
- deviceId: resolvedDeviceId
1986
- })
1987
- attemptCount++
1988
- actionDevice = actionResult.device ?? actionDevice
1989
-
1990
- if (!actionResult.success) {
1991
- recordTraceStep('execute', 'retry', {
1992
- attempt: attemptCount,
1993
- mode: 'coordinate',
1994
- point: probePoint,
1995
- success: false
1996
- })
1997
- continue
1998
- }
1999
-
2000
- verificationResult = await runVerification()
2001
- observedState = verificationResult.observedState
2002
- lastObservedState = observedState
2003
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
2004
- attempt: attemptCount,
2005
- property,
2006
- target_value: normalizedTargetValue,
2007
- actual_state: observedState,
2008
- reason: verificationResult.verification?.reason ?? 'control did not converge yet'
2009
- })
2010
-
2011
- if (verificationResult.withinTolerance) {
2012
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2013
- const base = buildActionExecutionResult({
2014
- actionType,
2015
- sourceModule: 'interact',
2016
- device: actionDevice ?? currentDevice,
2017
- selector: targetSelector,
2018
- resolved: resolvedTarget,
2019
- success: true,
2020
- uiFingerprintBefore: fingerprintBefore,
2021
- uiFingerprintAfter,
2022
- details: {
2023
- target_value: normalizedTargetValue,
2024
- tolerance: effectiveTolerance,
2025
- property,
2026
- attempts: attemptCount,
2027
- adjustment_mode: lastAdjustmentMode,
2028
- actual_state: observedState,
2029
- converged: true,
2030
- within_tolerance: true,
2031
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
2032
- },
2033
- traceSteps
2034
- }) as AdjustControlResponse
2035
-
2036
- return {
2037
- ...base,
2038
- target_state: {
2039
- property,
2040
- target_value: normalizedTargetValue,
2041
- tolerance: effectiveTolerance
2042
- },
2043
- actual_state: observedState,
2044
- within_tolerance: true,
2045
- converged: true,
2046
- attempts: attemptCount,
2047
- adjustment_mode: lastAdjustmentMode
2048
- }
2049
- }
2050
-
2051
- coordinateTapsAwaitingFreshGesture = true
2052
- cachedResolvedMatch = null
2053
- if (attemptCount < attemptsLimit) {
2054
- continue adjustmentAttempts
2055
- }
2056
- }
2057
- }
2058
-
2059
- if (attemptCount < attemptsLimit) {
2060
- coordinateTapsAwaitingFreshGesture = false
2061
- lastAdjustmentMode = 'gesture'
2062
- recordTraceStep('execute', 'retry', {
2063
- attempt: attemptCount + 1,
2064
- mode: 'gesture',
2065
- start: currentPoint,
2066
- end: targetPoint
2067
- })
2068
- const fallbackActionResult = await ToolsInteract.swipeHandler({
2069
- platform: resolvedPlatform,
2070
- x1: currentPoint.x,
2071
- y1: currentPoint.y,
2072
- x2: targetPoint.x,
2073
- y2: targetPoint.y,
2074
- duration: inferredSliderState ? 520 : 220,
2075
- deviceId: resolvedDeviceId
2076
- })
2077
- attemptCount++
2078
- if (!fallbackActionResult.success) {
2079
- recordTraceStep('execute', 'failure', {
2080
- attempt: attemptCount,
2081
- mode: 'gesture',
2082
- start: currentPoint,
2083
- end: targetPoint,
2084
- success: false
2085
- })
2086
- return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
2087
- }
2088
-
2089
- actionDevice = fallbackActionResult.device ?? actionDevice
2090
- verificationResult = await runVerification()
2091
- observedState = verificationResult.observedState
2092
- lastObservedState = observedState
2093
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
2094
- attempt: attemptCount,
2095
- property,
2096
- target_value: normalizedTargetValue,
2097
- actual_state: observedState,
2098
- reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
2099
- })
2100
-
2101
- if (verificationResult.withinTolerance) {
2102
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2103
- const base = buildActionExecutionResult({
2104
- actionType,
2105
- sourceModule: 'interact',
2106
- device: actionDevice ?? currentDevice,
2107
- selector: targetSelector,
2108
- resolved: resolvedTarget,
2109
- success: true,
2110
- uiFingerprintBefore: fingerprintBefore,
2111
- uiFingerprintAfter,
2112
- details: {
2113
- target_value: normalizedTargetValue,
2114
- tolerance: effectiveTolerance,
2115
- property,
2116
- attempts: attemptCount,
2117
- adjustment_mode: lastAdjustmentMode,
2118
- actual_state: observedState,
2119
- converged: true,
2120
- within_tolerance: true,
2121
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
2122
- },
2123
- traceSteps
2124
- }) as AdjustControlResponse
2125
-
2126
- return {
2127
- ...base,
2128
- target_state: {
2129
- property,
2130
- target_value: normalizedTargetValue,
2131
- tolerance: effectiveTolerance
2132
- },
2133
- actual_state: observedState,
2134
- within_tolerance: true,
2135
- converged: true,
2136
- attempts: attemptCount,
2137
- adjustment_mode: lastAdjustmentMode
2138
- }
2139
- }
2140
- }
2141
-
2142
- verification = verificationResult.verification
2143
- lastObservedState = observedState
2144
-
2145
- if (verificationResult.withinTolerance) {
2146
- recordTraceStep('verify', 'success', {
2147
- attempt: attemptCount,
2148
- property,
2149
- target_value: normalizedTargetValue,
2150
- actual_state: observedState,
2151
- reason: verification?.reason ?? 'control converged to target value'
2152
- })
2153
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2154
- const base = buildActionExecutionResult({
2155
- actionType,
2156
- sourceModule: 'interact',
2157
- device: actionDevice ?? currentDevice,
2158
- selector: targetSelector,
2159
- resolved: resolvedTarget,
2160
- success: true,
2161
- uiFingerprintBefore: fingerprintBefore,
2162
- uiFingerprintAfter,
2163
- details: {
2164
- target_value: normalizedTargetValue,
2165
- tolerance: effectiveTolerance,
2166
- property,
2167
- attempts: attemptCount,
2168
- adjustment_mode: lastAdjustmentMode,
2169
- actual_state: observedState,
2170
- converged: true,
2171
- within_tolerance: true,
2172
- reason: verification?.reason ?? 'control converged to target value'
2173
- },
2174
- traceSteps
2175
- }) as AdjustControlResponse
2176
-
2177
- return {
2178
- ...base,
2179
- target_state: {
2180
- property,
2181
- target_value: normalizedTargetValue,
2182
- tolerance: effectiveTolerance
2183
- },
2184
- actual_state: observedState,
2185
- within_tolerance: true,
2186
- converged: true,
2187
- attempts: attemptCount,
2188
- adjustment_mode: lastAdjustmentMode
2189
- }
2190
- }
2191
-
2192
- cachedResolvedMatch = inferredSliderState
2193
- ? null
2194
- : {
2195
- el: {
2196
- ...currentEl,
2197
- state: {
2198
- ...(currentEl.state ?? null),
2199
- ...(observedState ? {
2200
- [observedState.property]: observedState.value,
2201
- raw_value: observedState.raw_value ?? observedState.value
2202
- } : {})
2203
- }
2204
- },
2205
- idx: resolved.match.idx
2206
- }
2207
- }
2208
-
2209
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2210
- return buildFailure('CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter)
209
+ static async adjustControlHandler(input: AdjustmentInput): Promise<AdjustControlResponse> {
210
+ return adjustControl(input, {
211
+ observe: (platform, deviceId) => ToolsObserve.getUITreeHandler({ platform, deviceId }),
212
+ fingerprint: (platform, deviceId) => ToolsInteract._captureFingerprint(platform, deviceId),
213
+ getHandle: id => ToolsInteract._resolvedUiElements.get(id),
214
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
215
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx),
216
+ wait: input => ToolsInteract.waitForUIHandler(input), find: input => ToolsInteract.findElementHandler(input),
217
+ expectState: input => ToolsInteract.expectStateHandler(input), service: (platform, deviceId) => ToolsInteract.getInteractionService(platform, deviceId),
218
+ tap: input => ToolsInteract.tapHandler(input), swipe: input => ToolsInteract.swipeHandler(input)
219
+ })
2211
220
  }
2212
221
 
2213
222
  static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId, timeoutMs }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string, timeoutMs?: number }) {
@@ -2230,1454 +239,40 @@ export class ToolsInteract {
2230
239
  return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id)
2231
240
  }
2232
241
 
2233
- static async findElementHandler({ query, exact = false, timeoutMs = 3000, platform, deviceId }: { query: string, exact?: boolean, timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string }): Promise<FindElementResponse> {
2234
- // Try to use observe layer to fetch the current UI tree and perform a fast semantic search
2235
- const start = Date.now()
2236
- const deadline = start + timeoutMs
2237
- const normalize = ToolsInteract._normalize
2238
-
2239
- const q = normalize(query)
2240
- if (!q) return { found: false, error: 'Empty query' }
2241
-
2242
- let best: RankedResolutionCandidate | null = null
2243
- let bestTree: any = null
2244
- let bestIterationCandidates: RankedResolutionCandidate[] = []
2245
- let shouldStop = false
2246
-
2247
- const scoreElement = (el: UiElement | null, idx: number): RankedResolutionCandidate | null => {
2248
- if (!el || !el.visible) return null
2249
- const bounds = el.bounds || [0,0,0,0]
2250
- if (!Array.isArray(bounds) || bounds.length < 4) return null
2251
- const [l,t,r,b] = bounds
2252
- if (r <= l || b <= t) return null
2253
- // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
2254
- const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el))
2255
-
2256
- const text = normalize(el.text ?? el.label ?? el.value ?? '')
2257
- const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '')
2258
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
2259
- const className = normalize(el.type ?? el.class ?? '')
2260
- const semanticRole = normalize(el.semantic?.semantic_role ?? '')
2261
- const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : []
2262
-
2263
- let score = 0
2264
- let reason = 'best_scoring_candidate'
2265
- if (exact) {
2266
- if (text && text === q) {
2267
- score = 1.0
2268
- reason = 'exact_text_match'
2269
- } else if (content && content === q) {
2270
- score = 0.95
2271
- reason = 'exact_content_desc_match'
2272
- } else if (resourceId && resourceId === q) {
2273
- score = 0.92
2274
- reason = 'exact_resource_id_match'
2275
- } else if (className && className === q) {
2276
- score = 0.3
2277
- reason = 'exact_class_match'
2278
- }
2279
- } else {
2280
- if (text && text === q) {
2281
- score = 1.0
2282
- reason = 'exact_text_match'
2283
- } else if (content && content === q) {
2284
- score = 0.95
2285
- reason = 'exact_content_desc_match'
2286
- } else if (resourceId && resourceId === q) {
2287
- score = 0.92
2288
- reason = 'exact_resource_id_match'
2289
- } else if (text && text.includes(q)) {
2290
- score = 0.6
2291
- reason = 'partial_text_match'
2292
- } else if (content && content.includes(q)) {
2293
- score = 0.55
2294
- reason = 'partial_content_desc_match'
2295
- } else if (resourceId && resourceId.includes(q)) {
2296
- score = 0.7
2297
- reason = 'partial_resource_id_match'
2298
- } else if (className && className.includes(q)) {
2299
- score = 0.3
2300
- reason = 'partial_class_match'
2301
- }
2302
- }
2303
- if (!exact) {
2304
- if (!score && semanticRole && semanticRole.includes(q)) {
2305
- score = 0.5
2306
- reason = 'semantic_role_match'
2307
- }
2308
- if (semanticActions.some((action) => action.includes(q))) {
2309
- score = Math.max(score, score > 0 ? 0.65 : 0.6)
2310
- reason = 'semantic_action_match'
2311
- }
2312
- if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
2313
- score = 0.45
2314
- reason = 'semantic_control_match'
2315
- }
2316
- } else {
2317
- if (!score && semanticRole && semanticRole === q) {
2318
- score = 0.5
2319
- reason = 'semantic_role_match'
2320
- }
2321
- if (semanticActions.some((action) => action === q)) {
2322
- score = Math.max(score, score > 0 ? 0.65 : 0.6)
2323
- reason = 'semantic_action_match'
2324
- }
2325
- }
2326
- if (score > 0 && interactable) score += 0.05
2327
- if (score <= 0) return null
2328
- return { el, idx, score, reason, interactable }
2329
- }
2330
-
2331
- while (Date.now() <= deadline) {
2332
- try {
2333
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
2334
- if (tree && Array.isArray((tree as any).elements)) {
2335
- const elements = ((tree as any).elements as UiElement[])
2336
- const iterationCandidates: RankedResolutionCandidate[] = []
2337
- let iterationImprovedBest = false
2338
- for (let i = 0; i < elements.length; i++) {
2339
- const el = elements[i]
2340
- try {
2341
- const candidate = scoreElement(el, i)
2342
- if (!candidate) continue
2343
- iterationCandidates.push(candidate)
2344
- if (!best || candidate.score > best.score) {
2345
- best = candidate
2346
- bestTree = tree
2347
- iterationImprovedBest = true
2348
- if (best.score >= 0.95) {
2349
- shouldStop = true
2350
- break
2351
- }
2352
- }
2353
- } catch (e) { checkDebuggerGuard(); console.error('Error scoring element:', e) }
2354
- }
2355
- if (iterationImprovedBest) {
2356
- bestIterationCandidates = iterationCandidates.slice()
2357
- }
2358
- }
2359
- } catch (e) { checkDebuggerGuard(); console.error('Error fetching UI tree:', e) }
2360
- if (shouldStop || Date.now() > deadline) break
2361
- await new Promise(r => setTimeout(r, 100))
2362
- }
2363
-
2364
- if (!best) return { found: false, error: 'Element not found' }
2365
-
2366
- // If the best match is not interactable, try to resolve an actionable ancestor.
2367
- try {
2368
- const elements = (bestTree && Array.isArray(bestTree.elements)) ? (bestTree.elements as UiElement[]) : []
2369
- const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution as UiResolution : null
2370
- let chosen = best as { el: UiElement, idx: number }
2371
- const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null
2372
-
2373
- // Strategy 1: if parentId references an index, climb that chain
2374
- let resolvedAncestor: { el: UiElement, idx: number } | null = null
2375
- if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
2376
- let cur = chosen
2377
- let safety = 0
2378
- while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
2379
- let pid = cur.el.parentId
2380
- let idx: number | null = null
2381
- if (typeof pid === 'number') idx = pid
2382
- else if (typeof pid === 'string' && /^\d+$/.test(pid)) idx = Number(pid)
2383
- // If parentId is not an index, try to find by matching resourceId or id field
2384
- if (idx !== null && elements[idx]) {
2385
- cur = { el: elements[idx], idx }
2386
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
2387
- } else if (typeof pid === 'string') {
2388
- // fallback: search elements for matching resourceId or id
2389
- const foundIndex = elements.findIndex((el: UiElement)=> (el.resourceId === pid || el.id === pid))
2390
- const found = foundIndex >= 0 ? elements[foundIndex] : null
2391
- if (found) {
2392
- cur = { el: found, idx: foundIndex }
2393
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
2394
- // otherwise continue climbing if this found element has its own parentId
2395
- } else {
2396
- break
2397
- }
2398
- } else {
2399
- break
2400
- }
2401
- safety++
2402
- }
2403
- }
2404
-
2405
- // Strategy 2: fallback - find a clickable element whose bounds fully contain the child's bounds
2406
- if (!resolvedAncestor && childBounds) {
2407
- const [cl,ct,cr,cb] = childBounds
2408
- // find candidates that are clickable and contain the child bounds
2409
- const candidates = elements
2410
- .map((el: UiElement, idx: number) => ({ el, idx }))
2411
- .filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds!.length >= 4)
2412
- let bestCandidate: { el: UiElement, idx: number } | null = null
2413
- let bestCandidateArea = Infinity
2414
- for (const c of candidates) {
2415
- const bounds = c.el.bounds as number[]
2416
- const [pl,pt,pr,pb] = bounds
2417
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
2418
- const area = (pr-pl) * (pb-pt)
2419
- if (area < bestCandidateArea) { bestCandidateArea = area; bestCandidate = c }
2420
- }
2421
- }
2422
- if (bestCandidate) resolvedAncestor = bestCandidate
2423
- }
2424
-
2425
- if (resolvedAncestor) {
2426
- best = {
2427
- el: resolvedAncestor.el,
2428
- idx: resolvedAncestor.idx,
2429
- score: Math.min(1, best.score + 0.02),
2430
- reason: 'clickable_parent_preferred',
2431
- interactable: true
2432
- }
2433
- }
2434
-
2435
- if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
2436
- const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen)
2437
- if (nearbyActionable) {
2438
- best = {
2439
- el: nearbyActionable.el,
2440
- idx: nearbyActionable.idx,
2441
- score: Math.min(1, best.score + 0.02),
2442
- reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
2443
- interactable: true
2444
- }
2445
- }
2446
- }
2447
- } catch (e) { checkDebuggerGuard(); console.error('Error resolving ancestor:', e) }
2448
-
2449
- if (!best) return { found: false, error: 'Element not found' }
2450
-
2451
- const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null
2452
- const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null
2453
- const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el))
2454
- const alternateCandidates = uniqueRanked
2455
- .filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
2456
- .slice(0, 3)
2457
- .map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate))
2458
-
2459
- const outEl = {
2460
- text: best.el.text ?? null,
2461
- resourceId: best.el.resourceId ?? null,
2462
- contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
2463
- class: best.el.type ?? best.el.class ?? null,
2464
- bounds: boundsObj,
2465
- clickable: !!best.el.clickable,
2466
- enabled: !!best.el.enabled,
2467
- stable_id: best.el.stable_id ?? null,
2468
- role: best.el.role ?? null,
2469
- test_tag: best.el.test_tag ?? null,
2470
- selector: best.el.selector ?? null,
2471
- semantic: best.el.semantic ?? null,
2472
- tapCoordinates,
2473
- telemetry: {
2474
- matchedIndex: best.idx ?? null,
2475
- matchedInteractable: !!best.interactable,
2476
- sliderLike: best.reason === 'slider_track_preferred'
2477
- }
2478
- }
2479
- if (best.reason === 'slider_track_preferred') {
2480
- const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left)
2481
- const interactionHint = {
2482
- kind: 'slider',
2483
- axis: isVertical ? 'vertical' : 'horizontal',
2484
- trackBounds: boundsObj
2485
- }
2486
- ;(outEl as any).interactionHint = interactionHint
2487
- }
2488
- const scoreVal = Math.min(1, Number(best.score.toFixed(3)))
2489
- const resolution: FindElementResolutionSummary = {
2490
- confidence: scoreVal,
2491
- reason: best.reason,
2492
- fallback_available: alternateCandidates.length > 0,
2493
- matched_count: uniqueRanked.length,
2494
- alternates: alternateCandidates
2495
- }
2496
- return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution }
242
+ static async findElementHandler(input: FindInput): Promise<FindElementResponse> {
243
+ return findElement(input, target => ToolsObserve.getUITreeHandler(target))
2497
244
  }
2498
245
 
2499
- static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false, resolveTapTarget = false }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string, singleObservation?: boolean, singleAttempt?: boolean, rejectAmbiguous?: boolean, resolveTapTarget?: boolean }) {
2500
- const readinessTracker = currentReadinessWait()
2501
- const overallStart = Date.now()
2502
- const requestedIndex = typeof match?.index === 'number' ? match.index : null
2503
- const requested = {
2504
- selector: selector ?? {},
2505
- condition,
2506
- match: requestedIndex === null ? null : { index: requestedIndex }
2507
- }
2508
-
2509
- // Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
2510
- const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
2511
- const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
2512
- const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
2513
-
2514
- if (!hasText && !hasResId && !hasAccId) {
2515
- return {
2516
- status: 'timeout',
2517
- error: {
2518
- code: 'INVALID_SELECTOR',
2519
- message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
2520
- },
2521
- metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
2522
- requested,
2523
- observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
2524
- };
2525
- }
2526
-
2527
- // Validate condition
2528
- if (!['exists','not_exists','visible','clickable'].includes(condition)) {
2529
- return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
2530
- }
2531
-
2532
- // Platform check
2533
- if (platform && !['android','ios'].includes(platform)) {
2534
- return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
2535
- }
2536
-
2537
- const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000))
2538
- const maxAttempts = (retry && retry.max_attempts) ? Math.max(1, retry.max_attempts) : 1
2539
- const backoff = (retry && retry.backoff_ms) ? Math.max(0, retry.backoff_ms) : 0
2540
-
2541
- let attempts = 0
2542
- let totalPollCount = 0
2543
- let lastMatchedCount = 0
2544
- let lastMatchedElement: ActionTargetResolved | null = null
2545
- let lastConditionSatisfied = false
2546
- let matchedAt: number | null = null
2547
- let stableMatchCount = 0
2548
- const stableObservationCount = singleObservation ? 1 : 2
2549
- const snapshotStaleThresholdMs = 500
2550
-
2551
- // Precompute normalized selector values and helpers (constant across polls)
2552
- const normalize = ToolsInteract._normalize
2553
- const containsFlag = !!selector?.contains
2554
- const selText = normalize(selector?.text)
2555
- const selRid = normalize(selector?.resource_id)
2556
- const selAid = normalize(selector?.accessibility_id)
2557
-
2558
- try {
2559
- while (attempts < maxAttempts) {
2560
- attempts++
2561
- const attemptStart = Date.now()
2562
- const deadline = attemptStart + (timeout_ms || 0)
2563
-
2564
- while (Date.now() <= deadline || (singleAttempt && totalPollCount === 0)) {
2565
- remainingBudget()
2566
- totalPollCount++
2567
- try {
2568
- const acquisitionStarted = Date.now()
2569
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
2570
- observeReadinessWait(tree, acquisitionStarted)
2571
- // A failed acquisition cannot establish that the requested tap target is absent.
2572
- if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error))) throw new Error('ACTION_TIMEOUT')
2573
- const elements = (tree && Array.isArray(tree.elements)) ? tree.elements as any[] : []
2574
-
2575
- const matches: { el: any, idx: number }[] = []
2576
-
2577
- for (let i = 0; i < elements.length; i++) {
2578
- const el = elements[i]
2579
- let ok = true
2580
-
2581
- // text
2582
- if (selector.text !== undefined && selector.text !== null) {
2583
- const val = normalize(el.text || el.label || el.value || '')
2584
- if (containsFlag) {
2585
- if (!val.includes(selText)) ok = false
2586
- } else {
2587
- if (val !== selText) ok = false
2588
- }
2589
- }
2590
-
2591
- // resource_id
2592
- if (ok && selector.resource_id !== undefined && selector.resource_id !== null) {
2593
- const rid = normalize(el.resourceId || el.resourceID || el.id || '')
2594
- if (containsFlag) {
2595
- if (!rid.includes(selRid)) ok = false
2596
- } else {
2597
- if (rid !== selRid) ok = false
2598
- }
2599
- }
2600
-
2601
- // accessibility_id
2602
- if (ok && selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
2603
- const aid = normalize(el.contentDescription || el.contentDesc || el.accessibilityLabel || el.label || '')
2604
- if (containsFlag) {
2605
- if (!aid.includes(selAid)) ok = false
2606
- } else {
2607
- if (aid !== selAid) ok = false
2608
- }
2609
- }
2610
-
2611
- if (ok) matches.push({ el, idx: i })
2612
- }
2613
-
2614
- // Evaluate condition
2615
- const matchedCount = matches.length
2616
- if (rejectAmbiguous && matchedCount > 1 && match?.index === undefined) return { status: 'timeout', error: { code: 'AMBIGUOUS_TARGET' } }
2617
- const pickIndex = (typeof match?.index === 'number') ? match!.index as number : undefined
2618
- let chosen: { el: any, idx: number } | null = null
2619
- if (matches.length > 0) {
2620
- if (pickIndex !== undefined) {
2621
- // If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
2622
- if (pickIndex >= 0 && pickIndex < matches.length) chosen = matches[pickIndex]
2623
- else chosen = null
2624
- } else {
2625
- chosen = matches[0]
2626
- }
2627
- } else {
2628
- chosen = null
2629
- }
2630
-
2631
- let conditionMet = false
2632
- let matchedElement = chosen
2633
- if (condition === 'exists') {
2634
- // when an index is specified, existence requires that specific index be present
2635
- conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1)
2636
- } else if (condition === 'not_exists') {
2637
- // when an index is specified, not_exists is true if that index is absent
2638
- conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0)
2639
- } else if (condition === 'visible') {
2640
- if (chosen) {
2641
- const b = chosen.el.bounds
2642
- const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
2643
- conditionMet = visibleFlag
2644
- } else conditionMet = false
2645
- } else if (condition === 'clickable') {
2646
- matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
2647
- ? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen as { el: UiElement, idx: number }) || chosen) : null
2648
- if (matchedElement) {
2649
- const b = matchedElement.el.bounds
2650
- const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
2651
- const enabled = !!matchedElement.el.enabled
2652
- const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable
2653
- conditionMet = visibleFlag && enabled && clickable
2654
- } else conditionMet = false
2655
- }
2656
-
2657
- // Waiting for a label to exist/be visible is distinct from resolving its
2658
- // executable control. Reuse the clickable-wait ancestor policy without
2659
- // promoting hidden or disabled labels into an unrelated enabled target.
2660
- if (resolveTapTarget && conditionMet && matchedElement &&
2661
- ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
2662
- matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement
2663
- }
2664
- const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
2665
- const resolvedDeviceId = tree?.device?.id || deviceId
2666
- lastMatchedCount = matchedCount
2667
- lastConditionSatisfied = conditionMet
2668
- lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
2669
- const now = Date.now()
2670
-
2671
- const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null
2672
- const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs
2673
-
2674
- if (conditionMet && snapshotFresh) {
2675
- if (matchedAt === null) matchedAt = now
2676
- stableMatchCount++
2677
- if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2678
- const latency_ms = now - overallStart
2679
- const outEl = lastMatchedElement
2680
-
2681
- return {
2682
- status: 'success',
2683
- ...(singleObservation ? { _tree: tree } : {}),
2684
- matched: matchedCount,
2685
- element: outEl,
2686
- metrics: { latency_ms, poll_count: totalPollCount, attempts },
2687
- requested,
2688
- observed: {
2689
- matched_count: matchedCount,
2690
- condition_satisfied: true,
2691
- selected_index: outEl?.index ?? null,
2692
- last_matched_element: outEl
2693
- }
2694
- }
2695
- }
2696
- } else {
2697
- stableMatchCount = 0
2698
- matchedAt = null
2699
- }
2700
-
2701
- } catch (e) { checkDebuggerGuard();
2702
- // Deadline expiry does not invalidate the last successfully observed state.
2703
- if (e instanceof Error && e.message === 'ACTION_TIMEOUT') throw e
2704
- if (readinessTracker && !String(e).includes('READINESS_')) readinessTracker.unavailable()
2705
- if (readinessTracker && isDeviceUnavailable(e)) throw e
2706
- // Non-fatal per-poll error; record and continue
2707
- if (!readinessTracker) console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e))
2708
- }
2709
-
2710
- if (singleAttempt) break
2711
- // Sleep until next poll
2712
- await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)))
2713
- }
2714
-
2715
- // Attempt timed out; if more attempts allowed, backoff then retry
2716
- if (attempts < maxAttempts) {
2717
- if (backoff > 0) await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)))
2718
- continue
2719
- }
2720
-
2721
- // Final failure for this call
2722
- const elapsed = Date.now() - overallStart
2723
- const observed = {
2724
- matched_count: lastMatchedCount,
2725
- condition_satisfied: lastConditionSatisfied,
2726
- selected_index: lastMatchedElement?.index ?? null,
2727
- last_matched_element: lastMatchedElement
2728
- }
2729
- const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
2730
- ? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
2731
- : ` observed ${lastMatchedCount} match(es)`
2732
- return {
2733
- status: 'timeout',
2734
- error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
2735
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2736
- requested,
2737
- observed
2738
- }
2739
- }
2740
-
2741
- } catch (err) { checkDebuggerGuard();
2742
- if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT') throw err
2743
- if (readinessTracker && isDeviceUnavailable(err)) throw err
2744
- const elapsed = Date.now() - overallStart
2745
- return {
2746
- status: 'timeout',
2747
- error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
2748
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2749
- requested,
2750
- observed: {
2751
- matched_count: lastMatchedCount,
2752
- condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
2753
- selected_index: lastMatchedElement?.index ?? null,
2754
- last_matched_element: lastMatchedElement
2755
- }
2756
- }
2757
- }
246
+ static async waitForUIHandler(input: UIWaitInput) {
247
+ return waitForUI(input, { observe: target => ToolsObserve.getUITreeHandler(target),
248
+ resolve: (platform, deviceId, element, index) => ToolsInteract._buildResolvedElement(platform, deviceId, element, index) })
2758
249
  }
2759
250
 
2760
- // Helper: normalize various log objects into plain message strings for comparison
2761
- private static _logsToMessages(logsArr: any[]): string[] {
2762
- if (!Array.isArray(logsArr)) return []
2763
- return logsArr.map((l: any) => {
2764
- if (typeof l === 'string') return l
2765
- if (l && (l.message || l.msg)) return l.message || l.msg
2766
- try { return JSON.stringify(l) } catch { return String(l) }
2767
- })
251
+ static async waitForScreenChangeHandler(input: ScreenWaitInput) {
252
+ return waitForScreenChange(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target) })
2768
253
  }
2769
254
 
2770
- static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }: { platform?: 'android' | 'ios', previousFingerprint: string, timeoutMs?: number, pollIntervalMs?: number, deviceId?: string }) {
2771
- const start = Date.now()
2772
- let lastFingerprint: string | null = null
2773
- let lastActivity: string | null = null
2774
-
2775
- while (Date.now() - start < timeoutMs) {
2776
- try {
2777
- const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
2778
- const fp = res?.fingerprint ?? null
2779
- lastActivity = (res as any)?.activity ?? lastActivity
2780
- if (fp === null || fp === undefined) {
2781
- lastFingerprint = null
2782
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
2783
- continue
2784
- }
2785
-
2786
- lastFingerprint = fp
2787
-
2788
- if (fp !== previousFingerprint) {
2789
- // Stability confirmation
2790
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
2791
- try {
2792
- const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
2793
- const confirmFp = confirmRes?.fingerprint ?? null
2794
- lastActivity = (confirmRes as any)?.activity ?? lastActivity
2795
- if (confirmFp === fp) {
2796
- return {
2797
- success: true,
2798
- previousFingerprint,
2799
- newFingerprint: fp,
2800
- elapsedMs: Date.now() - start,
2801
- observed_screen: {
2802
- fingerprint: fp,
2803
- activity: lastActivity
2804
- }
2805
- }
2806
- }
2807
- lastFingerprint = confirmFp
2808
- continue
2809
- } catch (e) { checkDebuggerGuard(); console.error('Error confirming fingerprint:', e); continue }
2810
- }
2811
- } catch (e) { checkDebuggerGuard(); console.error('Error getting screen fingerprint:', e) }
2812
-
2813
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
2814
- }
2815
-
2816
- return {
2817
- success: false,
2818
- reason: 'timeout',
2819
- previousFingerprint,
2820
- lastFingerprint,
2821
- elapsedMs: Date.now() - start,
2822
- observed_screen: {
2823
- fingerprint: lastFingerprint,
2824
- activity: lastActivity
2825
- }
2826
- }
255
+ static async waitForUIChangeHandler(input: ChangeInput): Promise<WaitForUIChangeResponse> {
256
+ return waitForUIChange(input, { observe: target => ToolsObserve.getUITreeHandler(target), scope: (tree, scope, target) => ToolsInteract._resolveUiChangeScope(tree, scope, target) })
2827
257
  }
2828
258
 
2829
- static async waitForUIChangeHandler({
2830
- platform,
2831
- deviceId,
2832
- timeout_ms = 60000,
2833
- stability_window_ms = 300,
2834
- expected_change,
2835
- scope = 'screen',
2836
- target = null
2837
- }: {
2838
- platform?: 'android' | 'ios',
2839
- deviceId?: string,
2840
- timeout_ms?: number,
2841
- stability_window_ms?: number,
2842
- expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change',
2843
- scope?: 'screen' | 'subtree',
2844
- target?: string | null
2845
- }): Promise<WaitForUIChangeResponse> {
2846
- const readinessTracker = currentReadinessWait()
2847
- const start = Date.now()
2848
- const pollIntervalMs = 300
2849
- const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
2850
- let baseline: UiChangeSignatureSet | null = null
2851
- let baselineScope: UiChangeScopeResult | null = null
2852
- let lastObservedRevision: number | null = null
2853
- let lastLoadingState: any = null
2854
- let lastSnapshotFreshnessMs: number | null = null
2855
- let candidateSignatures: UiChangeSignatureSet | null = null
2856
- let candidateObservedChange: 'hierarchy_diff' | 'text_change' | 'state_change' | null = null
2857
- let candidateSinceMs: number | null = null
2858
- let lastChangeSummary: ReturnType<typeof ToolsInteract._summarizeUiChangeDelta> | null = null
2859
- let lastScopeResolution: UiChangeScopeResolution = {
2860
- scope: scope === 'subtree' ? 'subtree' : 'screen',
2861
- target: target && typeof target === 'string' ? target : null,
2862
- resolved: scope !== 'subtree',
2863
- resolvedIndex: null,
2864
- resolvedStableId: null,
2865
- reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
2866
- }
2867
-
2868
- while (Date.now() - start < timeout_ms) {
2869
- try {
2870
- if (readinessTracker) remainingBudget()
2871
- const acquisitionStarted = Date.now()
2872
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
2873
- observeReadinessWait(tree, acquisitionStarted)
2874
- const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
2875
- if (scopedTree.error) {
2876
- lastScopeResolution = scopedTree.resolution
2877
- return {
2878
- success: false,
2879
- observed_change: null,
2880
- snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
2881
- snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
2882
- timeout: true,
2883
- elapsed_ms: Date.now() - start,
2884
- expected_change,
2885
- loading_state: tree?.loading_state ?? lastLoadingState ?? null,
2886
- scope: scopedTree.resolution.scope,
2887
- target: scopedTree.resolution.target,
2888
- stability_state: 'transient',
2889
- change_summary: lastChangeSummary,
2890
- reason: scopedTree.error.message,
2891
- error: scopedTree.error
2892
- }
2893
- }
2894
-
2895
- const scopedElements = scopedTree.elements
2896
- const scopedSignatureTree = {
2897
- ...tree,
2898
- elements: scopedElements
2899
- }
2900
- const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree)
2901
- lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
2902
- lastLoadingState = tree?.loading_state ?? lastLoadingState
2903
- lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs
2904
- lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary
2905
- lastScopeResolution = scopedTree.resolution
2906
- baselineScope = baselineScope ?? scopedTree
2907
-
2908
- if (!baseline) {
2909
- baseline = signatures
2910
- baselineScope = scopedTree
2911
- } else {
2912
- const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
2913
- if (observedChange) {
2914
- if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
2915
- candidateSignatures = signatures
2916
- candidateObservedChange = observedChange
2917
- candidateSinceMs = Date.now()
2918
- }
2919
-
2920
- const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
2921
- if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2922
- return {
2923
- success: true,
2924
- observed_change: candidateObservedChange ?? observedChange,
2925
- snapshot_revision: lastObservedRevision ?? undefined,
2926
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2927
- timeout: false,
2928
- elapsed_ms: Date.now() - start,
2929
- expected_change,
2930
- loading_state: lastLoadingState ?? null,
2931
- scope: lastScopeResolution.scope,
2932
- target: lastScopeResolution.target,
2933
- stability_state: 'stable',
2934
- change_summary: lastChangeSummary,
2935
- reason: 'UI change observed'
2936
- }
2937
- }
2938
- } else {
2939
- candidateSignatures = null
2940
- candidateObservedChange = null
2941
- candidateSinceMs = null
2942
- }
2943
- }
2944
- } catch (error) { checkDebuggerGuard();
2945
- if (readinessTracker && !String(error).includes('READINESS_')) readinessTracker.unavailable()
2946
- if (readinessTracker && isDeviceUnavailable(error)) throw error
2947
- if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) { readinessTracker.deadline(); break }
2948
- // Default waits retain best-effort polling.
2949
- }
2950
-
2951
- await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)))
2952
- }
2953
-
2954
- return {
2955
- success: false,
2956
- observed_change: null,
2957
- snapshot_revision: lastObservedRevision ?? undefined,
2958
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2959
- timeout: true,
2960
- elapsed_ms: Date.now() - start,
2961
- expected_change,
2962
- loading_state: lastLoadingState ?? null,
2963
- scope: lastScopeResolution.scope,
2964
- target: lastScopeResolution.target,
2965
- stability_state: 'transient',
2966
- change_summary: lastChangeSummary,
2967
- reason: 'timeout'
2968
- }
259
+ static async expectScreenHandler(input: ScreenExpectation): Promise<ExpectScreenResponse> {
260
+ return expectScreen(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), current: target => ToolsObserve.getCurrentScreenHandler(target) })
2969
261
  }
2970
262
 
2971
- static async expectScreenHandler({
2972
- platform,
2973
- fingerprint,
2974
- screen,
2975
- deviceId
2976
- }: {
2977
- platform?: 'android' | 'ios',
2978
- fingerprint?: string,
2979
- screen?: string,
2980
- deviceId?: string
2981
- }): Promise<ExpectScreenResponse> {
2982
- const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
2983
- const observedScreen = {
2984
- fingerprint: observedFingerprint?.fingerprint ?? null,
2985
- screen: observedFingerprint?.activity ?? null
2986
- }
2987
-
2988
- let observedScreenLabel = observedScreen.screen
2989
- if (!fingerprint && screen && platform !== 'ios') {
2990
- try {
2991
- const current = await ToolsObserve.getCurrentScreenHandler({ deviceId }) as any
2992
- observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel
2993
- } catch {
2994
- // Keep fingerprint-derived activity when current-screen lookup is unavailable.
2995
- }
2996
- }
2997
-
2998
- const expectedScreen = {
2999
- fingerprint: fingerprint ?? null,
3000
- screen: screen ?? null
3001
- }
3002
-
3003
- let success = false
3004
- let basis: 'fingerprint' | 'screen' | 'none' = 'none'
3005
- let reason = 'No fingerprint or screen expectation provided'
3006
- if (fingerprint) {
3007
- basis = 'fingerprint'
3008
- success = observedScreen.fingerprint === fingerprint
3009
- reason = success
3010
- ? `observed fingerprint matches expected fingerprint ${fingerprint}`
3011
- : `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`
3012
- } else if (screen) {
3013
- basis = 'screen'
3014
- const candidates = new Set<string>()
3015
- if (observedScreen.screen) candidates.add(observedScreen.screen)
3016
- if (observedScreenLabel) candidates.add(observedScreenLabel)
3017
- success = candidates.has(screen)
3018
- reason = success
3019
- ? `observed screen matches expected screen ${screen}`
3020
- : `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`
3021
- }
3022
-
3023
- return {
3024
- success,
3025
- observed_screen: {
3026
- fingerprint: observedScreen.fingerprint,
3027
- screen: observedScreenLabel
3028
- },
3029
- expected_screen: expectedScreen,
3030
- confidence: success ? 1 : 0,
3031
- comparison: {
3032
- basis,
3033
- matched: success,
3034
- reason
3035
- },
3036
- trace: buildObservationTrace({
3037
- actionType: 'expect_screen',
3038
- stage: 'verify',
3039
- success,
3040
- attempts: 1,
3041
- metadata: {
3042
- expected_screen: expectedScreen,
3043
- observed_screen: {
3044
- fingerprint: observedScreen.fingerprint,
3045
- screen: observedScreenLabel
3046
- },
3047
- comparison: {
3048
- basis,
3049
- matched: success
3050
- }
3051
- }
3052
- })
3053
- }
263
+ static async expectElementVisibleHandler(input: VisibilityInput): Promise<ExpectElementVisibleResponse> {
264
+ return expectVisible(input, request => ToolsInteract.waitForUIHandler(request))
3054
265
  }
3055
266
 
3056
- static async expectElementVisibleHandler({
3057
- selector,
3058
- element_id,
3059
- timeout_ms = 5000,
3060
- poll_interval_ms = 300,
3061
- platform,
3062
- deviceId
3063
- }: {
3064
- selector: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
3065
- element_id?: string,
3066
- timeout_ms?: number,
3067
- poll_interval_ms?: number,
3068
- platform?: 'android' | 'ios',
3069
- deviceId?: string
3070
- }): Promise<ExpectElementVisibleResponse> {
3071
- const result = await ToolsInteract.waitForUIHandler({
3072
- selector,
3073
- condition: 'visible',
3074
- timeout_ms,
3075
- poll_interval_ms,
3076
- platform,
3077
- deviceId
3078
- }) as any
3079
-
3080
- if (result?.status === 'success' && result?.element) {
3081
- return {
3082
- success: true,
3083
- selector,
3084
- element_id: result.element.elementId ?? element_id ?? null,
3085
- expected_condition: 'visible',
3086
- element: {
3087
- elementId: result.element.elementId ?? null,
3088
- text: result.element.text ?? null,
3089
- resource_id: result.element.resource_id ?? null,
3090
- accessibility_id: result.element.accessibility_id ?? null,
3091
- class: result.element.class ?? null,
3092
- bounds: result.element.bounds ?? null,
3093
- index: typeof result.element.index === 'number' ? result.element.index : null,
3094
- state: (result.element as any).state ?? null,
3095
- stable_id: (result.element as any).stable_id ?? null,
3096
- role: (result.element as any).role ?? null,
3097
- test_tag: (result.element as any).test_tag ?? null,
3098
- selector: (result.element as any).selector ?? null,
3099
- semantic: (result.element as any).semantic ?? null
3100
- },
3101
- observed: {
3102
- status: result.status,
3103
- matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
3104
- condition_satisfied: true,
3105
- selected_index: typeof result.element.index === 'number' ? result.element.index : null,
3106
- last_matched_element: {
3107
- elementId: result.element.elementId ?? null,
3108
- text: result.element.text ?? null,
3109
- resource_id: result.element.resource_id ?? null,
3110
- accessibility_id: result.element.accessibility_id ?? null,
3111
- class: result.element.class ?? null,
3112
- bounds: result.element.bounds ?? null,
3113
- index: typeof result.element.index === 'number' ? result.element.index : null,
3114
- state: (result.element as any).state ?? null,
3115
- stable_id: (result.element as any).stable_id ?? null,
3116
- role: (result.element as any).role ?? null,
3117
- test_tag: (result.element as any).test_tag ?? null,
3118
- selector: (result.element as any).selector ?? null,
3119
- semantic: (result.element as any).semantic ?? null
3120
- }
3121
- },
3122
- reason: 'selector is visible',
3123
- trace: buildObservationTrace({
3124
- actionType: 'expect_element_visible',
3125
- stage: 'verify',
3126
- success: true,
3127
- attempts: 1,
3128
- metadata: {
3129
- selector,
3130
- element_id: result.element.elementId ?? element_id ?? null,
3131
- status: result.status
3132
- }
3133
- })
3134
- }
3135
- }
3136
-
3137
- const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT'
3138
- return {
3139
- success: false,
3140
- selector,
3141
- element_id: element_id ?? null,
3142
- expected_condition: 'visible',
3143
- observed: {
3144
- status: result?.status,
3145
- matched_count: result?.observed?.matched_count,
3146
- condition_satisfied: result?.observed?.condition_satisfied ?? false,
3147
- selected_index: result?.observed?.selected_index ?? null,
3148
- last_matched_element: result?.observed?.last_matched_element ?? null
3149
- },
3150
- reason: result?.error?.message ?? 'selector is not visible',
3151
- failure_code: errorCode,
3152
- retryable: errorCode === 'TIMEOUT',
3153
- trace: buildObservationTrace({
3154
- actionType: 'expect_element_visible',
3155
- stage: 'verify',
3156
- success: false,
3157
- attempts: 1,
3158
- metadata: {
3159
- selector,
3160
- element_id: element_id ?? null,
3161
- status: result?.status ?? null,
3162
- reason: result?.error?.message ?? 'selector is not visible'
3163
- }
3164
- })
3165
- }
267
+ static async expectStateHandler(input: StateInput): Promise<ExpectStateResponse> {
268
+ return expectState(input, { observe: target => ToolsObserve.getUITreeHandler(target), getHandle: id => ToolsInteract._resolvedUiElements.get(id),
269
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
270
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx) })
3166
271
  }
3167
272
 
3168
- static async expectStateHandler({
3169
- selector,
3170
- element_id,
3171
- property,
3172
- expected,
3173
- platform,
3174
- deviceId,
3175
- stabilization_window_ms = 1000,
3176
- stable_observation_count = 2,
3177
- snapshot_stale_threshold_ms = 500,
3178
- poll_interval_ms = 150,
3179
- exact_budget = false
3180
- }: {
3181
- selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
3182
- element_id?: string,
3183
- property: string,
3184
- expected: boolean | number | string | Record<string, unknown>,
3185
- platform?: 'android' | 'ios',
3186
- deviceId?: string,
3187
- stabilization_window_ms?: number,
3188
- stable_observation_count?: number,
3189
- snapshot_stale_threshold_ms?: number,
3190
- poll_interval_ms?: number,
3191
- exact_budget?: boolean
3192
- }): Promise<ExpectStateResponse> {
3193
- const compareBoolean = (value: unknown) => typeof value === 'boolean' ? value : null
3194
- const compareString = (value: unknown) => typeof value === 'string' ? value : null
3195
- const compareNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value) ? value : null
3196
- const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
3197
- const start = Date.now()
3198
- const deadline = start + (exact_budget ? stabilization_window_ms : Math.max(500, stabilization_window_ms))
3199
- const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2))
3200
- const pollDelay = exact_budget ? Math.max(50, Math.min(poll_interval_ms || 150, 1000)) : Math.max(100, Math.min(poll_interval_ms || 150, 200))
3201
- const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800))
3202
-
3203
- let attempts = 0
3204
- let stableCount = 0
3205
- let lastReason = 'element not found'
3206
- let lastFailureCode: 'ELEMENT_NOT_FOUND' | 'UNKNOWN' = 'ELEMENT_NOT_FOUND'
3207
- let lastObservedElement: (ActionTargetResolved & { state?: UIElementState | null }) | null = null
3208
- let lastObservedValue: boolean | number | string | Record<string, unknown> | null = null
3209
- let lastRawValue: boolean | number | string | null = null
3210
- let lastResolvedElementId: string | null = element_id ?? null
3211
- const traceSteps: TraceStep[] = []
3212
- let traceAttemptIndex = 0
3213
- let resolveRecorded = false
3214
-
3215
- const recordTraceStep = (
3216
- stage: TraceStep['stage'],
3217
- result: TraceStep['result'],
3218
- metadata?: Record<string, unknown>
3219
- ) => {
3220
- traceSteps.push(createTraceStep({
3221
- stage,
3222
- timestamp: Date.now(),
3223
- result,
3224
- attemptIndex: traceAttemptIndex++,
3225
- metadata
3226
- }))
3227
- }
3228
-
3229
- const buildStateTrace = (outcome: 'success' | 'failure'): ActionTrace => ({
3230
- action_id: nextActionId('expect_state', Date.now()),
3231
- steps: traceSteps,
3232
- final_outcome: outcome,
3233
- attempts
3234
- })
3235
-
3236
- while (Date.now() <= deadline) {
3237
- attempts++
3238
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
3239
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
3240
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
3241
- const treeDeviceId = tree?.device?.id || deviceId
3242
- const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null
3243
- let matched: { el: UiElement, idx: number } | null = null
3244
-
3245
- if (element_id) {
3246
- const resolved = ToolsInteract._resolvedUiElements.get(element_id)
3247
- if (resolved) {
3248
- const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
3249
- if (current) matched = { el: current.el, idx: current.index }
3250
- }
3251
- }
3252
-
3253
- if (!matched && selector) {
3254
- matched = ToolsInteract._findFirstMatchingElement(elements, selector)
3255
- }
3256
-
3257
- if (!matched) {
3258
- lastReason = 'element not found'
3259
- lastFailureCode = 'ELEMENT_NOT_FOUND'
3260
- stableCount = 0
3261
- recordTraceStep('resolve', 'retry', {
3262
- selector: selector ?? null,
3263
- element_id: lastResolvedElementId,
3264
- matched: false,
3265
- reason: lastReason,
3266
- attempt: attempts
3267
- })
3268
- recordTraceStep('stabilize', 'retry', {
3269
- stabilization_attempts: attempts,
3270
- stable_observation_count: stableCount,
3271
- snapshot_freshness_ms: treeAgeMs,
3272
- reason: lastReason
3273
- })
3274
- await sleep(pollDelay)
3275
- continue
3276
- }
3277
-
3278
- const resolvedElement = ToolsInteract._resolvedTargetFromElement(
3279
- ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx),
3280
- matched.el,
3281
- matched.idx
3282
- )
3283
- lastResolvedElementId = resolvedElement.elementId
3284
- lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null }
3285
-
3286
- if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
3287
- lastReason = 'stale snapshot'
3288
- lastFailureCode = 'UNKNOWN'
3289
- stableCount = 0
3290
- recordTraceStep('resolve', 'retry', {
3291
- selector: selector ?? null,
3292
- element_id: lastResolvedElementId,
3293
- matched: true,
3294
- reason: lastReason,
3295
- attempt: attempts
3296
- })
3297
- recordTraceStep('stabilize', 'retry', {
3298
- stabilization_attempts: attempts,
3299
- stable_observation_count: stableCount,
3300
- snapshot_freshness_ms: treeAgeMs,
3301
- reason: lastReason
3302
- })
3303
- await sleep(pollDelay)
3304
- continue
3305
- }
3306
-
3307
- if (!resolveRecorded) {
3308
- recordTraceStep('resolve', 'success', {
3309
- selector: selector ?? null,
3310
- element_id: lastResolvedElementId,
3311
- matched: true,
3312
- reason: 'element resolved'
3313
- })
3314
- resolveRecorded = true
3315
- }
3316
-
3317
- const observedState = matched.el.state ?? null
3318
- const actual = observedState?.[property as keyof UIElementState] ?? null
3319
-
3320
- let success = false
3321
- let reason = ''
3322
- let rawValue: boolean | number | string | null = null
3323
- let observedValue: boolean | number | string | Record<string, unknown> | null = actual as any
3324
-
3325
- switch (property) {
3326
- case 'checked':
3327
- case 'focused':
3328
- case 'expanded':
3329
- case 'enabled': {
3330
- const expectedBool = compareBoolean(expected)
3331
- const actualBool = compareBoolean(actual)
3332
- if (expectedBool === null) {
3333
- reason = `expected ${property} must be boolean`
3334
- } else if (actualBool === null) {
3335
- reason = `${property} state unavailable`
3336
- } else {
3337
- rawValue = actualBool
3338
- success = actualBool === expectedBool
3339
- reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`
3340
- }
3341
- observedValue = actualBool
3342
- break
3343
- }
3344
- case 'value':
3345
- case 'raw_value': {
3346
- const expectedNumber = compareNumber(expected)
3347
- const actualNumber = compareNumber(actual)
3348
- if (expectedNumber !== null && actualNumber !== null) {
3349
- success = actualNumber === expectedNumber
3350
- rawValue = actualNumber
3351
- observedValue = actualNumber
3352
- reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`
3353
- break
3354
- }
3355
- const expectedString = typeof expected === 'string' ? expected : null
3356
- const actualString = compareString(actual)
3357
- if (expectedString !== null && actualString !== null) {
3358
- success = actualString === expectedString
3359
- rawValue = actualString
3360
- observedValue = actualString
3361
- reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`
3362
- } else {
3363
- reason = 'value state unavailable'
3364
- }
3365
- break
3366
- }
3367
- case 'selected': {
3368
- const expectedBool = typeof expected === 'boolean' ? expected : null
3369
- const expectedString = typeof expected === 'string'
3370
- ? expected
3371
- : expected && typeof expected === 'object'
3372
- ? String((expected as { id?: unknown; label?: unknown }).id ?? (expected as { id?: unknown; label?: unknown }).label ?? '')
3373
- : null
3374
- if (!observedState || observedState.selected === undefined || observedState.selected === null) {
3375
- reason = 'selected state unavailable'
3376
- break
3377
- }
3378
- if (expectedBool !== null) {
3379
- const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null
3380
- if (actualBool === null) {
3381
- reason = 'selected state is not boolean'
3382
- break
3383
- }
3384
- rawValue = actualBool
3385
- observedValue = actualBool
3386
- success = actualBool === expectedBool
3387
- reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`
3388
- break
3389
- }
3390
- const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
3391
- ? String((observedState.selected as { id?: unknown; label?: unknown }).id ?? (observedState.selected as { id?: unknown; label?: unknown }).label ?? '')
3392
- : String(observedState.selected)
3393
- const actualString = actualSelected.trim()
3394
- if (!expectedString) {
3395
- reason = 'expected selected must be boolean, string, or object with id/label'
3396
- break
3397
- }
3398
- rawValue = actualString
3399
- observedValue = actualString
3400
- success = actualString === expectedString
3401
- reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`
3402
- break
3403
- }
3404
- case 'text_value': {
3405
- const expectedString = typeof expected === 'string' ? expected : null
3406
- const actualString = compareString(actual)
3407
- if (!expectedString) {
3408
- reason = 'expected text_value must be string'
3409
- } else if (!actualString) {
3410
- reason = 'text_value state unavailable'
3411
- } else {
3412
- success = actualString === expectedString
3413
- rawValue = actualString
3414
- observedValue = actualString
3415
- reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`
3416
- }
3417
- break
3418
- }
3419
- default: {
3420
- if (actual !== null && actual !== undefined) {
3421
- success = actual === expected
3422
- observedValue = actual as any
3423
- rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null
3424
- reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`
3425
- } else {
3426
- reason = `unsupported or unavailable state property: ${property}`
3427
- }
3428
- }
3429
- }
3430
-
3431
- if (success) {
3432
- stableCount++
3433
- recordTraceStep('stabilize', 'success', {
3434
- stabilization_attempts: attempts,
3435
- stable_observation_count: stableCount,
3436
- snapshot_freshness_ms: treeAgeMs,
3437
- reason
3438
- })
3439
- recordTraceStep('verify', 'success', {
3440
- property,
3441
- expected,
3442
- observed_value: observedValue,
3443
- reason
3444
- })
3445
- if (stableCount >= stableTarget) {
3446
- return {
3447
- success: true,
3448
- selector,
3449
- element_id: lastResolvedElementId,
3450
- expected_state: { property, expected },
3451
- element: lastObservedElement,
3452
- observed_state: {
3453
- property,
3454
- value: observedValue,
3455
- ...(rawValue !== null ? { raw_value: rawValue } : {})
3456
- },
3457
- reason,
3458
- stabilization_attempts: attempts,
3459
- stabilization_window_ms: Date.now() - start,
3460
- stable_observation_count: stableCount,
3461
- snapshot_freshness_ms: treeAgeMs ?? undefined,
3462
- trace: buildStateTrace('success')
3463
- } as ExpectStateResponse & {
3464
- stabilization_attempts?: number;
3465
- stabilization_window_ms?: number;
3466
- stable_observation_count?: number;
3467
- snapshot_freshness_ms?: number;
3468
- }
3469
- }
3470
- } else {
3471
- stableCount = 0
3472
- lastReason = reason || lastReason
3473
- lastFailureCode = 'UNKNOWN'
3474
- recordTraceStep('stabilize', 'retry', {
3475
- stabilization_attempts: attempts,
3476
- stable_observation_count: stableCount,
3477
- snapshot_freshness_ms: treeAgeMs,
3478
- reason: lastReason
3479
- })
3480
- recordTraceStep('verify', 'retry', {
3481
- property,
3482
- expected,
3483
- observed_value: observedValue,
3484
- reason: lastReason
3485
- })
3486
- }
3487
-
3488
- if (!success) {
3489
- lastObservedValue = observedValue
3490
- lastRawValue = rawValue
3491
- }
3492
-
3493
- await sleep(pollDelay)
3494
- }
3495
-
3496
- return {
3497
- success: false,
3498
- selector,
3499
- element_id: lastResolvedElementId,
3500
- expected_state: { property, expected },
3501
- element: lastObservedElement,
3502
- observed_state: {
3503
- property,
3504
- value: lastObservedValue,
3505
- ...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
3506
- },
3507
- reason: lastReason,
3508
- failure_code: lastFailureCode,
3509
- retryable: true,
3510
- trace: buildStateTrace('failure')
3511
- }
3512
- }
3513
-
3514
- static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }: { type?: 'ui' | 'log' | 'screen' | 'idle', query?: string, timeoutMs?: number, pollIntervalMs?: number, includeSnapshotOnFailure?: boolean, match?: 'present'|'absent', stability_ms?: number, observationDelayMs?: number, platform?: 'android' | 'ios', deviceId?: string }) {
3515
- const start = Date.now()
3516
- const deadline = start + (timeoutMs || 0)
3517
- const q = (query === null || query === undefined) ? '' : String(query)
3518
-
3519
- // Clamp polling interval to 250-500ms for consistent behavior
3520
- const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500))
3521
-
3522
- // Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
3523
- let initialFingerprint: string | null = null
3524
- let baselineLastLine: string | null = null
3525
- try {
3526
- const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as Promise<ScreenFingerprintResponse | null>
3527
- const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as Promise<any>
3528
- const withTimeout = (p: Promise<any>, ms: number) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))])
3529
- const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)])
3530
- if (fpRes && typeof fpRes === 'object') initialFingerprint = (fpRes as ScreenFingerprintResponse).fingerprint ?? null
3531
- if (gl) {
3532
- const logsArr = Array.isArray((gl as any).logs) ? (gl as any).logs : []
3533
- // Normalize to last message string for baseline comparison
3534
- const msgs = ToolsInteract._logsToMessages(logsArr)
3535
- baselineLastLine = msgs.length ? msgs[msgs.length - 1] : null
3536
- }
3537
- } catch (err) { checkDebuggerGuard();
3538
- try { console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err)) } catch { }
3539
- }
3540
-
3541
- // Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
3542
- let lastChangeAt = Date.now()
3543
- let prevFingerprint = initialFingerprint
3544
-
3545
- const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))
3546
-
3547
- // Optional initial observation delay requested by caller
3548
- if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
3549
- try { console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`) } catch { }
3550
- await sleep(observationDelayMs)
3551
- }
3552
-
3553
- // Telemetry
3554
- let pollCount = 0
3555
- let matchedAt: number | null = null
3556
- let lastObservedState: boolean | null = null
3557
- let stableDuration = 0
3558
- let matchSource: string | null = null
3559
-
3560
- while (Date.now() <= deadline) {
3561
- pollCount++
3562
- const now = Date.now()
3563
- // Evaluate condition per type
3564
- if (type === 'ui') {
3565
- try {
3566
- // Prefer using the public findElementHandler which tests can override. This avoids relying
3567
- // on resolveObserve/getUITree for unit tests which may not have devices available.
3568
- try {
3569
- const findRes = await (ToolsInteract as any).findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId })
3570
- const isPresent = !!(findRes && (findRes as any).found)
3571
- const conditionTrue = (match === 'present') ? isPresent : !isPresent
3572
- if (conditionTrue) {
3573
- if (matchedAt === null) matchedAt = Date.now()
3574
- stableDuration = Date.now() - (matchedAt as number)
3575
- lastObservedState = true
3576
- if (stableDuration >= stability_ms) {
3577
- matchSource = 'ui-find'
3578
- const element = isPresent ? (findRes as any).element : null
3579
- const now2 = Date.now()
3580
- return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null }
3581
- }
3582
- } else {
3583
- matchedAt = null
3584
- stableDuration = 0
3585
- lastObservedState = false
3586
- }
3587
- } catch (err) { checkDebuggerGuard(); console.error('waitForUI(ui) find error:', err) }
3588
- } catch (err) { checkDebuggerGuard(); console.error('waitForUI(ui) outer error:', err) }
3589
- } else if (type === 'log') {
3590
- try {
3591
- // Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
3592
- const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 }) as any
3593
- const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : []
3594
- for (const ent of entries) {
3595
- const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : ''
3596
- if (q && String(msg).includes(q)) {
3597
- const now2 = Date.now()
3598
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true }
3599
- }
3600
- }
3601
-
3602
- const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as any
3603
- const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : []
3604
- // Normalize to messages for comparison
3605
- const msgs = ToolsInteract._logsToMessages(logsArr)
3606
- let startIndex = 0
3607
- if (baselineLastLine) {
3608
- const idx = msgs.lastIndexOf(baselineLastLine)
3609
- startIndex = idx >= 0 ? idx + 1 : 0
3610
- }
3611
- for (let i = startIndex; i < msgs.length; i++) {
3612
- const line = msgs[i]
3613
- if (q && String(line).includes(q)) {
3614
- const now2 = Date.now()
3615
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true }
3616
- }
3617
- }
3618
- } catch (err) { checkDebuggerGuard(); console.error('waitForUI(log) error:', err) }
3619
- } else if (type === 'screen') {
3620
- try {
3621
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
3622
- const fp = fpRes?.fingerprint ?? null
3623
- if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
3624
- // when screen changed, require stability_ms where fingerprint remains the same
3625
- if (matchedAt === null) matchedAt = now
3626
- const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null)?.fingerprint ?? null
3627
- if (confirmFp === fp) {
3628
- stableDuration = Date.now() - (matchedAt as number)
3629
- lastObservedState = true
3630
- if (stableDuration >= stability_ms) {
3631
- const now2 = Date.now()
3632
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null }
3633
- }
3634
- } else {
3635
- matchedAt = null
3636
- stableDuration = 0
3637
- lastObservedState = false
3638
- }
3639
- }
3640
- } catch (err) { checkDebuggerGuard(); console.error('waitForUI(screen) error:', err) }
3641
- } else if (type === 'idle') {
3642
- try {
3643
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
3644
- const fp = fpRes?.fingerprint ?? null
3645
- if (fp !== prevFingerprint) {
3646
- prevFingerprint = fp
3647
- lastChangeAt = Date.now()
3648
- matchedAt = null
3649
- stableDuration = 0
3650
- lastObservedState = false
3651
- } else {
3652
- const idleMs = Date.now() - lastChangeAt
3653
- lastObservedState = true
3654
- if (idleMs >= stability_ms) {
3655
- const now2 = Date.now()
3656
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null }
3657
- }
3658
- }
3659
- } catch (err) { checkDebuggerGuard(); console.error('waitForUI(idle) error:', err) }
3660
- }
3661
-
3662
- // Respect poll interval and avoid tight loop
3663
- await sleep(pollInterval)
3664
- }
3665
-
3666
- // On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
3667
- let snapshot: any = null
3668
- if (includeSnapshotOnFailure) {
3669
- try {
3670
- // Use dynamic import to avoid circular-initialization issues where the ToolsObserve
3671
- // binding captured earlier may not reflect test-time overrides. Importing at call
3672
- // time ensures the latest exported ToolsObserve object is used.
3673
- const Obs = await import('../observe/index.js')
3674
- snapshot = await (Obs as any).ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId })
3675
- } catch (err) { checkDebuggerGuard();
3676
- snapshot = { error: err instanceof Error ? err.message : String(err) }
3677
- }
3678
- }
3679
-
3680
- const elapsed = Date.now() - start
3681
- return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null }
273
+ static async waitForUICore(input: LegacyWaitInput) {
274
+ return waitForUICore(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), logs: target => ToolsObserve.getLogsHandler(target),
275
+ stream: target => ToolsObserve.readLogStreamHandler(target), find: input => ToolsInteract.findElementHandler(input),
276
+ snapshot: async input => { const { ToolsObserve } = await import('../observe/index.js'); return ToolsObserve.captureDebugSnapshotHandler(input) } })
3682
277
  }
3683
278
  }