mobile-debug-mcp 0.33.0 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (448) hide show
  1. package/.agents/skills/mcp-builder/SKILL.md +44 -64
  2. package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
  3. package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
  4. package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
  5. package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
  6. package/.agents/skills/test-authoring/SKILL.md +7 -6
  7. package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
  8. package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
  9. package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
  10. package/.github/copilot-instructions.md +14 -36
  11. package/.github/workflows/ci.yml +23 -2
  12. package/AGENTS.md +12 -3
  13. package/CONTRIBUTING.md +79 -0
  14. package/README.md +158 -124
  15. package/agent-plugin/.claude-plugin/plugin.json +2 -2
  16. package/agent-plugin/.codex-plugin/plugin.json +1 -1
  17. package/agent-plugin/.cursor-plugin/plugin.json +2 -2
  18. package/agent-plugin/.mcp.json +1 -1
  19. package/agent-plugin/README.md +20 -6
  20. package/agent-plugin/mcp.json +1 -1
  21. package/agent-plugin/plugin.json +1 -1
  22. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +113 -4
  23. package/dist/contracts/execution-controls.js +24 -0
  24. package/dist/contracts/selector.js +8 -0
  25. package/dist/contracts/target.js +15 -0
  26. package/dist/contracts/ui.js +1 -0
  27. package/dist/debugger/android.js +79 -0
  28. package/dist/debugger/diagnostics.js +54 -0
  29. package/dist/debugger/guard.js +30 -0
  30. package/dist/debugger/protocol.js +33 -0
  31. package/dist/debugger/sessions.js +304 -0
  32. package/dist/debugger/worker.js +176 -0
  33. package/dist/interact/adjustment/adjustment-context.js +41 -0
  34. package/dist/interact/adjustment/adjustment-gestures.js +55 -0
  35. package/dist/interact/adjustment/adjustment-plan.js +18 -0
  36. package/dist/interact/adjustment/adjustment-resolution.js +127 -0
  37. package/dist/interact/adjustment/adjustment-result.js +19 -0
  38. package/dist/interact/adjustment/adjustment-semantic.js +64 -0
  39. package/dist/interact/adjustment/adjustment-verification.js +35 -0
  40. package/dist/interact/adjustment/adjustment.js +52 -0
  41. package/dist/interact/adjustment/control-geometry.js +87 -0
  42. package/dist/interact/adjustment/control-values.js +136 -0
  43. package/dist/interact/adjustment/slider-labels.js +84 -0
  44. package/dist/interact/assertions/screen-assertion.js +37 -0
  45. package/dist/interact/assertions/state-assertion-context.js +35 -0
  46. package/dist/interact/assertions/state-assertion.js +76 -0
  47. package/dist/interact/assertions/state-comparison.js +141 -0
  48. package/dist/interact/assertions/visibility-assertion.js +34 -0
  49. package/dist/interact/conditional-action.js +12 -7
  50. package/dist/interact/index.js +65 -3074
  51. package/dist/interact/input-service.js +1 -0
  52. package/dist/interact/interaction-trace.js +21 -0
  53. package/dist/interact/model.js +1 -0
  54. package/dist/interact/observation.js +1 -0
  55. package/dist/interact/targeting/candidate-ancestor.js +58 -0
  56. package/dist/interact/targeting/candidate-result.js +23 -0
  57. package/dist/interact/targeting/candidate-scoring.js +50 -0
  58. package/dist/interact/targeting/element-helpers.js +216 -0
  59. package/dist/interact/targeting/find-element.js +65 -0
  60. package/dist/interact/targeting/nearby-control.js +56 -0
  61. package/dist/interact/targeting/resolved-elements.js +21 -0
  62. package/dist/interact/targeting/tap-target.js +17 -0
  63. package/dist/interact/targeting/target-scope.js +104 -0
  64. package/dist/interact/waits/legacy-log-wait.js +38 -0
  65. package/dist/interact/waits/legacy-wait-observations.js +103 -0
  66. package/dist/interact/waits/legacy-wait-state.js +34 -0
  67. package/dist/interact/waits/legacy-wait.js +48 -0
  68. package/dist/interact/waits/screen-wait.js +49 -0
  69. package/dist/interact/waits/ui-change-helpers.js +157 -0
  70. package/dist/interact/waits/ui-change-state.js +18 -0
  71. package/dist/interact/waits/ui-change-wait.js +77 -0
  72. package/dist/interact/waits/ui-wait-condition.js +56 -0
  73. package/dist/interact/waits/ui-wait-state.js +38 -0
  74. package/dist/interact/waits/ui-wait.js +106 -0
  75. package/dist/manage/android.js +55 -51
  76. package/dist/manage/index.js +38 -75
  77. package/dist/manage/ios/build-types.js +1 -0
  78. package/dist/manage/ios/build.js +210 -0
  79. package/dist/manage/ios/project-discovery.js +29 -0
  80. package/dist/manage/ios/scheme.js +18 -0
  81. package/dist/manage/ios.js +42 -256
  82. package/dist/observe/android/context.js +1 -0
  83. package/dist/observe/android/current-screen.js +55 -0
  84. package/dist/observe/android/fingerprint.js +13 -0
  85. package/dist/observe/android/log-stream.js +132 -0
  86. package/dist/observe/android/logs.js +107 -0
  87. package/dist/observe/android/screenshot.js +70 -0
  88. package/dist/observe/android/ui-tree.js +107 -0
  89. package/dist/observe/android.js +16 -520
  90. package/dist/observe/evidence.js +122 -76
  91. package/dist/observe/index.js +16 -263
  92. package/dist/observe/ios/context.js +16 -0
  93. package/dist/observe/ios/fingerprint.js +12 -0
  94. package/dist/observe/ios/log-stream.js +105 -0
  95. package/dist/observe/ios/logs.js +168 -0
  96. package/dist/observe/ios/screenshot.js +22 -0
  97. package/dist/observe/ios/ui-elements.js +241 -0
  98. package/dist/observe/ios/ui-tree.js +137 -0
  99. package/dist/observe/ios.js +17 -715
  100. package/dist/observe/logs/model.js +1 -0
  101. package/dist/{network → observe/network}/index.js +2 -2
  102. package/dist/observe/readiness-service.js +13 -2
  103. package/dist/observe/readiness.js +44 -18
  104. package/dist/observe/screenshot-encoding.js +41 -0
  105. package/dist/observe/snapshot-collection.js +79 -0
  106. package/dist/observe/snapshot-logs.js +30 -0
  107. package/dist/observe/snapshot-metadata.js +43 -31
  108. package/dist/observe/snapshot-semantic.js +115 -0
  109. package/dist/observe/system-trace-android.js +135 -0
  110. package/dist/observe/system-trace.js +294 -0
  111. package/dist/runtime/action-results.js +270 -0
  112. package/dist/runtime/clock.js +4 -0
  113. package/dist/server/common.js +1 -256
  114. package/dist/server/definitions/assert.js +162 -0
  115. package/dist/server/definitions/diagnostic.js +118 -0
  116. package/dist/server/definitions/input.js +274 -0
  117. package/dist/server/definitions/manage.js +194 -0
  118. package/dist/server/definitions/observe.js +149 -0
  119. package/dist/server/definitions/target.js +138 -0
  120. package/dist/server/definitions/wait.js +105 -0
  121. package/dist/server/handlers/action-execution.js +163 -0
  122. package/dist/server/handlers/correlation.js +23 -0
  123. package/dist/server/handlers/debugger-handlers.js +48 -0
  124. package/dist/server/handlers/debugger-response.js +9 -0
  125. package/dist/server/handlers/diagnostic.js +69 -0
  126. package/dist/server/handlers/handler-types.js +1 -0
  127. package/dist/server/handlers/input.js +154 -0
  128. package/dist/server/handlers/interact.js +113 -0
  129. package/dist/server/handlers/journey-assertions.js +35 -0
  130. package/dist/server/handlers/journey.js +147 -0
  131. package/dist/server/handlers/manage.js +119 -0
  132. package/dist/server/handlers/observe.js +95 -0
  133. package/dist/server/handlers/payload.js +4 -0
  134. package/dist/server/handlers/tool-handlers.js +160 -0
  135. package/dist/server/schemas/condition-schema.js +2 -0
  136. package/dist/server/schemas/debugger-schema.js +47 -0
  137. package/dist/server/{journey-schema.js → schemas/journey-schema.js} +8 -22
  138. package/dist/server/schemas/system-trace-schema.js +17 -0
  139. package/dist/server/tool-definitions.js +62 -1134
  140. package/dist/server-core.js +4 -4
  141. package/dist/server.js +34 -0
  142. package/dist/system/android-status.js +101 -0
  143. package/dist/system/android.js +37 -27
  144. package/dist/system/index.js +13 -438
  145. package/dist/system/ios-status.js +132 -0
  146. package/dist/system/ios.js +11 -8
  147. package/dist/system/model.js +1 -0
  148. package/dist/system/probes.js +105 -0
  149. package/dist/system/status-summary.js +43 -0
  150. package/dist/system/unsupported-status.js +52 -0
  151. package/dist/utils/android/commands.js +89 -0
  152. package/dist/utils/android/gradle.js +139 -0
  153. package/dist/utils/android/logs.js +103 -0
  154. package/dist/utils/android/ui-tree.js +233 -0
  155. package/dist/utils/android/utils.js +6 -564
  156. package/dist/utils/cli/idb/idb-helper.js +4 -6
  157. package/dist/utils/cli/idb/install-idb.js +13 -9
  158. package/dist/utils/cli/ios/preflight-ios.js +42 -38
  159. package/dist/utils/exec.js +2 -0
  160. package/dist/utils/ios/utils.js +45 -46
  161. package/dist/utils/java.js +75 -57
  162. package/dist/utils/operation-budget.js +2 -0
  163. package/dist/utils/resolve-device.js +6 -2
  164. package/dist/utils/ui/index.js +26 -26
  165. package/docs/CHANGELOG.md +48 -0
  166. package/docs/README.md +38 -0
  167. package/docs/benchmarks/README.md +18 -0
  168. package/docs/benchmarks/interaction-latency/README.md +14 -0
  169. package/docs/benchmarks/interaction-latency/final.md +4 -1
  170. package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
  171. package/docs/benchmarks/ui-tree-performance/README.md +24 -16
  172. package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
  173. package/docs/code-structure.md +118 -0
  174. package/docs/specs/001-state-verification/spec.md +16 -14
  175. package/docs/specs/002-richer-element-identity/spec.md +12 -10
  176. package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
  177. package/docs/specs/004-action-verification-routing/spec.md +3 -3
  178. package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
  179. package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
  180. package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
  181. package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
  182. package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
  183. package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
  184. package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
  185. package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
  186. package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
  187. package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
  188. package/docs/specs/014-actionability-resolution/spec.md +20 -18
  189. package/docs/specs/015-adjustable-control-support/spec.md +17 -15
  190. package/docs/specs/016-environment-tools/spec.md +20 -20
  191. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
  192. package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
  193. package/docs/specs/020-conditional-actions/intent.md +5 -3
  194. package/docs/specs/020-conditional-actions/plan.md +8 -9
  195. package/docs/specs/020-conditional-actions/spec.md +4 -2
  196. package/docs/specs/021-app-readiness-signals/intent.md +4 -2
  197. package/docs/specs/021-app-readiness-signals/plan.md +7 -6
  198. package/docs/specs/021-app-readiness-signals/spec.md +3 -1
  199. package/docs/specs/022-correlated-evidence/intent.md +4 -4
  200. package/docs/specs/022-correlated-evidence/plan.md +10 -11
  201. package/docs/specs/022-correlated-evidence/spec.md +3 -1
  202. package/docs/specs/023-android-system-traces/device-evidence.json +38 -0
  203. package/docs/specs/023-android-system-traces/emulator-recheck.json +38 -0
  204. package/docs/specs/023-android-system-traces/intent.md +41 -0
  205. package/docs/specs/023-android-system-traces/plan.md +115 -0
  206. package/docs/specs/023-android-system-traces/spec.md +122 -0
  207. package/docs/specs/024-android-heap-dump-capture/intent.md +41 -0
  208. package/docs/specs/025-android-runtime-debugger/intent.md +41 -0
  209. package/docs/specs/025-android-runtime-debugger/plan.md +106 -0
  210. package/docs/specs/025-android-runtime-debugger/spec.md +183 -0
  211. package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
  212. package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
  213. package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
  214. package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
  215. package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
  216. package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
  217. package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
  218. package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
  219. package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
  220. package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
  221. package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
  222. package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
  223. package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
  224. package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
  225. package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
  226. package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
  227. package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
  228. package/docs/specs/030-android-coroutine-validation/report.md +240 -0
  229. package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
  230. package/docs/tools/TOOLS.md +31 -16
  231. package/docs/tools/{evidence.md → correlated-evidence.md} +14 -3
  232. package/docs/tools/debugger.md +180 -0
  233. package/docs/tools/interact.md +42 -20
  234. package/docs/tools/manage.md +25 -44
  235. package/docs/tools/observe.md +64 -25
  236. package/docs/tools/system-traces.md +134 -0
  237. package/docs/tools/system.md +12 -5
  238. package/package.json +11 -8
  239. package/scripts/architecture-graph.mjs +69 -0
  240. package/scripts/check-architecture.mjs +47 -0
  241. package/scripts/code-quality-baseline.json +1 -0
  242. package/scripts/code-quality.mjs +161 -0
  243. package/src/contracts/execution-controls.ts +26 -0
  244. package/src/contracts/selector.ts +17 -0
  245. package/src/contracts/target.ts +19 -0
  246. package/src/contracts/ui.ts +94 -0
  247. package/src/debugger/android.ts +66 -0
  248. package/src/debugger/diagnostics.ts +43 -0
  249. package/src/debugger/guard.ts +31 -0
  250. package/src/debugger/java/DebuggerWorker.java +166 -0
  251. package/src/debugger/protocol.ts +37 -0
  252. package/src/debugger/sessions.ts +327 -0
  253. package/src/debugger/worker.ts +135 -0
  254. package/src/interact/adjustment/adjustment-context.ts +83 -0
  255. package/src/interact/adjustment/adjustment-gestures.ts +48 -0
  256. package/src/interact/adjustment/adjustment-plan.ts +23 -0
  257. package/src/interact/adjustment/adjustment-resolution.ts +104 -0
  258. package/src/interact/adjustment/adjustment-result.ts +34 -0
  259. package/src/interact/adjustment/adjustment-semantic.ts +53 -0
  260. package/src/interact/adjustment/adjustment-verification.ts +60 -0
  261. package/src/interact/adjustment/adjustment.ts +49 -0
  262. package/src/interact/adjustment/control-geometry.ts +120 -0
  263. package/src/interact/adjustment/control-values.ts +141 -0
  264. package/src/interact/adjustment/slider-labels.ts +84 -0
  265. package/src/interact/assertions/screen-assertion.ts +39 -0
  266. package/src/interact/assertions/state-assertion-context.ts +48 -0
  267. package/src/interact/assertions/state-assertion.ts +64 -0
  268. package/src/interact/assertions/state-comparison.ts +168 -0
  269. package/src/interact/assertions/visibility-assertion.ts +41 -0
  270. package/src/interact/conditional-action.ts +18 -12
  271. package/src/interact/index.ts +76 -3491
  272. package/src/interact/input-service.ts +11 -0
  273. package/src/interact/interaction-trace.ts +36 -0
  274. package/src/interact/model.ts +109 -0
  275. package/src/interact/observation.ts +7 -0
  276. package/src/interact/targeting/candidate-ancestor.ts +49 -0
  277. package/src/interact/targeting/candidate-result.ts +24 -0
  278. package/src/interact/targeting/candidate-scoring.ts +42 -0
  279. package/src/interact/targeting/element-helpers.ts +243 -0
  280. package/src/interact/targeting/find-element.ts +47 -0
  281. package/src/interact/targeting/nearby-control.ts +46 -0
  282. package/src/interact/targeting/resolved-elements.ts +20 -0
  283. package/src/interact/targeting/tap-target.ts +22 -0
  284. package/src/interact/targeting/target-scope.ts +121 -0
  285. package/src/interact/waits/legacy-log-wait.ts +30 -0
  286. package/src/interact/waits/legacy-wait-observations.ts +55 -0
  287. package/src/interact/waits/legacy-wait-state.ts +48 -0
  288. package/src/interact/waits/legacy-wait.ts +34 -0
  289. package/src/interact/waits/screen-wait.ts +37 -0
  290. package/src/interact/waits/ui-change-helpers.ts +184 -0
  291. package/src/interact/waits/ui-change-state.ts +26 -0
  292. package/src/interact/waits/ui-change-wait.ts +59 -0
  293. package/src/interact/waits/ui-wait-condition.ts +52 -0
  294. package/src/interact/waits/ui-wait-state.ts +55 -0
  295. package/src/interact/waits/ui-wait.ts +84 -0
  296. package/src/manage/android.ts +56 -50
  297. package/src/manage/index.ts +63 -93
  298. package/src/manage/ios/build-types.ts +14 -0
  299. package/src/manage/ios/build.ts +230 -0
  300. package/src/manage/ios/project-discovery.ts +26 -0
  301. package/src/manage/ios/scheme.ts +16 -0
  302. package/src/manage/ios.ts +65 -298
  303. package/src/observe/android/context.ts +1 -0
  304. package/src/observe/android/current-screen.ts +63 -0
  305. package/src/observe/android/fingerprint.ts +13 -0
  306. package/src/observe/android/log-stream.ts +127 -0
  307. package/src/observe/android/logs.ts +110 -0
  308. package/src/observe/android/screenshot.ts +79 -0
  309. package/src/observe/android/ui-tree.ts +113 -0
  310. package/src/observe/android.ts +17 -531
  311. package/src/observe/evidence.ts +130 -71
  312. package/src/observe/index.ts +24 -286
  313. package/src/observe/ios/context.ts +22 -0
  314. package/src/observe/ios/fingerprint.ts +12 -0
  315. package/src/observe/ios/log-stream.ts +103 -0
  316. package/src/observe/ios/logs.ts +179 -0
  317. package/src/observe/ios/screenshot.ts +26 -0
  318. package/src/observe/ios/ui-elements.ts +266 -0
  319. package/src/observe/ios/ui-tree.ts +143 -0
  320. package/src/observe/ios.ts +18 -749
  321. package/src/observe/logs/model.ts +11 -0
  322. package/src/{network → observe/network}/index.ts +2 -2
  323. package/src/observe/readiness-service.ts +10 -3
  324. package/src/observe/readiness.ts +49 -22
  325. package/src/observe/screenshot-encoding.ts +32 -0
  326. package/src/observe/snapshot-collection.ts +69 -0
  327. package/src/observe/snapshot-logs.ts +26 -0
  328. package/src/observe/snapshot-metadata.ts +51 -31
  329. package/src/observe/snapshot-semantic.ts +153 -0
  330. package/src/observe/system-trace-android.ts +133 -0
  331. package/src/observe/system-trace.ts +227 -0
  332. package/src/runtime/action-results.ts +376 -0
  333. package/src/runtime/clock.ts +9 -0
  334. package/src/server/common.ts +9 -359
  335. package/src/server/definitions/assert.ts +164 -0
  336. package/src/server/definitions/diagnostic.ts +120 -0
  337. package/src/server/definitions/input.ts +276 -0
  338. package/src/server/definitions/manage.ts +196 -0
  339. package/src/server/definitions/observe.ts +151 -0
  340. package/src/server/definitions/target.ts +140 -0
  341. package/src/server/definitions/wait.ts +107 -0
  342. package/src/server/handlers/action-execution.ts +164 -0
  343. package/src/server/handlers/correlation.ts +20 -0
  344. package/src/server/handlers/debugger-handlers.ts +39 -0
  345. package/src/server/handlers/debugger-response.ts +7 -0
  346. package/src/server/handlers/diagnostic.ts +79 -0
  347. package/src/server/handlers/handler-types.ts +7 -0
  348. package/src/server/handlers/input.ts +173 -0
  349. package/src/server/handlers/interact.ts +135 -0
  350. package/src/server/handlers/journey-assertions.ts +39 -0
  351. package/src/server/handlers/journey.ts +132 -0
  352. package/src/server/handlers/manage.ts +141 -0
  353. package/src/server/handlers/observe.ts +116 -0
  354. package/src/server/handlers/payload.ts +36 -0
  355. package/src/server/handlers/tool-handlers.ts +159 -0
  356. package/src/server/schemas/condition-schema.ts +2 -0
  357. package/src/server/schemas/debugger-schema.ts +43 -0
  358. package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +8 -22
  359. package/src/server/schemas/system-trace-schema.ts +17 -0
  360. package/src/server/tool-definitions.ts +69 -1140
  361. package/src/server-core.ts +4 -4
  362. package/src/server.ts +25 -1
  363. package/src/system/android-status.ts +124 -0
  364. package/src/system/android.ts +33 -21
  365. package/src/system/index.ts +21 -523
  366. package/src/system/ios-status.ts +147 -0
  367. package/src/system/ios.ts +14 -10
  368. package/src/system/model.ts +41 -0
  369. package/src/system/probes.ts +114 -0
  370. package/src/system/status-summary.ts +47 -0
  371. package/src/system/unsupported-status.ts +55 -0
  372. package/src/types.ts +16 -106
  373. package/src/utils/android/commands.ts +81 -0
  374. package/src/utils/android/gradle.ts +132 -0
  375. package/src/utils/android/logs.ts +103 -0
  376. package/src/utils/android/ui-tree.ts +239 -0
  377. package/src/utils/android/utils.ts +16 -556
  378. package/src/utils/cli/idb/check-idb.ts +4 -4
  379. package/src/utils/cli/idb/idb-helper.ts +6 -8
  380. package/src/utils/cli/idb/install-idb.ts +15 -9
  381. package/src/utils/cli/ios/preflight-ios.ts +39 -31
  382. package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
  383. package/src/utils/exec.ts +2 -0
  384. package/src/utils/ios/utils.ts +57 -61
  385. package/src/utils/java.ts +71 -51
  386. package/src/utils/operation-budget.ts +2 -0
  387. package/src/utils/resolve-device.ts +11 -6
  388. package/src/utils/ui/index.ts +28 -28
  389. package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
  390. package/test/device/automated/debugger/coroutine-support.ts +140 -0
  391. package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
  392. package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
  393. package/test/device/automated/debugger/runtime.android.test.ts +219 -0
  394. package/test/device/automated/interact/interaction.android.test.ts +56 -0
  395. package/test/device/automated/observe/system-trace.smoke.ts +104 -0
  396. package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
  397. package/test/fixtures/contracts/tool-definitions.json +3536 -0
  398. package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
  399. package/test/fixtures/coroutine-app/README.md +48 -0
  400. package/test/fixtures/coroutine-app/build.ts +35 -0
  401. package/test/fixtures/debugger-app/MainActivity.java +67 -0
  402. package/test/fixtures/debugger-app/MainActivity.kt +32 -0
  403. package/test/fixtures/debugger-app/NoLocals.java +7 -0
  404. package/test/fixtures/debugger-app/README.md +65 -0
  405. package/test/fixtures/debugger-app/build.ts +39 -0
  406. package/test/fixtures/latency-journey-app/README.md +2 -0
  407. package/test/fixtures/readiness-app/README.md +2 -0
  408. package/test/helpers/perfetto.ts +59 -0
  409. package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
  410. package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
  411. package/test/unit/debugger/diagnostics.test.ts +80 -0
  412. package/test/unit/debugger/guards.test.ts +50 -0
  413. package/test/unit/debugger/native.test.ts +33 -0
  414. package/test/unit/debugger/preparation.test.ts +32 -0
  415. package/test/unit/debugger/protocol.test.ts +15 -0
  416. package/test/unit/debugger/sessions.test.ts +83 -0
  417. package/test/unit/interact/adjustment_service.test.ts +42 -0
  418. package/test/unit/interact/adjustment_verification.test.ts +35 -0
  419. package/test/unit/interact/clock_dependencies.test.ts +63 -0
  420. package/test/unit/interact/element_helpers.test.ts +19 -0
  421. package/test/unit/interact/legacy_wait_service.test.ts +17 -0
  422. package/test/unit/interact/state_comparison.test.ts +20 -0
  423. package/test/unit/interact/target_scope.test.ts +35 -0
  424. package/test/unit/network/get_network_activity.test.ts +1 -1
  425. package/test/unit/observe/readiness.test.ts +3 -1
  426. package/test/unit/observe/screenshot-encoding.test.ts +30 -0
  427. package/test/unit/observe/snapshot-logs.test.ts +14 -0
  428. package/test/unit/observe/system-trace-android.test.ts +52 -0
  429. package/test/unit/observe/system-trace.test.ts +166 -0
  430. package/test/unit/quality/architecture.test.ts +60 -0
  431. package/test/unit/quality/code-quality.test.ts +58 -0
  432. package/test/unit/quality/interaction_boundaries.test.ts +24 -0
  433. package/test/unit/server/conditional_actions.test.ts +1 -1
  434. package/test/unit/server/conditional_schema.test.ts +2 -2
  435. package/test/unit/server/debugger-response.test.ts +63 -0
  436. package/test/unit/server/system-trace.test.ts +64 -0
  437. package/test/unit/server/tool_definitions_parity.test.ts +8 -0
  438. package/dist/server/tool-handlers.js +0 -893
  439. package/docs/ROADMAP.md +0 -646
  440. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
  441. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
  442. package/docs/specs/baseline-spec-v0.md +0 -312
  443. package/docs/specs/mcp-tooling-spec-v1.md +0 -446
  444. package/src/server/tool-handlers.ts +0 -923
  445. /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
  446. /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
  447. /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
  448. /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
@@ -1,179 +1,29 @@
1
- import { currentReadinessWait, observeReadinessWait, isDeviceUnavailable } from '../observe/readiness.js';
2
- import { readinessDelay } from '../observe/readiness-service.js';
1
+ import { resolveTap } from './targeting/tap-target.js';
2
+ import { waitForUICore } from './waits/legacy-wait.js';
3
+ import { expectState } from './assertions/state-assertion.js';
4
+ import { waitForUIChange } from './waits/ui-change-wait.js';
5
+ import { waitForScreenChange } from './waits/screen-wait.js';
6
+ import { expectScreen } from './assertions/screen-assertion.js';
7
+ import { expectVisible } from './assertions/visibility-assertion.js';
8
+ import { findElement } from './targeting/find-element.js';
9
+ import { waitForUI } from './waits/ui-wait.js';
10
+ import { adjustControl } from './adjustment/adjustment.js';
11
+ import { ResolvedElements } from './targeting/resolved-elements.js';
12
+ import { findScopedElement, resolveUiChangeScope } from './targeting/target-scope.js';
13
+ import { _normalizeBounds, _computeElementId, _resolvedTargetFromElement } from './targeting/element-helpers.js';
14
+ import { _collectSubtreeIndices } from './waits/ui-change-helpers.js';
3
15
  import { isConditionBlocked } from './conditional-action.js';
4
- import { remainingBudget } from '../utils/operation-budget.js';
5
- import { createHash } from 'crypto';
6
16
  import { AndroidInteract } from './android.js';
7
17
  import { iOSInteract } from './ios.js';
8
18
  export { AndroidInteract, iOSInteract };
9
19
  import { resolveTargetDevice } from '../utils/resolve-device.js';
10
20
  import { ToolsObserve } from '../observe/index.js';
11
- import { computeSnapshotSignature } from '../observe/snapshot-metadata.js';
12
- import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js';
13
- function buildObservationTrace({ actionType, stage, success, attempts, metadata }) {
14
- const now = Date.now();
15
- const actionId = nextActionId(actionType, now);
16
- const steps = [
17
- createTraceStep({
18
- stage,
19
- timestamp: now,
20
- result: success ? 'success' : 'failure',
21
- attemptIndex: 0,
22
- metadata
23
- })
24
- ];
25
- return {
26
- action_id: actionId,
27
- steps,
28
- final_outcome: success ? 'success' : 'failure',
29
- attempts: Math.max(1, Math.floor(attempts || 1))
30
- };
31
- }
21
+ import { buildActionExecutionResult } from '../runtime/action-results.js';
32
22
  export class ToolsInteract {
33
23
  static _maxResolvedUiElements = 256;
34
- static _uiChangeKinds = ['hierarchy_diff', 'text_change', 'state_change'];
35
- static _sliderSearchLookahead = 8;
36
- static _sliderNegativeGapTolerancePx = 32;
37
- static _sliderPositiveGapLimitPx = 640;
38
- static _sliderTrackMinLengthPx = 220;
39
- static _sliderTrackMaxThicknessPx = 180;
40
- static _sliderTrackLengthRatio = 0.18;
41
- static _sliderTrackThicknessRatio = 0.08;
42
- static _sliderLabelWidthRatio = 1.5;
43
- static _resolvedUiElements = new Map();
44
- static _normalize(s) {
45
- if (s === null || s === undefined)
46
- return '';
47
- try {
48
- return String(s).toLowerCase().trim();
49
- }
50
- catch {
51
- return '';
52
- }
53
- }
54
- static _normalizeBounds(bounds) {
55
- if (!Array.isArray(bounds) || bounds.length < 4)
56
- return null;
57
- const normalized = bounds.slice(0, 4).map((value) => Number(value));
58
- if (normalized.some((value) => Number.isNaN(value)))
59
- return null;
60
- return normalized;
61
- }
62
- static _hash(value) {
63
- return createHash('sha256').update(JSON.stringify(value)).digest('hex');
64
- }
65
- static _matchesSelector(el, selector) {
66
- if (!selector)
67
- return false;
68
- const normalize = ToolsInteract._normalize;
69
- const containsFlag = !!selector.contains;
70
- const text = normalize(el.text ?? el.label ?? el.value ?? '');
71
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
72
- const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '');
73
- if (selector.text !== undefined && selector.text !== null) {
74
- const q = normalize(selector.text);
75
- if (containsFlag ? !text.includes(q) : text !== q)
76
- return false;
77
- }
78
- if (selector.resource_id !== undefined && selector.resource_id !== null) {
79
- const q = normalize(selector.resource_id);
80
- if (containsFlag ? !resourceId.includes(q) : resourceId !== q)
81
- return false;
82
- }
83
- if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
84
- const q = normalize(selector.accessibility_id);
85
- if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q)
86
- return false;
87
- }
88
- return true;
89
- }
90
- static _findFirstMatchingElement(elements, selector) {
91
- if (!selector)
92
- return null;
93
- for (let i = 0; i < elements.length; i++) {
94
- const el = elements[i];
95
- if (!el)
96
- continue;
97
- if (ToolsInteract._matchesSelector(el, selector))
98
- return { el, idx: i };
99
- }
100
- return null;
101
- }
102
- static _resolveParentIndex(elements, parentId) {
103
- if (parentId === undefined || parentId === null)
104
- return null;
105
- if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
106
- return parentId;
107
- }
108
- if (typeof parentId === 'string') {
109
- const normalized = ToolsInteract._normalize(parentId);
110
- if (!normalized)
111
- return null;
112
- if (/^\d+$/.test(normalized)) {
113
- const index = Number(normalized);
114
- if (index >= 0 && index < elements.length)
115
- return index;
116
- }
117
- const foundIndex = elements.findIndex((el) => {
118
- if (!el)
119
- return false;
120
- return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
121
- ToolsInteract._normalize(el.stable_id ?? '') === normalized;
122
- });
123
- return foundIndex >= 0 ? foundIndex : null;
124
- }
125
- return null;
126
- }
127
- static _isVisibleElement(el) {
128
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
129
- return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
130
- }
131
- static _isTapActionable(el, storedStableId, platform) {
132
- if (!ToolsInteract._isVisibleElement(el)) {
133
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
134
- }
135
- if (el.enabled === false) {
136
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
137
- }
138
- const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
139
- (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap')));
140
- if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
141
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
142
- }
143
- if (storedStableId) {
144
- if (!el.stable_id || el.stable_id !== storedStableId) {
145
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
146
- }
147
- }
148
- return { actionable: true };
149
- }
150
- static _isAdjustableActionable(el, storedStableId) {
151
- if (!ToolsInteract._isVisibleElement(el)) {
152
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
153
- }
154
- if (el.enabled === false) {
155
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
156
- }
157
- if (storedStableId) {
158
- if (!el.stable_id || el.stable_id !== storedStableId) {
159
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
160
- }
161
- }
162
- return { actionable: true };
163
- }
164
- static _computeElementId(platform, deviceId, el, index) {
165
- const identity = {
166
- platform,
167
- deviceId: deviceId || '',
168
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
169
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
170
- accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
171
- class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
172
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
173
- index
174
- };
175
- return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`;
176
- }
24
+ static _resolvedUiElements = new ResolvedElements(ToolsInteract._maxResolvedUiElements);
25
+ static _normalizeBounds = _normalizeBounds;
26
+ static _computeElementId = _computeElementId;
177
27
  static _buildResolvedElement(platform, deviceId, el, index) {
178
28
  const bounds = ToolsInteract._normalizeBounds(el.bounds);
179
29
  const elementId = ToolsInteract._computeElementId(platform, deviceId, el, index);
@@ -202,224 +52,14 @@ export class ToolsInteract {
202
52
  };
203
53
  }
204
54
  static _resolveUiChangeScope(tree, scope, target) {
205
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
206
- const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
207
- if (normalizedScope === 'screen') {
208
- return {
209
- elements,
210
- resolution: {
211
- scope: 'screen',
212
- target: null,
213
- resolved: true,
214
- resolvedIndex: null,
215
- resolvedStableId: null,
216
- reason: 'screen scope'
217
- }
218
- };
219
- }
220
- const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
221
- if (!requestedTarget) {
222
- return {
223
- elements: [],
224
- resolution: {
225
- scope: 'subtree',
226
- target: null,
227
- resolved: false,
228
- resolvedIndex: null,
229
- resolvedStableId: null,
230
- reason: 'subtree scope requires a target element id'
231
- },
232
- error: {
233
- code: 'INVALID_SCOPE',
234
- message: 'scope=subtree requires a target element_id'
235
- }
236
- };
237
- }
238
- const resolved = ToolsInteract._findScopedElement(tree, requestedTarget);
239
- if (!resolved) {
240
- return {
241
- elements: [],
242
- resolution: {
243
- scope: 'subtree',
244
- target: requestedTarget,
245
- resolved: false,
246
- resolvedIndex: null,
247
- resolvedStableId: null,
248
- reason: 'target element could not be resolved'
249
- },
250
- error: {
251
- code: 'ELEMENT_NOT_FOUND',
252
- message: `Target element ${requestedTarget} could not be resolved for subtree scope`
253
- }
254
- };
255
- }
256
- const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index);
257
- const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
258
- return {
259
- elements: scopedElements,
260
- resolution: {
261
- scope: 'subtree',
262
- target: requestedTarget,
263
- resolved: true,
264
- resolvedIndex: resolved.index,
265
- resolvedStableId: resolved.stableId,
266
- reason: resolved.reason
267
- }
268
- };
269
- }
270
- static _findScopedElement(tree, targetElementId) {
271
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
272
- const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
273
- const deviceId = tree?.device?.id ?? undefined;
274
- const normalizedTarget = ToolsInteract._normalize(targetElementId);
275
- for (let i = 0; i < elements.length; i++) {
276
- const el = elements[i];
277
- if (!el)
278
- continue;
279
- const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i);
280
- if (computedElementId === targetElementId) {
281
- return {
282
- index: i,
283
- stableId: el.stable_id ?? null,
284
- reason: 'element_id_match'
285
- };
286
- }
287
- }
288
- for (let i = 0; i < elements.length; i++) {
289
- const el = elements[i];
290
- if (!el)
291
- continue;
292
- if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
293
- return {
294
- index: i,
295
- stableId: el.stable_id,
296
- reason: 'stable_id_match'
297
- };
298
- }
299
- }
300
- const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId);
301
- if (storedContext?.stable_id) {
302
- const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id);
303
- for (let i = 0; i < elements.length; i++) {
304
- const el = elements[i];
305
- if (!el?.stable_id)
306
- continue;
307
- if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
308
- return {
309
- index: i,
310
- stableId: el.stable_id,
311
- reason: 'stored_stable_id_match'
312
- };
313
- }
314
- }
315
- }
316
- return null;
317
- }
318
- static _collectSubtreeIndices(elements, rootIndex) {
319
- if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length)
320
- return [];
321
- const visited = new Set();
322
- const stack = [rootIndex];
323
- const result = [];
324
- while (stack.length > 0) {
325
- const index = stack.pop();
326
- if (index === undefined || visited.has(index) || index < 0 || index >= elements.length)
327
- continue;
328
- visited.add(index);
329
- result.push(index);
330
- const element = elements[index];
331
- if (!element)
332
- continue;
333
- const directChildren = new Set();
334
- if (Array.isArray(element.children)) {
335
- for (const childIndex of element.children) {
336
- if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
337
- directChildren.add(childIndex);
338
- }
339
- }
340
- }
341
- for (let i = 0; i < elements.length; i++) {
342
- if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
343
- directChildren.add(i);
344
- }
345
- }
346
- for (const childIndex of directChildren) {
347
- if (!visited.has(childIndex))
348
- stack.push(childIndex);
349
- }
350
- }
351
- return result.sort((left, right) => left - right);
55
+ return resolveUiChangeScope(tree, scope, target, (snapshot, id) => ToolsInteract._findScopedElement(snapshot, id));
352
56
  }
353
- static _changeIdentityForElement(el, index) {
354
- const stableId = ToolsInteract._normalize(el.stable_id);
355
- if (stableId)
356
- return `stable:${stableId}`;
357
- return `fallback:${ToolsInteract._hash({
358
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
359
- contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
360
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
361
- type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
362
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
363
- index
364
- })}`;
365
- }
366
- static _summarizeUiChangeDelta(initialElements, currentElements) {
367
- const buildMap = (elements) => {
368
- const map = new Map();
369
- for (let i = 0; i < elements.length; i++) {
370
- const element = elements[i];
371
- if (!element)
372
- continue;
373
- const key = ToolsInteract._changeIdentityForElement(element, i);
374
- map.set(key, ToolsInteract._hash({
375
- text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
376
- contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
377
- resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
378
- type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
379
- bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
380
- state: element.state ?? null,
381
- visible: !!element.visible,
382
- enabled: !!element.enabled,
383
- clickable: !!element.clickable
384
- }));
385
- }
386
- return map;
387
- };
388
- const initialMap = buildMap(initialElements);
389
- const currentMap = buildMap(currentElements);
390
- let added = 0;
391
- let removed = 0;
392
- let mutated = 0;
393
- for (const [key, value] of currentMap.entries()) {
394
- if (!initialMap.has(key)) {
395
- added++;
396
- }
397
- else if (initialMap.get(key) !== value) {
398
- mutated++;
399
- }
400
- }
401
- for (const key of initialMap.keys()) {
402
- if (!currentMap.has(key))
403
- removed++;
404
- }
405
- return {
406
- total_elements: currentElements.length,
407
- added_elements: added,
408
- removed_elements: removed,
409
- mutated_elements: mutated
410
- };
57
+ static _findScopedElement(tree, id) {
58
+ return findScopedElement(tree, id, ToolsInteract._resolvedUiElements);
411
59
  }
60
+ static _collectSubtreeIndices = _collectSubtreeIndices;
412
61
  static _rememberResolvedElement(elementId, context) {
413
- if (ToolsInteract._resolvedUiElements.has(elementId)) {
414
- ToolsInteract._resolvedUiElements.delete(elementId);
415
- }
416
- ToolsInteract._resolvedUiElements.set(elementId, context);
417
- while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
418
- const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value;
419
- if (!oldestElementId)
420
- break;
421
- ToolsInteract._resolvedUiElements.delete(oldestElementId);
422
- }
62
+ ToolsInteract._resolvedUiElements.remember(elementId, context);
423
63
  }
424
64
  static async _captureFingerprint(platform, deviceId) {
425
65
  if (isConditionBlocked())
@@ -432,402 +72,7 @@ export class ToolsInteract {
432
72
  return null;
433
73
  }
434
74
  }
435
- static _buildUiChangeSignatures(tree) {
436
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
437
- const textPayload = [];
438
- const statePayload = [];
439
- for (const el of elements) {
440
- textPayload.push({
441
- text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
442
- contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
443
- resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
444
- });
445
- statePayload.push({
446
- checked: el?.state?.checked ?? null,
447
- selected: el?.state?.selected ?? null,
448
- focused: el?.state?.focused ?? null,
449
- expanded: el?.state?.expanded ?? null,
450
- enabled: el?.state?.enabled ?? null,
451
- text_value: el?.state?.text_value ?? null,
452
- value: el?.state?.value ?? null,
453
- raw_value: el?.state?.raw_value ?? null,
454
- value_range: el?.state?.value_range ?? null
455
- });
456
- }
457
- return {
458
- hierarchy: computeSnapshotSignature(tree),
459
- text: ToolsInteract._hash({
460
- screen: ToolsInteract._normalize(tree?.screen),
461
- elements: textPayload
462
- }),
463
- state: ToolsInteract._hash({
464
- screen: ToolsInteract._normalize(tree?.screen),
465
- elements: statePayload
466
- })
467
- };
468
- }
469
- static _matchesUiChange(expected, initial, current) {
470
- const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds;
471
- for (const changeKind of candidates) {
472
- if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy)
473
- return changeKind;
474
- if (changeKind === 'text_change' && initial.text !== current.text)
475
- return changeKind;
476
- if (changeKind === 'state_change' && initial.state !== current.state)
477
- return changeKind;
478
- }
479
- return null;
480
- }
481
- static _uiChangeSignaturesEqual(left, right) {
482
- return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state;
483
- }
484
- static _resolvedTargetFromElement(elementId, element, index) {
485
- return {
486
- elementId,
487
- text: element.text ?? null,
488
- resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
489
- accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
490
- class: element.type ?? element.class ?? null,
491
- bounds: ToolsInteract._normalizeBounds(element.bounds),
492
- index,
493
- state: element.state ?? null,
494
- stable_id: element.stable_id ?? null,
495
- role: element.role ?? null,
496
- test_tag: element.test_tag ?? null,
497
- selector: element.selector ?? null,
498
- semantic: element.semantic ?? null
499
- };
500
- }
501
- static _summarizeResolutionCandidate(candidate) {
502
- const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds);
503
- return {
504
- text: candidate.el.text ?? null,
505
- resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
506
- accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
507
- class: candidate.el.type ?? candidate.el.class ?? null,
508
- bounds: bounds
509
- ? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
510
- : null,
511
- clickable: !!candidate.el.clickable,
512
- enabled: !!candidate.el.enabled,
513
- score: candidate.score,
514
- reason: candidate.reason
515
- };
516
- }
517
- static _isAdjustableControl(el) {
518
- if (!el)
519
- return false;
520
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
521
- const role = ToolsInteract._normalize(el.role ?? '');
522
- return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
523
- }
524
- static _isSemanticActionable(el) {
525
- if (!el?.semantic)
526
- return false;
527
- if (el.semantic.adjustable)
528
- return true;
529
- return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0;
530
- }
531
- static _readNumericControlValue(el, property) {
532
- if (!el?.state)
533
- return null;
534
- const stateValue = el.state[property];
535
- if (typeof stateValue === 'number' && Number.isFinite(stateValue))
536
- return stateValue;
537
- const parsedStateValue = ToolsInteract._parseNumericControlValue(stateValue);
538
- if (parsedStateValue !== null)
539
- return parsedStateValue;
540
- if (property === 'value' || property === 'raw_value') {
541
- const fallback = el.state.raw_value ?? el.state.value;
542
- if (typeof fallback === 'number' && Number.isFinite(fallback))
543
- return fallback;
544
- return ToolsInteract._parseNumericControlValue(fallback);
545
- }
546
- return null;
547
- }
548
- static _parseNumericControlValue(value) {
549
- if (typeof value === 'number' && Number.isFinite(value))
550
- return value;
551
- if (typeof value !== 'string')
552
- return null;
553
- const trimmed = value.trim();
554
- if (!trimmed)
555
- return null;
556
- const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/);
557
- if (percent)
558
- return Number(percent[1]);
559
- const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i);
560
- if (fraction)
561
- return Number(fraction[1]);
562
- const plain = Number(trimmed);
563
- if (Number.isFinite(plain))
564
- return plain;
565
- const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/);
566
- if (trailing)
567
- return Number(trailing[1]);
568
- return null;
569
- }
570
- static _readNumericControlRange(el) {
571
- const min = typeof el?.state?.value_range?.min === 'number' ? el.state.value_range.min : null;
572
- const max = typeof el?.state?.value_range?.max === 'number' ? el.state.value_range.max : null;
573
- if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
574
- return { min, max };
575
- }
576
- const value = el?.state?.value ?? el?.state?.raw_value;
577
- if (typeof value === 'string') {
578
- const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i);
579
- if (fraction) {
580
- const parsedMax = Number(fraction[1]);
581
- if (Number.isFinite(parsedMax) && parsedMax > 0)
582
- return { min: 0, max: parsedMax };
583
- }
584
- }
585
- return null;
586
- }
587
- static _readControlStep(el) {
588
- const candidates = [
589
- el?.state?.step,
590
- el?.state?.value_range?.step
591
- ];
592
- for (const candidate of candidates) {
593
- if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0)
594
- return candidate;
595
- }
596
- return null;
597
- }
598
- static _quantizeTargetValue(targetValue, min, max, step) {
599
- if (!step)
600
- return targetValue;
601
- const quantized = min + (Math.round((targetValue - min) / step) * step);
602
- const bounded = Math.max(min, Math.min(max, quantized));
603
- const precision = ToolsInteract._decimalPrecisionForStep(step);
604
- return Number(bounded.toFixed(precision));
605
- }
606
- static _decimalPrecisionForStep(step) {
607
- const normalized = String(step).toLowerCase();
608
- const [coefficient, exponentPart] = normalized.split('e');
609
- const coefficientDecimals = coefficient.includes('.')
610
- ? coefficient.split('.')[1]?.length ?? 0
611
- : 0;
612
- const exponent = exponentPart !== undefined ? Number(exponentPart) : 0;
613
- const precision = exponent < 0
614
- ? coefficientDecimals + Math.abs(exponent)
615
- : Math.max(0, coefficientDecimals - exponent);
616
- return Math.max(0, Math.min(12, precision));
617
- }
618
- static _defaultAdjustTolerance(range, step) {
619
- if (step)
620
- return 0;
621
- if (!range)
622
- return 0;
623
- return Math.max(0, 0.01 * Math.abs(range.max - range.min));
624
- }
625
- static _runtimeActionSet(el) {
626
- const values = [
627
- ...(Array.isArray(el?.actions) ? el.actions : []),
628
- ...(Array.isArray(el?.accessibilityActions) ? el.accessibilityActions : []),
629
- ...(Array.isArray(el?.runtimeActions) ? el.runtimeActions : []),
630
- ...(Array.isArray(el?.platformActions) ? el.platformActions : [])
631
- ];
632
- return new Set(values.map((value) => ToolsInteract._normalize(value)).filter(Boolean));
633
- }
634
- static _hasDirectAdjustmentAction(el) {
635
- const actions = ToolsInteract._runtimeActionSet(el);
636
- return [
637
- 'action_set_progress',
638
- 'set_progress',
639
- 'setprogress',
640
- 'semanticsactions.setprogress',
641
- 'set_value',
642
- 'setvalue',
643
- 'direct_set_value'
644
- ].some((action) => actions.has(action));
645
- }
646
- static _adjustmentDirections(el) {
647
- const actions = ToolsInteract._runtimeActionSet(el);
648
- return {
649
- increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
650
- decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
651
- };
652
- }
653
- static _textFromElement(el) {
654
- return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim();
655
- }
656
- static _parseLeadingNumber(text) {
657
- const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/);
658
- if (!match)
659
- return null;
660
- const value = Number(match[1]);
661
- return Number.isFinite(value) ? value : null;
662
- }
663
- static _inferLabeledSliderState(elements, track) {
664
- const trackBounds = ToolsInteract._normalizeBounds(track?.bounds);
665
- if (!track || !trackBounds)
666
- return null;
667
- const type = ToolsInteract._normalize(track.type ?? track.class ?? '');
668
- const role = ToolsInteract._normalize(track.role ?? '');
669
- const width = trackBounds[2] - trackBounds[0];
670
- const height = trackBounds[3] - trackBounds[1];
671
- const wideTrackLike = width >= 240 && height <= 220;
672
- const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role);
673
- if (!wideTrackLike && !knownTrackLike)
674
- return null;
675
- const trackTop = trackBounds[1];
676
- const trackBottom = trackBounds[3];
677
- const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2;
678
- const labels = elements
679
- .map((el, index) => {
680
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
681
- const text = ToolsInteract._textFromElement(el);
682
- const value = ToolsInteract._parseLeadingNumber(text);
683
- return bounds && text && value !== null
684
- ? {
685
- index,
686
- el,
687
- text,
688
- value,
689
- bounds,
690
- centerX: (bounds[0] + bounds[2]) / 2,
691
- centerY: (bounds[1] + bounds[3]) / 2,
692
- hasUnit: /[a-zA-Z%]/.test(text)
693
- }
694
- : null;
695
- })
696
- .filter((label) => !!label);
697
- const rangeLabels = labels
698
- .filter((label) => {
699
- const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260;
700
- const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120;
701
- return verticallyNearTrack && horizontallyNearTrack && label.hasUnit;
702
- })
703
- .sort((a, b) => a.centerX - b.centerX);
704
- if (rangeLabels.length < 2)
705
- return null;
706
- const minLabel = rangeLabels[0];
707
- const maxLabel = rangeLabels[rangeLabels.length - 1];
708
- if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value)
709
- return null;
710
- const currentCandidates = labels
711
- .filter((label) => {
712
- if (label.index === minLabel.index || label.index === maxLabel.index)
713
- return false;
714
- if (label.value < minLabel.value || label.value > maxLabel.value)
715
- return false;
716
- if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20)
717
- return false;
718
- return label.centerX >= trackCenterX - 80;
719
- })
720
- .sort((a, b) => {
721
- const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY);
722
- if (yDistance !== 0)
723
- return yDistance;
724
- return b.centerX - a.centerX;
725
- });
726
- const currentLabel = currentCandidates[0];
727
- if (!currentLabel)
728
- return null;
729
- const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop;
730
- const labelDerivedBounds = [
731
- Math.min(minLabel.bounds[0], trackBounds[0]),
732
- Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
733
- Math.max(maxLabel.bounds[2], trackBounds[2]),
734
- Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
735
- ];
736
- return {
737
- current: currentLabel.value,
738
- min: minLabel.value,
739
- max: maxLabel.value,
740
- value_range: { min: minLabel.value, max: maxLabel.value },
741
- touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
742
- source: 'nearby_labels'
743
- };
744
- }
745
- static _buildControlPoint(bounds, ratio, axis) {
746
- const clampedRatio = Math.max(0, Math.min(1, ratio));
747
- const [left, top, right, bottom] = bounds;
748
- const width = Math.max(1, right - left);
749
- const height = Math.max(1, bottom - top);
750
- const insetX = Math.max(8, Math.floor(width * 0.08));
751
- const insetY = Math.max(8, Math.floor(height * 0.08));
752
- if (axis === 'vertical') {
753
- const usableHeight = Math.max(1, height - (insetY * 2));
754
- return {
755
- x: Math.floor((left + right) / 2),
756
- y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
757
- };
758
- }
759
- const usableWidth = Math.max(1, width - (insetX * 2));
760
- return {
761
- x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
762
- y: Math.floor((top + bottom) / 2)
763
- };
764
- }
765
- static _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis) {
766
- const range = Math.max(1, max - min);
767
- const targetRatio = (targetValue - min) / range;
768
- const stepRatio = 1 / range;
769
- const centerBias = stepRatio / 2;
770
- const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue);
771
- const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0]);
772
- const edgeWindow = Math.max(3, Math.floor(range * 0.1));
773
- const isNearLowEdge = targetValue - min <= edgeWindow;
774
- const isNearHighEdge = max - targetValue <= edgeWindow;
775
- const directionBias = direction > 0
776
- ? -stepRatio * 0.15
777
- : direction < 0
778
- ? stepRatio * 0.65
779
- : 0;
780
- const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx));
781
- const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin);
782
- const edgeBias = isNearLowEdge
783
- ? endpointMargin
784
- : isNearHighEdge
785
- ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
786
- : 0;
787
- const safeRatio = Math.min(1 - (endpointMargin * 0.25), Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias));
788
- return ToolsInteract._buildControlPoint(bounds, safeRatio, axis);
789
- }
790
- static _buildAdjustmentProbePoints(bounds, targetValue, currentValue, min, max, axis) {
791
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis);
792
- const currentPoint = currentValue !== null
793
- ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
794
- : ToolsInteract._buildControlPoint(bounds, 0.5, axis);
795
- const [left, top, right, bottom] = bounds;
796
- const width = Math.max(1, right - left);
797
- const height = Math.max(1, bottom - top);
798
- const crossAxisBumps = axis === 'horizontal'
799
- ? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
800
- : [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))];
801
- const clampPoint = (point) => ({
802
- x: axis === 'horizontal'
803
- ? Math.max(left, Math.min(right, point.x))
804
- : Math.max(left, Math.min(right + Math.max(width, height), point.x)),
805
- y: axis === 'vertical'
806
- ? Math.max(top, Math.min(bottom, point.y))
807
- : Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
808
- });
809
- const probes = [targetPoint, currentPoint];
810
- for (const bump of crossAxisBumps) {
811
- if (axis === 'horizontal') {
812
- probes.push({ x: targetPoint.x, y: bottom + bump }, { x: currentPoint.x, y: bottom + bump });
813
- }
814
- else {
815
- probes.push({ x: right + bump, y: targetPoint.y }, { x: right + bump, y: currentPoint.y });
816
- }
817
- }
818
- return Array.from(new Map(probes
819
- .map(clampPoint)
820
- .map((point) => [`${point.x}:${point.y}`, point])).values());
821
- }
822
- static _controlAxis(el, bounds) {
823
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
824
- const role = ToolsInteract._normalize(el.role ?? '');
825
- if (/vertical/.test(type) || /vertical/.test(role))
826
- return 'vertical';
827
- if (/horizontal/.test(type) || /horizontal/.test(role))
828
- return 'horizontal';
829
- return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal';
830
- }
75
+ static _resolvedTargetFromElement = _resolvedTargetFromElement;
831
76
  static _actionFailure(actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter, sourceModule = 'interact') {
832
77
  return buildActionExecutionResult({
833
78
  actionType,
@@ -850,111 +95,11 @@ export class ToolsInteract {
850
95
  }
851
96
  return null;
852
97
  }
853
- static _resolveActionableAncestor(elements, chosen) {
854
- if (!chosen)
855
- return null;
856
- if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el))
857
- return chosen;
858
- let current = chosen;
859
- let safety = 0;
860
- while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
861
- const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId);
862
- if (parentIndex !== null && elements[parentIndex]) {
863
- current = { el: elements[parentIndex], idx: parentIndex };
864
- if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
865
- return current;
866
- }
867
- else {
868
- break;
869
- }
870
- safety++;
871
- }
872
- const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
873
- if (!childBounds)
874
- return null;
875
- const [cl, ct, cr, cb] = childBounds;
876
- let best = null;
877
- let bestArea = Infinity;
878
- for (let i = 0; i < elements.length; i++) {
879
- const el = elements[i];
880
- if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el)))
881
- continue;
882
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
883
- if (!bounds)
884
- continue;
885
- const [pl, pt, pr, pb] = bounds;
886
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
887
- const area = (pr - pl) * (pb - pt);
888
- if (area < bestArea) {
889
- bestArea = area;
890
- best = { el, idx: i };
891
- }
892
- }
893
- }
894
- return best;
895
- }
896
- static _resolveNearbyActionableControl(elements, chosen, screen) {
897
- if (!chosen)
898
- return null;
899
- const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
900
- if (!labelBounds)
901
- return null;
902
- const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds;
903
- const labelWidth = labelRight - labelLeft;
904
- const labelHeight = labelBottom - labelTop;
905
- const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0;
906
- const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0;
907
- const minTrackLengthPx = Math.max(ToolsInteract._sliderTrackMinLengthPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0);
908
- const maxTrackThicknessPx = Math.max(ToolsInteract._sliderTrackMaxThicknessPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0);
909
- let best = null;
910
- let bestScore = Infinity;
911
- for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
912
- const candidate = elements[i];
913
- if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false)
914
- continue;
915
- const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds);
916
- if (!candidateBounds)
917
- continue;
918
- const [left, top, right] = candidateBounds;
919
- const width = right - left;
920
- const height = candidateBounds[3] - top;
921
- const verticalGap = top - labelBottom;
922
- if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx)
923
- continue;
924
- const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left);
925
- if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx)
926
- continue;
927
- const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '');
928
- const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '');
929
- const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '');
930
- let score = verticalGap;
931
- const horizontalTrackLike = width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
932
- height <= maxTrackThicknessPx;
933
- const verticalTrackLike = height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
934
- width <= maxTrackThicknessPx;
935
- const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike;
936
- if (!candidateText && !candidateContent)
937
- score -= 18;
938
- if (trackLike)
939
- score -= 30;
940
- if (/view|layout|group|frame/i.test(candidateClass))
941
- score -= 10;
942
- if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio)
943
- score -= 8;
944
- if (candidateText || candidateContent)
945
- score += 20;
946
- if (score < bestScore) {
947
- bestScore = score;
948
- best = { el: candidate, idx: i, sliderLike: trackLike };
949
- }
950
- }
951
- return best;
952
- }
953
98
  static async getInteractionService(platform, deviceId) {
954
99
  const effectivePlatform = platform || 'android';
955
100
  const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
956
101
  const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract();
957
- return { interact: interact, resolved, platform: effectivePlatform };
102
+ return { interact, resolved, platform: effectivePlatform };
958
103
  }
959
104
  static async tapHandler({ platform, x, y, deviceId, timeoutMs }) {
960
105
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
@@ -969,24 +114,14 @@ export class ToolsInteract {
969
114
  }
970
115
  const fingerprintBefore = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null;
971
116
  const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
972
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform;
973
- const treeDeviceId = tree?.device?.id || resolved.deviceId;
974
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
975
- const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
976
- if (!currentMatch) {
977
- return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
978
- }
979
- const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
980
- const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform);
981
- if (!tapActionability.actionable) {
982
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
983
- }
984
- const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
985
- if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
986
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
987
- }
988
- const x = Math.floor((bounds[0] + bounds[2]) / 2);
989
- const y = Math.floor((bounds[1] + bounds[3]) / 2);
117
+ const tap = resolveTap(tree, resolved, {
118
+ current: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
119
+ target: (element, index) => ToolsInteract._resolvedTargetFromElement(resolved.elementId, element, index)
120
+ });
121
+ if ('failure' in tap)
122
+ return ToolsInteract._actionFailure(actionType, selector, tap.target, tap.failure, true, fingerprintBefore);
123
+ const resolvedTarget = tap.target;
124
+ const { x, y } = tap.point;
990
125
  const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId, timeoutMs });
991
126
  if (!tapResult.success) {
992
127
  const fingerprintAfterFailure = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null;
@@ -1004,763 +139,17 @@ export class ToolsInteract {
1004
139
  sourceModule: 'interact'
1005
140
  });
1006
141
  }
1007
- static async adjustControlHandler({ selector, element_id, property = 'value', targetValue, tolerance, maxAttempts, platform, deviceId }) {
1008
- const actionType = 'adjust_control';
1009
- const targetSelector = selector ?? (element_id ? { elementId: element_id } : null);
1010
- const requestedTolerance = typeof tolerance === 'number' && Number.isFinite(tolerance) ? Math.max(0, tolerance) : null;
1011
- const attemptsLimit = maxAttempts === undefined || maxAttempts === null
1012
- ? 5
1013
- : Math.max(1, Math.floor(Number(maxAttempts) || 1));
1014
- const sourcePlatform = platform || 'android';
1015
- let resolvedPlatform = sourcePlatform;
1016
- let resolvedDeviceId = deviceId;
1017
- const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null;
1018
- const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1019
- let semanticFallbackElement = null;
1020
- const traceSteps = [];
1021
- let traceAttemptIndex = 0;
1022
- const recordTraceStep = (stage, result, metadata) => {
1023
- traceSteps.push(createTraceStep({
1024
- stage,
1025
- timestamp: Date.now(),
1026
- result,
1027
- attemptIndex: traceAttemptIndex++,
1028
- metadata
1029
- }));
1030
- };
1031
- const buildFailure = (failureCode, reason, resolved, device, actualState, attempts, adjustmentMode = 'gesture', retryable = false, uiFingerprintAfter = null) => {
1032
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1033
- recordTraceStep('resolve', 'failure', {
1034
- reason,
1035
- failure_code: failureCode
1036
- });
1037
- }
1038
- if (!traceSteps.some((step) => step.stage === 'recover')) {
1039
- recordTraceStep('recover', retryable ? 'retry' : 'failure', {
1040
- reason,
1041
- failure_code: failureCode,
1042
- retry_allowed: retryable,
1043
- recovery_attempts: attempts,
1044
- retry_depth: attempts
1045
- });
1046
- }
1047
- const responseTolerance = requestedTolerance ?? 0;
1048
- const base = buildActionExecutionResult({
1049
- actionType,
1050
- sourceModule: 'interact',
1051
- device,
1052
- selector: targetSelector,
1053
- resolved,
1054
- success: false,
1055
- uiFingerprintBefore: fingerprintBefore,
1056
- uiFingerprintAfter,
1057
- failure: { failureCode, retryable },
1058
- details: {
1059
- target_value: targetValue,
1060
- tolerance: responseTolerance,
1061
- property,
1062
- attempts,
1063
- adjustment_mode: adjustmentMode,
1064
- actual_state: actualState,
1065
- converged: false,
1066
- within_tolerance: false,
1067
- reason
1068
- },
1069
- traceSteps
1070
- });
1071
- return {
1072
- ...base,
1073
- target_state: {
1074
- property,
1075
- target_value: targetValue,
1076
- tolerance: responseTolerance
1077
- },
1078
- actual_state: actualState,
1079
- within_tolerance: false,
1080
- converged: false,
1081
- attempts,
1082
- adjustment_mode: adjustmentMode
1083
- };
1084
- };
1085
- const resolveCurrentMatch = async () => {
1086
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
1087
- resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform;
1088
- resolvedDeviceId = tree?.device?.id || resolvedDeviceId;
1089
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
1090
- if (element_id) {
1091
- const stored = ToolsInteract._resolvedUiElements.get(element_id);
1092
- if (!stored) {
1093
- return null;
1094
- }
1095
- const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored);
1096
- if (!current) {
1097
- return null;
1098
- }
1099
- return {
1100
- tree,
1101
- device: tree?.device,
1102
- match: { el: current.el, idx: current.index },
1103
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index), current.el, current.index)
1104
- };
1105
- }
1106
- if (semanticFallbackElement) {
1107
- const fallbackBounds = ToolsInteract._normalizeBounds(Array.isArray(semanticFallbackElement.bounds)
1108
- ? semanticFallbackElement.bounds
1109
- : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
1110
- ? [
1111
- Number(semanticFallbackElement.bounds.left),
1112
- Number(semanticFallbackElement.bounds.top),
1113
- Number(semanticFallbackElement.bounds.right),
1114
- Number(semanticFallbackElement.bounds.bottom)
1115
- ]
1116
- : null);
1117
- let matchedIndex = -1;
1118
- if (fallbackBounds) {
1119
- matchedIndex = elements.findIndex((el) => {
1120
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
1121
- return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3];
1122
- });
1123
- }
1124
- if (matchedIndex === -1 && fallbackBounds) {
1125
- const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2);
1126
- const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2);
1127
- let bestDistance = Infinity;
1128
- for (let i = 0; i < elements.length; i++) {
1129
- const el = elements[i];
1130
- if (!ToolsInteract._isAdjustableControl(el))
1131
- continue;
1132
- const bounds = ToolsInteract._normalizeBounds(el.bounds);
1133
- if (!bounds)
1134
- continue;
1135
- const centerX = Math.floor((bounds[0] + bounds[2]) / 2);
1136
- const centerY = Math.floor((bounds[1] + bounds[3]) / 2);
1137
- const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY);
1138
- if (distance < bestDistance) {
1139
- bestDistance = distance;
1140
- matchedIndex = i;
1141
- }
1142
- }
1143
- }
1144
- if (matchedIndex >= 0 && elements[matchedIndex]) {
1145
- const matched = { el: elements[matchedIndex], idx: matchedIndex };
1146
- return {
1147
- tree,
1148
- device: tree?.device,
1149
- match: matched,
1150
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
1151
- };
1152
- }
1153
- }
1154
- if (selector) {
1155
- const matched = ToolsInteract._findFirstMatchingElement(elements, selector);
1156
- if (!matched) {
1157
- return null;
1158
- }
1159
- return {
1160
- tree,
1161
- device: tree?.device,
1162
- match: matched,
1163
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
1164
- };
1165
- }
1166
- return null;
1167
- };
1168
- if ((!selector && !element_id) || (selector && element_id)) {
1169
- return buildFailure('ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', null, undefined, null, 0, 'gesture', false);
1170
- }
1171
- if (selector && !element_id) {
1172
- const waitResult = await ToolsInteract.waitForUIHandler({
1173
- selector,
1174
- condition: 'clickable',
1175
- timeout_ms: 5000,
1176
- poll_interval_ms: 300,
1177
- platform: resolvedPlatform,
1178
- deviceId: resolvedDeviceId
1179
- });
1180
- if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
1181
- const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? '';
1182
- if (!semanticQuery) {
1183
- return buildFailure(waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', waitResult?.error?.code === 'ELEMENT_NOT_FOUND');
1184
- }
1185
- const fallback = await ToolsInteract.findElementHandler({
1186
- query: semanticQuery,
1187
- exact: false,
1188
- timeoutMs: 3000,
1189
- platform: resolvedPlatform,
1190
- deviceId: resolvedDeviceId
1191
- });
1192
- if (!fallback.found || !fallback.element) {
1193
- return buildFailure('ELEMENT_NOT_FOUND', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', true);
1194
- }
1195
- semanticFallbackElement = fallback.element;
1196
- }
1197
- else {
1198
- element_id = waitResult.element.elementId;
1199
- semanticFallbackElement = null;
1200
- }
1201
- }
1202
- let lastObservedState = null;
1203
- let lastAdjustmentMode = 'gesture';
1204
- let resolvedTarget = null;
1205
- let currentDevice = undefined;
1206
- let attemptCount = 0;
1207
- let cachedResolvedMatch = null;
1208
- let coordinateTapsAwaitingFreshGesture = false;
1209
- adjustmentAttempts: for (let attempt = 0; attempt < attemptsLimit && attemptCount < attemptsLimit; attempt++) {
1210
- const resolved = cachedResolvedMatch
1211
- ? {
1212
- tree: null,
1213
- device: currentDevice,
1214
- match: cachedResolvedMatch,
1215
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx), cachedResolvedMatch.el, cachedResolvedMatch.idx)
1216
- }
1217
- : await resolveCurrentMatch();
1218
- if (!resolved || !resolved.match || !resolved.resolvedTarget) {
1219
- return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
1220
- }
1221
- currentDevice = resolved.device;
1222
- resolvedTarget = resolved.resolvedTarget;
1223
- const currentEl = resolved.match.el;
1224
- cachedResolvedMatch = resolved.match;
1225
- const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id);
1226
- if (!adjustableActionability.actionable) {
1227
- return buildFailure(adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', adjustableActionability.reason ?? 'adjustable control is not actionable', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
1228
- }
1229
- const elementsForInference = Array.isArray(resolved.tree?.elements)
1230
- ? resolved.tree.elements
1231
- : cachedResolvedMatch
1232
- ? [cachedResolvedMatch.el]
1233
- : [currentEl];
1234
- const inferredSliderState = ToolsInteract._inferLabeledSliderState(elementsForInference, currentEl);
1235
- const nativeBounds = ToolsInteract._normalizeBounds(currentEl.bounds);
1236
- const bounds = inferredSliderState?.touch_bounds ?? nativeBounds;
1237
- const valueRange = ToolsInteract._readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null;
1238
- const step = ToolsInteract._readControlStep(currentEl);
1239
- const normalizedTargetValue = valueRange
1240
- ? ToolsInteract._quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step)
1241
- : targetValue;
1242
- const effectiveTolerance = requestedTolerance ?? ToolsInteract._defaultAdjustTolerance(valueRange, step);
1243
- const currentValue = ToolsInteract._readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null;
1244
- const actualState = currentValue !== null
1245
- ? {
1246
- property,
1247
- value: currentValue,
1248
- raw_value: typeof currentEl.state?.raw_value === 'number'
1249
- ? currentEl.state.raw_value
1250
- : inferredSliderState?.current
1251
- }
1252
- : null;
1253
- lastObservedState = actualState;
1254
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1255
- recordTraceStep('resolve', 'success', {
1256
- resolved_target: resolvedTarget,
1257
- current_value: currentValue,
1258
- adjustment_mode: lastAdjustmentMode
1259
- });
1260
- }
1261
- if (property !== 'value' && property !== 'raw_value') {
1262
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1263
- }
1264
- const isAdjustableControl = ToolsInteract._isAdjustableControl(currentEl) || !!inferredSliderState;
1265
- if (!isAdjustableControl) {
1266
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1267
- }
1268
- if (currentValue === null) {
1269
- return buildFailure('UNKNOWN', 'current numeric value is unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1270
- }
1271
- if (valueRange && (targetValue < valueRange.min || targetValue > valueRange.max)) {
1272
- return buildFailure('CONTROL_CONVERGENCE_FAILED', `targetValue ${targetValue} is outside the control range ${valueRange.min}..${valueRange.max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1273
- }
1274
- if (currentValue !== null && Math.abs(currentValue - normalizedTargetValue) <= effectiveTolerance) {
1275
- recordTraceStep('verify', 'success', {
1276
- property,
1277
- target_value: normalizedTargetValue,
1278
- actual_state: actualState,
1279
- reason: 'control already within tolerance'
1280
- });
1281
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1282
- const base = buildActionExecutionResult({
1283
- actionType,
1284
- sourceModule: 'interact',
1285
- device: currentDevice,
1286
- selector: targetSelector,
1287
- resolved: resolvedTarget,
1288
- success: true,
1289
- uiFingerprintBefore: fingerprintBefore,
1290
- uiFingerprintAfter,
1291
- details: {
1292
- target_value: normalizedTargetValue,
1293
- tolerance: effectiveTolerance,
1294
- property,
1295
- attempts: attemptCount,
1296
- adjustment_mode: 'semantic',
1297
- actual_state: actualState,
1298
- converged: true,
1299
- within_tolerance: true,
1300
- reason: 'control already within tolerance'
1301
- },
1302
- traceSteps
1303
- });
1304
- return {
1305
- ...base,
1306
- target_state: {
1307
- property,
1308
- target_value: normalizedTargetValue,
1309
- tolerance: effectiveTolerance
1310
- },
1311
- actual_state: actualState,
1312
- within_tolerance: true,
1313
- converged: true,
1314
- attempts: attemptCount,
1315
- adjustment_mode: 'semantic'
1316
- };
1317
- }
1318
- if (!bounds) {
1319
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1320
- }
1321
- const runVerification = async () => {
1322
- if (inferredSliderState) {
1323
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
1324
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
1325
- const refreshed = ToolsInteract._inferLabeledSliderState(elements, currentEl);
1326
- const observedValue = refreshed?.current ?? null;
1327
- const observedState = observedValue !== null
1328
- ? { property, value: observedValue, raw_value: observedValue }
1329
- : actualState;
1330
- return {
1331
- verification: {
1332
- success: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance,
1333
- observed_state: observedState,
1334
- reason: observedValue !== null
1335
- ? 'visible labeled value read from nearby slider labels'
1336
- : 'visible labeled value unavailable'
1337
- },
1338
- observedState,
1339
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1340
- };
1341
- }
1342
- const verification = await ToolsInteract.expectStateHandler({
1343
- element_id: resolvedTarget?.elementId ?? element_id,
1344
- selector: selector ?? undefined,
1345
- property,
1346
- expected: normalizedTargetValue,
1347
- platform: resolvedPlatform,
1348
- deviceId: resolvedDeviceId
1349
- });
1350
- const observedValue = ToolsInteract._parseNumericControlValue(verification?.observed_state?.value) ??
1351
- ToolsInteract._parseNumericControlValue(verification?.observed_state?.raw_value);
1352
- const observedState = observedValue !== null
1353
- ? {
1354
- property,
1355
- value: observedValue,
1356
- raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
1357
- }
1358
- : actualState;
1359
- return {
1360
- verification,
1361
- observedState,
1362
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1363
- };
1364
- };
1365
- let actionDevice = currentDevice;
1366
- let observedState = actualState;
1367
- let verification = null;
1368
- let verificationResult = { verification: null, observedState: actualState, withinTolerance: false };
1369
- const directActionSupported = ToolsInteract._hasDirectAdjustmentAction(currentEl);
1370
- if (directActionSupported && attemptCount < attemptsLimit) {
1371
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId);
1372
- if (typeof interact.setAdjustableValue === 'function') {
1373
- lastAdjustmentMode = 'semantic';
1374
- recordTraceStep('execute', 'retry', {
1375
- attempt: attemptCount + 1,
1376
- mode: 'semantic',
1377
- strategy: 'direct_set_value',
1378
- target_value: normalizedTargetValue
1379
- });
1380
- const actionResult = await interact.setAdjustableValue({
1381
- deviceId: actionResolved.id,
1382
- target: resolvedTarget,
1383
- property,
1384
- value: normalizedTargetValue
1385
- });
1386
- attemptCount++;
1387
- actionDevice = actionResult?.device ?? actionDevice;
1388
- if (actionResult?.success !== false) {
1389
- verificationResult = await runVerification();
1390
- observedState = verificationResult.observedState;
1391
- lastObservedState = observedState;
1392
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1393
- attempt: attemptCount,
1394
- property,
1395
- target_value: normalizedTargetValue,
1396
- actual_state: observedState,
1397
- reason: verificationResult.verification?.reason ?? 'direct adjustment did not converge yet'
1398
- });
1399
- if (verificationResult.withinTolerance) {
1400
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1401
- const base = buildActionExecutionResult({
1402
- actionType,
1403
- sourceModule: 'interact',
1404
- device: actionDevice ?? currentDevice,
1405
- selector: targetSelector,
1406
- resolved: resolvedTarget,
1407
- success: true,
1408
- uiFingerprintBefore: fingerprintBefore,
1409
- uiFingerprintAfter,
1410
- details: {
1411
- target_value: normalizedTargetValue,
1412
- tolerance: effectiveTolerance,
1413
- property,
1414
- attempts: attemptCount,
1415
- adjustment_mode: lastAdjustmentMode,
1416
- actual_state: observedState,
1417
- converged: true,
1418
- within_tolerance: true,
1419
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1420
- },
1421
- traceSteps
1422
- });
1423
- return {
1424
- ...base,
1425
- target_state: {
1426
- property,
1427
- target_value: normalizedTargetValue,
1428
- tolerance: effectiveTolerance
1429
- },
1430
- actual_state: observedState,
1431
- within_tolerance: true,
1432
- converged: true,
1433
- attempts: attemptCount,
1434
- adjustment_mode: lastAdjustmentMode
1435
- };
1436
- }
1437
- cachedResolvedMatch = null;
1438
- if (attemptCount < attemptsLimit) {
1439
- continue;
1440
- }
1441
- }
1442
- }
1443
- }
1444
- const directions = ToolsInteract._adjustmentDirections(currentEl);
1445
- if ((directions.increment || directions.decrement) && attemptCount < attemptsLimit) {
1446
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId);
1447
- const adjustByDirection = typeof interact.adjustAccessibleValue === 'function'
1448
- ? interact.adjustAccessibleValue.bind(interact)
1449
- : null;
1450
- while (adjustByDirection && attemptCount < attemptsLimit) {
1451
- const direction = normalizedTargetValue > (observedState?.value ?? currentValue) ? 'increment' : 'decrement';
1452
- if ((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement))
1453
- break;
1454
- lastAdjustmentMode = 'semantic';
1455
- recordTraceStep('execute', 'retry', {
1456
- attempt: attemptCount + 1,
1457
- mode: 'semantic',
1458
- strategy: 'accessibility_increment_decrement',
1459
- direction
1460
- });
1461
- const actionResult = await adjustByDirection({
1462
- deviceId: actionResolved.id,
1463
- target: resolvedTarget,
1464
- property,
1465
- direction
1466
- });
1467
- attemptCount++;
1468
- actionDevice = actionResult?.device ?? actionDevice;
1469
- if (actionResult?.success === false)
1470
- break;
1471
- verificationResult = await runVerification();
1472
- observedState = verificationResult.observedState;
1473
- lastObservedState = observedState;
1474
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1475
- attempt: attemptCount,
1476
- property,
1477
- target_value: normalizedTargetValue,
1478
- actual_state: observedState,
1479
- reason: verificationResult.verification?.reason ?? 'incremental adjustment did not converge yet'
1480
- });
1481
- if (verificationResult.withinTolerance) {
1482
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1483
- const base = buildActionExecutionResult({
1484
- actionType,
1485
- sourceModule: 'interact',
1486
- device: actionDevice ?? currentDevice,
1487
- selector: targetSelector,
1488
- resolved: resolvedTarget,
1489
- success: true,
1490
- uiFingerprintBefore: fingerprintBefore,
1491
- uiFingerprintAfter,
1492
- details: {
1493
- target_value: normalizedTargetValue,
1494
- tolerance: effectiveTolerance,
1495
- property,
1496
- attempts: attemptCount,
1497
- adjustment_mode: lastAdjustmentMode,
1498
- actual_state: observedState,
1499
- converged: true,
1500
- within_tolerance: true,
1501
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1502
- },
1503
- traceSteps
1504
- });
1505
- return {
1506
- ...base,
1507
- target_state: {
1508
- property,
1509
- target_value: normalizedTargetValue,
1510
- tolerance: effectiveTolerance
1511
- },
1512
- actual_state: observedState,
1513
- within_tolerance: true,
1514
- converged: true,
1515
- attempts: attemptCount,
1516
- adjustment_mode: lastAdjustmentMode
1517
- };
1518
- }
1519
- cachedResolvedMatch = null;
1520
- if (attemptCount < attemptsLimit) {
1521
- continue adjustmentAttempts;
1522
- }
1523
- }
1524
- }
1525
- if (!bounds) {
1526
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1527
- }
1528
- if (!valueRange) {
1529
- return buildFailure('UNKNOWN', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
1530
- }
1531
- const min = valueRange.min;
1532
- const max = valueRange.max;
1533
- const axis = ToolsInteract._controlAxis(currentEl, bounds);
1534
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, normalizedTargetValue, currentValue, min, max, axis);
1535
- const currentPoint = ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis);
1536
- if (!coordinateTapsAwaitingFreshGesture) {
1537
- const probePoints = inferredSliderState
1538
- ? [targetPoint]
1539
- : ToolsInteract._buildAdjustmentProbePoints(bounds, normalizedTargetValue, currentValue, min, max, axis);
1540
- for (let i = 0; i < probePoints.length && attemptCount < attemptsLimit; i++) {
1541
- const probePoint = probePoints[i];
1542
- lastAdjustmentMode = 'coordinate';
1543
- recordTraceStep('execute', 'retry', {
1544
- attempt: attemptCount + 1,
1545
- mode: 'coordinate',
1546
- point: probePoint
1547
- });
1548
- const actionResult = await ToolsInteract.tapHandler({
1549
- platform: resolvedPlatform,
1550
- x: probePoint.x,
1551
- y: probePoint.y,
1552
- deviceId: resolvedDeviceId
1553
- });
1554
- attemptCount++;
1555
- actionDevice = actionResult.device ?? actionDevice;
1556
- if (!actionResult.success) {
1557
- recordTraceStep('execute', 'retry', {
1558
- attempt: attemptCount,
1559
- mode: 'coordinate',
1560
- point: probePoint,
1561
- success: false
1562
- });
1563
- continue;
1564
- }
1565
- verificationResult = await runVerification();
1566
- observedState = verificationResult.observedState;
1567
- lastObservedState = observedState;
1568
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1569
- attempt: attemptCount,
1570
- property,
1571
- target_value: normalizedTargetValue,
1572
- actual_state: observedState,
1573
- reason: verificationResult.verification?.reason ?? 'control did not converge yet'
1574
- });
1575
- if (verificationResult.withinTolerance) {
1576
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1577
- const base = buildActionExecutionResult({
1578
- actionType,
1579
- sourceModule: 'interact',
1580
- device: actionDevice ?? currentDevice,
1581
- selector: targetSelector,
1582
- resolved: resolvedTarget,
1583
- success: true,
1584
- uiFingerprintBefore: fingerprintBefore,
1585
- uiFingerprintAfter,
1586
- details: {
1587
- target_value: normalizedTargetValue,
1588
- tolerance: effectiveTolerance,
1589
- property,
1590
- attempts: attemptCount,
1591
- adjustment_mode: lastAdjustmentMode,
1592
- actual_state: observedState,
1593
- converged: true,
1594
- within_tolerance: true,
1595
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1596
- },
1597
- traceSteps
1598
- });
1599
- return {
1600
- ...base,
1601
- target_state: {
1602
- property,
1603
- target_value: normalizedTargetValue,
1604
- tolerance: effectiveTolerance
1605
- },
1606
- actual_state: observedState,
1607
- within_tolerance: true,
1608
- converged: true,
1609
- attempts: attemptCount,
1610
- adjustment_mode: lastAdjustmentMode
1611
- };
1612
- }
1613
- coordinateTapsAwaitingFreshGesture = true;
1614
- cachedResolvedMatch = null;
1615
- if (attemptCount < attemptsLimit) {
1616
- continue adjustmentAttempts;
1617
- }
1618
- }
1619
- }
1620
- if (attemptCount < attemptsLimit) {
1621
- coordinateTapsAwaitingFreshGesture = false;
1622
- lastAdjustmentMode = 'gesture';
1623
- recordTraceStep('execute', 'retry', {
1624
- attempt: attemptCount + 1,
1625
- mode: 'gesture',
1626
- start: currentPoint,
1627
- end: targetPoint
1628
- });
1629
- const fallbackActionResult = await ToolsInteract.swipeHandler({
1630
- platform: resolvedPlatform,
1631
- x1: currentPoint.x,
1632
- y1: currentPoint.y,
1633
- x2: targetPoint.x,
1634
- y2: targetPoint.y,
1635
- duration: inferredSliderState ? 520 : 220,
1636
- deviceId: resolvedDeviceId
1637
- });
1638
- attemptCount++;
1639
- if (!fallbackActionResult.success) {
1640
- recordTraceStep('execute', 'failure', {
1641
- attempt: attemptCount,
1642
- mode: 'gesture',
1643
- start: currentPoint,
1644
- end: targetPoint,
1645
- success: false
1646
- });
1647
- return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false);
1648
- }
1649
- actionDevice = fallbackActionResult.device ?? actionDevice;
1650
- verificationResult = await runVerification();
1651
- observedState = verificationResult.observedState;
1652
- lastObservedState = observedState;
1653
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1654
- attempt: attemptCount,
1655
- property,
1656
- target_value: normalizedTargetValue,
1657
- actual_state: observedState,
1658
- reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
1659
- });
1660
- if (verificationResult.withinTolerance) {
1661
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1662
- const base = buildActionExecutionResult({
1663
- actionType,
1664
- sourceModule: 'interact',
1665
- device: actionDevice ?? currentDevice,
1666
- selector: targetSelector,
1667
- resolved: resolvedTarget,
1668
- success: true,
1669
- uiFingerprintBefore: fingerprintBefore,
1670
- uiFingerprintAfter,
1671
- details: {
1672
- target_value: normalizedTargetValue,
1673
- tolerance: effectiveTolerance,
1674
- property,
1675
- attempts: attemptCount,
1676
- adjustment_mode: lastAdjustmentMode,
1677
- actual_state: observedState,
1678
- converged: true,
1679
- within_tolerance: true,
1680
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1681
- },
1682
- traceSteps
1683
- });
1684
- return {
1685
- ...base,
1686
- target_state: {
1687
- property,
1688
- target_value: normalizedTargetValue,
1689
- tolerance: effectiveTolerance
1690
- },
1691
- actual_state: observedState,
1692
- within_tolerance: true,
1693
- converged: true,
1694
- attempts: attemptCount,
1695
- adjustment_mode: lastAdjustmentMode
1696
- };
1697
- }
1698
- }
1699
- verification = verificationResult.verification;
1700
- lastObservedState = observedState;
1701
- if (verificationResult.withinTolerance) {
1702
- recordTraceStep('verify', 'success', {
1703
- attempt: attemptCount,
1704
- property,
1705
- target_value: normalizedTargetValue,
1706
- actual_state: observedState,
1707
- reason: verification?.reason ?? 'control converged to target value'
1708
- });
1709
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1710
- const base = buildActionExecutionResult({
1711
- actionType,
1712
- sourceModule: 'interact',
1713
- device: actionDevice ?? currentDevice,
1714
- selector: targetSelector,
1715
- resolved: resolvedTarget,
1716
- success: true,
1717
- uiFingerprintBefore: fingerprintBefore,
1718
- uiFingerprintAfter,
1719
- details: {
1720
- target_value: normalizedTargetValue,
1721
- tolerance: effectiveTolerance,
1722
- property,
1723
- attempts: attemptCount,
1724
- adjustment_mode: lastAdjustmentMode,
1725
- actual_state: observedState,
1726
- converged: true,
1727
- within_tolerance: true,
1728
- reason: verification?.reason ?? 'control converged to target value'
1729
- },
1730
- traceSteps
1731
- });
1732
- return {
1733
- ...base,
1734
- target_state: {
1735
- property,
1736
- target_value: normalizedTargetValue,
1737
- tolerance: effectiveTolerance
1738
- },
1739
- actual_state: observedState,
1740
- within_tolerance: true,
1741
- converged: true,
1742
- attempts: attemptCount,
1743
- adjustment_mode: lastAdjustmentMode
1744
- };
1745
- }
1746
- cachedResolvedMatch = inferredSliderState
1747
- ? null
1748
- : {
1749
- el: {
1750
- ...currentEl,
1751
- state: {
1752
- ...(currentEl.state ?? null),
1753
- ...(observedState ? {
1754
- [observedState.property]: observedState.value,
1755
- raw_value: observedState.raw_value ?? observedState.value
1756
- } : {})
1757
- }
1758
- },
1759
- idx: resolved.match.idx
1760
- };
1761
- }
1762
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
1763
- return buildFailure('CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter);
142
+ static async adjustControlHandler(input) {
143
+ return adjustControl(input, {
144
+ observe: (platform, deviceId) => ToolsObserve.getUITreeHandler({ platform, deviceId }),
145
+ fingerprint: (platform, deviceId) => ToolsInteract._captureFingerprint(platform, deviceId),
146
+ getHandle: id => ToolsInteract._resolvedUiElements.get(id),
147
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
148
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx),
149
+ wait: input => ToolsInteract.waitForUIHandler(input), find: input => ToolsInteract.findElementHandler(input),
150
+ expectState: input => ToolsInteract.expectStateHandler(input), service: (platform, deviceId) => ToolsInteract.getInteractionService(platform, deviceId),
151
+ tap: input => ToolsInteract.tapHandler(input), swipe: input => ToolsInteract.swipeHandler(input)
152
+ });
1764
153
  }
1765
154
  static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId, timeoutMs }) {
1766
155
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
@@ -1778,1431 +167,33 @@ export class ToolsInteract {
1778
167
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
1779
168
  return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id);
1780
169
  }
1781
- static async findElementHandler({ query, exact = false, timeoutMs = 3000, platform, deviceId }) {
1782
- // Try to use observe layer to fetch the current UI tree and perform a fast semantic search
1783
- const start = Date.now();
1784
- const deadline = start + timeoutMs;
1785
- const normalize = ToolsInteract._normalize;
1786
- const q = normalize(query);
1787
- if (!q)
1788
- return { found: false, error: 'Empty query' };
1789
- let best = null;
1790
- let bestTree = null;
1791
- let bestIterationCandidates = [];
1792
- let shouldStop = false;
1793
- const scoreElement = (el, idx) => {
1794
- if (!el || !el.visible)
1795
- return null;
1796
- const bounds = el.bounds || [0, 0, 0, 0];
1797
- if (!Array.isArray(bounds) || bounds.length < 4)
1798
- return null;
1799
- const [l, t, r, b] = bounds;
1800
- if (r <= l || b <= t)
1801
- return null;
1802
- // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
1803
- const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el));
1804
- const text = normalize(el.text ?? el.label ?? el.value ?? '');
1805
- const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
1806
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
1807
- const className = normalize(el.type ?? el.class ?? '');
1808
- const semanticRole = normalize(el.semantic?.semantic_role ?? '');
1809
- const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : [];
1810
- let score = 0;
1811
- let reason = 'best_scoring_candidate';
1812
- if (exact) {
1813
- if (text && text === q) {
1814
- score = 1.0;
1815
- reason = 'exact_text_match';
1816
- }
1817
- else if (content && content === q) {
1818
- score = 0.95;
1819
- reason = 'exact_content_desc_match';
1820
- }
1821
- else if (resourceId && resourceId === q) {
1822
- score = 0.92;
1823
- reason = 'exact_resource_id_match';
1824
- }
1825
- else if (className && className === q) {
1826
- score = 0.3;
1827
- reason = 'exact_class_match';
1828
- }
1829
- }
1830
- else {
1831
- if (text && text === q) {
1832
- score = 1.0;
1833
- reason = 'exact_text_match';
1834
- }
1835
- else if (content && content === q) {
1836
- score = 0.95;
1837
- reason = 'exact_content_desc_match';
1838
- }
1839
- else if (resourceId && resourceId === q) {
1840
- score = 0.92;
1841
- reason = 'exact_resource_id_match';
1842
- }
1843
- else if (text && text.includes(q)) {
1844
- score = 0.6;
1845
- reason = 'partial_text_match';
1846
- }
1847
- else if (content && content.includes(q)) {
1848
- score = 0.55;
1849
- reason = 'partial_content_desc_match';
1850
- }
1851
- else if (resourceId && resourceId.includes(q)) {
1852
- score = 0.7;
1853
- reason = 'partial_resource_id_match';
1854
- }
1855
- else if (className && className.includes(q)) {
1856
- score = 0.3;
1857
- reason = 'partial_class_match';
1858
- }
1859
- }
1860
- if (!exact) {
1861
- if (!score && semanticRole && semanticRole.includes(q)) {
1862
- score = 0.5;
1863
- reason = 'semantic_role_match';
1864
- }
1865
- if (semanticActions.some((action) => action.includes(q))) {
1866
- score = Math.max(score, score > 0 ? 0.65 : 0.6);
1867
- reason = 'semantic_action_match';
1868
- }
1869
- if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
1870
- score = 0.45;
1871
- reason = 'semantic_control_match';
1872
- }
1873
- }
1874
- else {
1875
- if (!score && semanticRole && semanticRole === q) {
1876
- score = 0.5;
1877
- reason = 'semantic_role_match';
1878
- }
1879
- if (semanticActions.some((action) => action === q)) {
1880
- score = Math.max(score, score > 0 ? 0.65 : 0.6);
1881
- reason = 'semantic_action_match';
1882
- }
1883
- }
1884
- if (score > 0 && interactable)
1885
- score += 0.05;
1886
- if (score <= 0)
1887
- return null;
1888
- return { el, idx, score, reason, interactable };
1889
- };
1890
- while (Date.now() <= deadline) {
1891
- try {
1892
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
1893
- if (tree && Array.isArray(tree.elements)) {
1894
- const elements = tree.elements;
1895
- const iterationCandidates = [];
1896
- let iterationImprovedBest = false;
1897
- for (let i = 0; i < elements.length; i++) {
1898
- const el = elements[i];
1899
- try {
1900
- const candidate = scoreElement(el, i);
1901
- if (!candidate)
1902
- continue;
1903
- iterationCandidates.push(candidate);
1904
- if (!best || candidate.score > best.score) {
1905
- best = candidate;
1906
- bestTree = tree;
1907
- iterationImprovedBest = true;
1908
- if (best.score >= 0.95) {
1909
- shouldStop = true;
1910
- break;
1911
- }
1912
- }
1913
- }
1914
- catch (e) {
1915
- console.error('Error scoring element:', e);
1916
- }
1917
- }
1918
- if (iterationImprovedBest) {
1919
- bestIterationCandidates = iterationCandidates.slice();
1920
- }
1921
- }
1922
- }
1923
- catch (e) {
1924
- console.error('Error fetching UI tree:', e);
1925
- }
1926
- if (shouldStop || Date.now() > deadline)
1927
- break;
1928
- await new Promise(r => setTimeout(r, 100));
1929
- }
1930
- if (!best)
1931
- return { found: false, error: 'Element not found' };
1932
- // If the best match is not interactable, try to resolve an actionable ancestor.
1933
- try {
1934
- const elements = (bestTree && Array.isArray(bestTree.elements)) ? bestTree.elements : [];
1935
- const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution : null;
1936
- let chosen = best;
1937
- const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null;
1938
- // Strategy 1: if parentId references an index, climb that chain
1939
- let resolvedAncestor = null;
1940
- if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
1941
- let cur = chosen;
1942
- let safety = 0;
1943
- while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
1944
- let pid = cur.el.parentId;
1945
- let idx = null;
1946
- if (typeof pid === 'number')
1947
- idx = pid;
1948
- else if (typeof pid === 'string' && /^\d+$/.test(pid))
1949
- idx = Number(pid);
1950
- // If parentId is not an index, try to find by matching resourceId or id field
1951
- if (idx !== null && elements[idx]) {
1952
- cur = { el: elements[idx], idx };
1953
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) {
1954
- resolvedAncestor = cur;
1955
- break;
1956
- }
1957
- }
1958
- else if (typeof pid === 'string') {
1959
- // fallback: search elements for matching resourceId or id
1960
- const foundIndex = elements.findIndex((el) => (el.resourceId === pid || el.id === pid));
1961
- const found = foundIndex >= 0 ? elements[foundIndex] : null;
1962
- if (found) {
1963
- cur = { el: found, idx: foundIndex };
1964
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) {
1965
- resolvedAncestor = cur;
1966
- break;
1967
- }
1968
- // otherwise continue climbing if this found element has its own parentId
1969
- }
1970
- else {
1971
- break;
1972
- }
1973
- }
1974
- else {
1975
- break;
1976
- }
1977
- safety++;
1978
- }
1979
- }
1980
- // Strategy 2: fallback - find a clickable element whose bounds fully contain the child's bounds
1981
- if (!resolvedAncestor && childBounds) {
1982
- const [cl, ct, cr, cb] = childBounds;
1983
- // find candidates that are clickable and contain the child bounds
1984
- const candidates = elements
1985
- .map((el, idx) => ({ el, idx }))
1986
- .filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds.length >= 4);
1987
- let bestCandidate = null;
1988
- let bestCandidateArea = Infinity;
1989
- for (const c of candidates) {
1990
- const bounds = c.el.bounds;
1991
- const [pl, pt, pr, pb] = bounds;
1992
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
1993
- const area = (pr - pl) * (pb - pt);
1994
- if (area < bestCandidateArea) {
1995
- bestCandidateArea = area;
1996
- bestCandidate = c;
1997
- }
1998
- }
1999
- }
2000
- if (bestCandidate)
2001
- resolvedAncestor = bestCandidate;
2002
- }
2003
- if (resolvedAncestor) {
2004
- best = {
2005
- el: resolvedAncestor.el,
2006
- idx: resolvedAncestor.idx,
2007
- score: Math.min(1, best.score + 0.02),
2008
- reason: 'clickable_parent_preferred',
2009
- interactable: true
2010
- };
2011
- }
2012
- if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
2013
- const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen);
2014
- if (nearbyActionable) {
2015
- best = {
2016
- el: nearbyActionable.el,
2017
- idx: nearbyActionable.idx,
2018
- score: Math.min(1, best.score + 0.02),
2019
- reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
2020
- interactable: true
2021
- };
2022
- }
2023
- }
2024
- }
2025
- catch (e) {
2026
- console.error('Error resolving ancestor:', e);
2027
- }
2028
- if (!best)
2029
- return { found: false, error: 'Element not found' };
2030
- const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null;
2031
- const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null;
2032
- const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el));
2033
- const alternateCandidates = uniqueRanked
2034
- .filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
2035
- .slice(0, 3)
2036
- .map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate));
2037
- const outEl = {
2038
- text: best.el.text ?? null,
2039
- resourceId: best.el.resourceId ?? null,
2040
- contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
2041
- class: best.el.type ?? best.el.class ?? null,
2042
- bounds: boundsObj,
2043
- clickable: !!best.el.clickable,
2044
- enabled: !!best.el.enabled,
2045
- stable_id: best.el.stable_id ?? null,
2046
- role: best.el.role ?? null,
2047
- test_tag: best.el.test_tag ?? null,
2048
- selector: best.el.selector ?? null,
2049
- semantic: best.el.semantic ?? null,
2050
- tapCoordinates,
2051
- telemetry: {
2052
- matchedIndex: best.idx ?? null,
2053
- matchedInteractable: !!best.interactable,
2054
- sliderLike: best.reason === 'slider_track_preferred'
2055
- }
2056
- };
2057
- if (best.reason === 'slider_track_preferred') {
2058
- const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left);
2059
- const interactionHint = {
2060
- kind: 'slider',
2061
- axis: isVertical ? 'vertical' : 'horizontal',
2062
- trackBounds: boundsObj
2063
- };
2064
- outEl.interactionHint = interactionHint;
2065
- }
2066
- const scoreVal = Math.min(1, Number(best.score.toFixed(3)));
2067
- const resolution = {
2068
- confidence: scoreVal,
2069
- reason: best.reason,
2070
- fallback_available: alternateCandidates.length > 0,
2071
- matched_count: uniqueRanked.length,
2072
- alternates: alternateCandidates
2073
- };
2074
- return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution };
2075
- }
2076
- static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false, resolveTapTarget = false }) {
2077
- const readinessTracker = currentReadinessWait();
2078
- const overallStart = Date.now();
2079
- const requestedIndex = typeof match?.index === 'number' ? match.index : null;
2080
- const requested = {
2081
- selector: selector ?? {},
2082
- condition,
2083
- match: requestedIndex === null ? null : { index: requestedIndex }
2084
- };
2085
- // Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
2086
- const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
2087
- const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
2088
- const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
2089
- if (!hasText && !hasResId && !hasAccId) {
2090
- return {
2091
- status: 'timeout',
2092
- error: {
2093
- code: 'INVALID_SELECTOR',
2094
- message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
2095
- },
2096
- metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
2097
- requested,
2098
- observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
2099
- };
2100
- }
2101
- // Validate condition
2102
- if (!['exists', 'not_exists', 'visible', 'clickable'].includes(condition)) {
2103
- return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } };
2104
- }
2105
- // Platform check
2106
- if (platform && !['android', 'ios'].includes(platform)) {
2107
- return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } };
2108
- }
2109
- const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000));
2110
- const maxAttempts = (retry && retry.max_attempts) ? Math.max(1, retry.max_attempts) : 1;
2111
- const backoff = (retry && retry.backoff_ms) ? Math.max(0, retry.backoff_ms) : 0;
2112
- let attempts = 0;
2113
- let totalPollCount = 0;
2114
- let lastMatchedCount = 0;
2115
- let lastMatchedElement = null;
2116
- let lastConditionSatisfied = false;
2117
- let matchedAt = null;
2118
- let stableMatchCount = 0;
2119
- const stableObservationCount = singleObservation ? 1 : 2;
2120
- const snapshotStaleThresholdMs = 500;
2121
- // Precompute normalized selector values and helpers (constant across polls)
2122
- const normalize = ToolsInteract._normalize;
2123
- const containsFlag = !!selector?.contains;
2124
- const selText = normalize(selector?.text);
2125
- const selRid = normalize(selector?.resource_id);
2126
- const selAid = normalize(selector?.accessibility_id);
2127
- try {
2128
- while (attempts < maxAttempts) {
2129
- attempts++;
2130
- const attemptStart = Date.now();
2131
- const deadline = attemptStart + (timeout_ms || 0);
2132
- while (Date.now() <= deadline || (singleAttempt && totalPollCount === 0)) {
2133
- remainingBudget();
2134
- totalPollCount++;
2135
- try {
2136
- const acquisitionStarted = Date.now();
2137
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
2138
- observeReadinessWait(tree, acquisitionStarted);
2139
- // A failed acquisition cannot establish that the requested tap target is absent.
2140
- if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error)))
2141
- throw new Error('ACTION_TIMEOUT');
2142
- const elements = (tree && Array.isArray(tree.elements)) ? tree.elements : [];
2143
- const matches = [];
2144
- for (let i = 0; i < elements.length; i++) {
2145
- const el = elements[i];
2146
- let ok = true;
2147
- // text
2148
- if (selector.text !== undefined && selector.text !== null) {
2149
- const val = normalize(el.text || el.label || el.value || '');
2150
- if (containsFlag) {
2151
- if (!val.includes(selText))
2152
- ok = false;
2153
- }
2154
- else {
2155
- if (val !== selText)
2156
- ok = false;
2157
- }
2158
- }
2159
- // resource_id
2160
- if (ok && selector.resource_id !== undefined && selector.resource_id !== null) {
2161
- const rid = normalize(el.resourceId || el.resourceID || el.id || '');
2162
- if (containsFlag) {
2163
- if (!rid.includes(selRid))
2164
- ok = false;
2165
- }
2166
- else {
2167
- if (rid !== selRid)
2168
- ok = false;
2169
- }
2170
- }
2171
- // accessibility_id
2172
- if (ok && selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
2173
- const aid = normalize(el.contentDescription || el.contentDesc || el.accessibilityLabel || el.label || '');
2174
- if (containsFlag) {
2175
- if (!aid.includes(selAid))
2176
- ok = false;
2177
- }
2178
- else {
2179
- if (aid !== selAid)
2180
- ok = false;
2181
- }
2182
- }
2183
- if (ok)
2184
- matches.push({ el, idx: i });
2185
- }
2186
- // Evaluate condition
2187
- const matchedCount = matches.length;
2188
- if (rejectAmbiguous && matchedCount > 1 && match?.index === undefined)
2189
- return { status: 'timeout', error: { code: 'AMBIGUOUS_TARGET' } };
2190
- const pickIndex = (typeof match?.index === 'number') ? match.index : undefined;
2191
- let chosen = null;
2192
- if (matches.length > 0) {
2193
- if (pickIndex !== undefined) {
2194
- // If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
2195
- if (pickIndex >= 0 && pickIndex < matches.length)
2196
- chosen = matches[pickIndex];
2197
- else
2198
- chosen = null;
2199
- }
2200
- else {
2201
- chosen = matches[0];
2202
- }
2203
- }
2204
- else {
2205
- chosen = null;
2206
- }
2207
- let conditionMet = false;
2208
- let matchedElement = chosen;
2209
- if (condition === 'exists') {
2210
- // when an index is specified, existence requires that specific index be present
2211
- conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1);
2212
- }
2213
- else if (condition === 'not_exists') {
2214
- // when an index is specified, not_exists is true if that index is absent
2215
- conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0);
2216
- }
2217
- else if (condition === 'visible') {
2218
- if (chosen) {
2219
- const b = chosen.el.bounds;
2220
- const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
2221
- conditionMet = visibleFlag;
2222
- }
2223
- else
2224
- conditionMet = false;
2225
- }
2226
- else if (condition === 'clickable') {
2227
- matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
2228
- ? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen) || chosen) : null;
2229
- if (matchedElement) {
2230
- const b = matchedElement.el.bounds;
2231
- const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
2232
- const enabled = !!matchedElement.el.enabled;
2233
- const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable;
2234
- conditionMet = visibleFlag && enabled && clickable;
2235
- }
2236
- else
2237
- conditionMet = false;
2238
- }
2239
- // Waiting for a label to exist/be visible is distinct from resolving its
2240
- // executable control. Reuse the clickable-wait ancestor policy without
2241
- // promoting hidden or disabled labels into an unrelated enabled target.
2242
- if (resolveTapTarget && conditionMet && matchedElement &&
2243
- ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
2244
- matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement;
2245
- }
2246
- const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
2247
- const resolvedDeviceId = tree?.device?.id || deviceId;
2248
- lastMatchedCount = matchedCount;
2249
- lastConditionSatisfied = conditionMet;
2250
- lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null;
2251
- const now = Date.now();
2252
- const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null;
2253
- const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs;
2254
- if (conditionMet && snapshotFresh) {
2255
- if (matchedAt === null)
2256
- matchedAt = now;
2257
- stableMatchCount++;
2258
- if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2259
- const latency_ms = now - overallStart;
2260
- const outEl = lastMatchedElement;
2261
- return {
2262
- status: 'success',
2263
- ...(singleObservation ? { _tree: tree } : {}),
2264
- matched: matchedCount,
2265
- element: outEl,
2266
- metrics: { latency_ms, poll_count: totalPollCount, attempts },
2267
- requested,
2268
- observed: {
2269
- matched_count: matchedCount,
2270
- condition_satisfied: true,
2271
- selected_index: outEl?.index ?? null,
2272
- last_matched_element: outEl
2273
- }
2274
- };
2275
- }
2276
- }
2277
- else {
2278
- stableMatchCount = 0;
2279
- matchedAt = null;
2280
- }
2281
- }
2282
- catch (e) {
2283
- // Deadline expiry does not invalidate the last successfully observed state.
2284
- if (e instanceof Error && e.message === 'ACTION_TIMEOUT')
2285
- throw e;
2286
- if (readinessTracker && !String(e).includes('READINESS_'))
2287
- readinessTracker.unavailable();
2288
- if (readinessTracker && isDeviceUnavailable(e))
2289
- throw e;
2290
- // Non-fatal per-poll error; record and continue
2291
- if (!readinessTracker)
2292
- console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e));
2293
- }
2294
- if (singleAttempt)
2295
- break;
2296
- // Sleep until next poll
2297
- await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)));
2298
- }
2299
- // Attempt timed out; if more attempts allowed, backoff then retry
2300
- if (attempts < maxAttempts) {
2301
- if (backoff > 0)
2302
- await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)));
2303
- continue;
2304
- }
2305
- // Final failure for this call
2306
- const elapsed = Date.now() - overallStart;
2307
- const observed = {
2308
- matched_count: lastMatchedCount,
2309
- condition_satisfied: lastConditionSatisfied,
2310
- selected_index: lastMatchedElement?.index ?? null,
2311
- last_matched_element: lastMatchedElement
2312
- };
2313
- const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
2314
- ? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
2315
- : ` observed ${lastMatchedCount} match(es)`;
2316
- return {
2317
- status: 'timeout',
2318
- error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
2319
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2320
- requested,
2321
- observed
2322
- };
2323
- }
2324
- }
2325
- catch (err) {
2326
- if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT')
2327
- throw err;
2328
- if (readinessTracker && isDeviceUnavailable(err))
2329
- throw err;
2330
- const elapsed = Date.now() - overallStart;
2331
- return {
2332
- status: 'timeout',
2333
- error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
2334
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2335
- requested,
2336
- observed: {
2337
- matched_count: lastMatchedCount,
2338
- condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
2339
- selected_index: lastMatchedElement?.index ?? null,
2340
- last_matched_element: lastMatchedElement
2341
- }
2342
- };
2343
- }
170
+ static async findElementHandler(input) {
171
+ return findElement(input, target => ToolsObserve.getUITreeHandler(target));
2344
172
  }
2345
- // Helper: normalize various log objects into plain message strings for comparison
2346
- static _logsToMessages(logsArr) {
2347
- if (!Array.isArray(logsArr))
2348
- return [];
2349
- return logsArr.map((l) => {
2350
- if (typeof l === 'string')
2351
- return l;
2352
- if (l && (l.message || l.msg))
2353
- return l.message || l.msg;
2354
- try {
2355
- return JSON.stringify(l);
2356
- }
2357
- catch {
2358
- return String(l);
2359
- }
2360
- });
173
+ static async waitForUIHandler(input) {
174
+ return waitForUI(input, { observe: target => ToolsObserve.getUITreeHandler(target),
175
+ resolve: (platform, deviceId, element, index) => ToolsInteract._buildResolvedElement(platform, deviceId, element, index) });
2361
176
  }
2362
- static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }) {
2363
- const start = Date.now();
2364
- let lastFingerprint = null;
2365
- let lastActivity = null;
2366
- while (Date.now() - start < timeoutMs) {
2367
- try {
2368
- const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
2369
- const fp = res?.fingerprint ?? null;
2370
- lastActivity = res?.activity ?? lastActivity;
2371
- if (fp === null || fp === undefined) {
2372
- lastFingerprint = null;
2373
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
2374
- continue;
2375
- }
2376
- lastFingerprint = fp;
2377
- if (fp !== previousFingerprint) {
2378
- // Stability confirmation
2379
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
2380
- try {
2381
- const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
2382
- const confirmFp = confirmRes?.fingerprint ?? null;
2383
- lastActivity = confirmRes?.activity ?? lastActivity;
2384
- if (confirmFp === fp) {
2385
- return {
2386
- success: true,
2387
- previousFingerprint,
2388
- newFingerprint: fp,
2389
- elapsedMs: Date.now() - start,
2390
- observed_screen: {
2391
- fingerprint: fp,
2392
- activity: lastActivity
2393
- }
2394
- };
2395
- }
2396
- lastFingerprint = confirmFp;
2397
- continue;
2398
- }
2399
- catch (e) {
2400
- console.error('Error confirming fingerprint:', e);
2401
- continue;
2402
- }
2403
- }
2404
- }
2405
- catch (e) {
2406
- console.error('Error getting screen fingerprint:', e);
2407
- }
2408
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
2409
- }
2410
- return {
2411
- success: false,
2412
- reason: 'timeout',
2413
- previousFingerprint,
2414
- lastFingerprint,
2415
- elapsedMs: Date.now() - start,
2416
- observed_screen: {
2417
- fingerprint: lastFingerprint,
2418
- activity: lastActivity
2419
- }
2420
- };
177
+ static async waitForScreenChangeHandler(input) {
178
+ return waitForScreenChange(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target) });
2421
179
  }
2422
- static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 300, expected_change, scope = 'screen', target = null }) {
2423
- const readinessTracker = currentReadinessWait();
2424
- const start = Date.now();
2425
- const pollIntervalMs = 300;
2426
- const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300);
2427
- let baseline = null;
2428
- let baselineScope = null;
2429
- let lastObservedRevision = null;
2430
- let lastLoadingState = null;
2431
- let lastSnapshotFreshnessMs = null;
2432
- let candidateSignatures = null;
2433
- let candidateObservedChange = null;
2434
- let candidateSinceMs = null;
2435
- let lastChangeSummary = null;
2436
- let lastScopeResolution = {
2437
- scope: scope === 'subtree' ? 'subtree' : 'screen',
2438
- target: target && typeof target === 'string' ? target : null,
2439
- resolved: scope !== 'subtree',
2440
- resolvedIndex: null,
2441
- resolvedStableId: null,
2442
- reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
2443
- };
2444
- while (Date.now() - start < timeout_ms) {
2445
- try {
2446
- if (readinessTracker)
2447
- remainingBudget();
2448
- const acquisitionStarted = Date.now();
2449
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
2450
- observeReadinessWait(tree, acquisitionStarted);
2451
- const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target);
2452
- if (scopedTree.error) {
2453
- lastScopeResolution = scopedTree.resolution;
2454
- return {
2455
- success: false,
2456
- observed_change: null,
2457
- snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
2458
- snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
2459
- timeout: true,
2460
- elapsed_ms: Date.now() - start,
2461
- expected_change,
2462
- loading_state: tree?.loading_state ?? lastLoadingState ?? null,
2463
- scope: scopedTree.resolution.scope,
2464
- target: scopedTree.resolution.target,
2465
- stability_state: 'transient',
2466
- change_summary: lastChangeSummary,
2467
- reason: scopedTree.error.message,
2468
- error: scopedTree.error
2469
- };
2470
- }
2471
- const scopedElements = scopedTree.elements;
2472
- const scopedSignatureTree = {
2473
- ...tree,
2474
- elements: scopedElements
2475
- };
2476
- const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree);
2477
- lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
2478
- lastLoadingState = tree?.loading_state ?? lastLoadingState;
2479
- lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs;
2480
- lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary;
2481
- lastScopeResolution = scopedTree.resolution;
2482
- baselineScope = baselineScope ?? scopedTree;
2483
- if (!baseline) {
2484
- baseline = signatures;
2485
- baselineScope = scopedTree;
2486
- }
2487
- else {
2488
- const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
2489
- if (observedChange) {
2490
- if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
2491
- candidateSignatures = signatures;
2492
- candidateObservedChange = observedChange;
2493
- candidateSinceMs = Date.now();
2494
- }
2495
- const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs;
2496
- if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2497
- return {
2498
- success: true,
2499
- observed_change: candidateObservedChange ?? observedChange,
2500
- snapshot_revision: lastObservedRevision ?? undefined,
2501
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2502
- timeout: false,
2503
- elapsed_ms: Date.now() - start,
2504
- expected_change,
2505
- loading_state: lastLoadingState ?? null,
2506
- scope: lastScopeResolution.scope,
2507
- target: lastScopeResolution.target,
2508
- stability_state: 'stable',
2509
- change_summary: lastChangeSummary,
2510
- reason: 'UI change observed'
2511
- };
2512
- }
2513
- }
2514
- else {
2515
- candidateSignatures = null;
2516
- candidateObservedChange = null;
2517
- candidateSinceMs = null;
2518
- }
2519
- }
2520
- }
2521
- catch (error) {
2522
- if (readinessTracker && !String(error).includes('READINESS_'))
2523
- readinessTracker.unavailable();
2524
- if (readinessTracker && isDeviceUnavailable(error))
2525
- throw error;
2526
- if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) {
2527
- readinessTracker.deadline();
2528
- break;
2529
- }
2530
- // Default waits retain best-effort polling.
2531
- }
2532
- await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)));
2533
- }
2534
- return {
2535
- success: false,
2536
- observed_change: null,
2537
- snapshot_revision: lastObservedRevision ?? undefined,
2538
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2539
- timeout: true,
2540
- elapsed_ms: Date.now() - start,
2541
- expected_change,
2542
- loading_state: lastLoadingState ?? null,
2543
- scope: lastScopeResolution.scope,
2544
- target: lastScopeResolution.target,
2545
- stability_state: 'transient',
2546
- change_summary: lastChangeSummary,
2547
- reason: 'timeout'
2548
- };
180
+ static async waitForUIChangeHandler(input) {
181
+ return waitForUIChange(input, { observe: target => ToolsObserve.getUITreeHandler(target), scope: (tree, scope, target) => ToolsInteract._resolveUiChangeScope(tree, scope, target) });
2549
182
  }
2550
- static async expectScreenHandler({ platform, fingerprint, screen, deviceId }) {
2551
- const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
2552
- const observedScreen = {
2553
- fingerprint: observedFingerprint?.fingerprint ?? null,
2554
- screen: observedFingerprint?.activity ?? null
2555
- };
2556
- let observedScreenLabel = observedScreen.screen;
2557
- if (!fingerprint && screen && platform !== 'ios') {
2558
- try {
2559
- const current = await ToolsObserve.getCurrentScreenHandler({ deviceId });
2560
- observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel;
2561
- }
2562
- catch {
2563
- // Keep fingerprint-derived activity when current-screen lookup is unavailable.
2564
- }
2565
- }
2566
- const expectedScreen = {
2567
- fingerprint: fingerprint ?? null,
2568
- screen: screen ?? null
2569
- };
2570
- let success = false;
2571
- let basis = 'none';
2572
- let reason = 'No fingerprint or screen expectation provided';
2573
- if (fingerprint) {
2574
- basis = 'fingerprint';
2575
- success = observedScreen.fingerprint === fingerprint;
2576
- reason = success
2577
- ? `observed fingerprint matches expected fingerprint ${fingerprint}`
2578
- : `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`;
2579
- }
2580
- else if (screen) {
2581
- basis = 'screen';
2582
- const candidates = new Set();
2583
- if (observedScreen.screen)
2584
- candidates.add(observedScreen.screen);
2585
- if (observedScreenLabel)
2586
- candidates.add(observedScreenLabel);
2587
- success = candidates.has(screen);
2588
- reason = success
2589
- ? `observed screen matches expected screen ${screen}`
2590
- : `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`;
2591
- }
2592
- return {
2593
- success,
2594
- observed_screen: {
2595
- fingerprint: observedScreen.fingerprint,
2596
- screen: observedScreenLabel
2597
- },
2598
- expected_screen: expectedScreen,
2599
- confidence: success ? 1 : 0,
2600
- comparison: {
2601
- basis,
2602
- matched: success,
2603
- reason
2604
- },
2605
- trace: buildObservationTrace({
2606
- actionType: 'expect_screen',
2607
- stage: 'verify',
2608
- success,
2609
- attempts: 1,
2610
- metadata: {
2611
- expected_screen: expectedScreen,
2612
- observed_screen: {
2613
- fingerprint: observedScreen.fingerprint,
2614
- screen: observedScreenLabel
2615
- },
2616
- comparison: {
2617
- basis,
2618
- matched: success
2619
- }
2620
- }
2621
- })
2622
- };
183
+ static async expectScreenHandler(input) {
184
+ return expectScreen(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), current: target => ToolsObserve.getCurrentScreenHandler(target) });
2623
185
  }
2624
- static async expectElementVisibleHandler({ selector, element_id, timeout_ms = 5000, poll_interval_ms = 300, platform, deviceId }) {
2625
- const result = await ToolsInteract.waitForUIHandler({
2626
- selector,
2627
- condition: 'visible',
2628
- timeout_ms,
2629
- poll_interval_ms,
2630
- platform,
2631
- deviceId
2632
- });
2633
- if (result?.status === 'success' && result?.element) {
2634
- return {
2635
- success: true,
2636
- selector,
2637
- element_id: result.element.elementId ?? element_id ?? null,
2638
- expected_condition: 'visible',
2639
- element: {
2640
- elementId: result.element.elementId ?? null,
2641
- text: result.element.text ?? null,
2642
- resource_id: result.element.resource_id ?? null,
2643
- accessibility_id: result.element.accessibility_id ?? null,
2644
- class: result.element.class ?? null,
2645
- bounds: result.element.bounds ?? null,
2646
- index: typeof result.element.index === 'number' ? result.element.index : null,
2647
- state: result.element.state ?? null,
2648
- stable_id: result.element.stable_id ?? null,
2649
- role: result.element.role ?? null,
2650
- test_tag: result.element.test_tag ?? null,
2651
- selector: result.element.selector ?? null,
2652
- semantic: result.element.semantic ?? null
2653
- },
2654
- observed: {
2655
- status: result.status,
2656
- matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
2657
- condition_satisfied: true,
2658
- selected_index: typeof result.element.index === 'number' ? result.element.index : null,
2659
- last_matched_element: {
2660
- elementId: result.element.elementId ?? null,
2661
- text: result.element.text ?? null,
2662
- resource_id: result.element.resource_id ?? null,
2663
- accessibility_id: result.element.accessibility_id ?? null,
2664
- class: result.element.class ?? null,
2665
- bounds: result.element.bounds ?? null,
2666
- index: typeof result.element.index === 'number' ? result.element.index : null,
2667
- state: result.element.state ?? null,
2668
- stable_id: result.element.stable_id ?? null,
2669
- role: result.element.role ?? null,
2670
- test_tag: result.element.test_tag ?? null,
2671
- selector: result.element.selector ?? null,
2672
- semantic: result.element.semantic ?? null
2673
- }
2674
- },
2675
- reason: 'selector is visible',
2676
- trace: buildObservationTrace({
2677
- actionType: 'expect_element_visible',
2678
- stage: 'verify',
2679
- success: true,
2680
- attempts: 1,
2681
- metadata: {
2682
- selector,
2683
- element_id: result.element.elementId ?? element_id ?? null,
2684
- status: result.status
2685
- }
2686
- })
2687
- };
2688
- }
2689
- const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT';
2690
- return {
2691
- success: false,
2692
- selector,
2693
- element_id: element_id ?? null,
2694
- expected_condition: 'visible',
2695
- observed: {
2696
- status: result?.status,
2697
- matched_count: result?.observed?.matched_count,
2698
- condition_satisfied: result?.observed?.condition_satisfied ?? false,
2699
- selected_index: result?.observed?.selected_index ?? null,
2700
- last_matched_element: result?.observed?.last_matched_element ?? null
2701
- },
2702
- reason: result?.error?.message ?? 'selector is not visible',
2703
- failure_code: errorCode,
2704
- retryable: errorCode === 'TIMEOUT',
2705
- trace: buildObservationTrace({
2706
- actionType: 'expect_element_visible',
2707
- stage: 'verify',
2708
- success: false,
2709
- attempts: 1,
2710
- metadata: {
2711
- selector,
2712
- element_id: element_id ?? null,
2713
- status: result?.status ?? null,
2714
- reason: result?.error?.message ?? 'selector is not visible'
2715
- }
2716
- })
2717
- };
186
+ static async expectElementVisibleHandler(input) {
187
+ return expectVisible(input, request => ToolsInteract.waitForUIHandler(request));
2718
188
  }
2719
- static async expectStateHandler({ selector, element_id, property, expected, platform, deviceId, stabilization_window_ms = 1000, stable_observation_count = 2, snapshot_stale_threshold_ms = 500, poll_interval_ms = 150, exact_budget = false }) {
2720
- const compareBoolean = (value) => typeof value === 'boolean' ? value : null;
2721
- const compareString = (value) => typeof value === 'string' ? value : null;
2722
- const compareNumber = (value) => typeof value === 'number' && Number.isFinite(value) ? value : null;
2723
- const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
2724
- const start = Date.now();
2725
- const deadline = start + (exact_budget ? stabilization_window_ms : Math.max(500, stabilization_window_ms));
2726
- const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2));
2727
- const pollDelay = exact_budget ? Math.max(50, Math.min(poll_interval_ms || 150, 1000)) : Math.max(100, Math.min(poll_interval_ms || 150, 200));
2728
- const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800));
2729
- let attempts = 0;
2730
- let stableCount = 0;
2731
- let lastReason = 'element not found';
2732
- let lastFailureCode = 'ELEMENT_NOT_FOUND';
2733
- let lastObservedElement = null;
2734
- let lastObservedValue = null;
2735
- let lastRawValue = null;
2736
- let lastResolvedElementId = element_id ?? null;
2737
- const traceSteps = [];
2738
- let traceAttemptIndex = 0;
2739
- let resolveRecorded = false;
2740
- const recordTraceStep = (stage, result, metadata) => {
2741
- traceSteps.push(createTraceStep({
2742
- stage,
2743
- timestamp: Date.now(),
2744
- result,
2745
- attemptIndex: traceAttemptIndex++,
2746
- metadata
2747
- }));
2748
- };
2749
- const buildStateTrace = (outcome) => ({
2750
- action_id: nextActionId('expect_state', Date.now()),
2751
- steps: traceSteps,
2752
- final_outcome: outcome,
2753
- attempts
2754
- });
2755
- while (Date.now() <= deadline) {
2756
- attempts++;
2757
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
2758
- const elements = Array.isArray(tree?.elements) ? tree.elements : [];
2759
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
2760
- const treeDeviceId = tree?.device?.id || deviceId;
2761
- const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null;
2762
- let matched = null;
2763
- if (element_id) {
2764
- const resolved = ToolsInteract._resolvedUiElements.get(element_id);
2765
- if (resolved) {
2766
- const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
2767
- if (current)
2768
- matched = { el: current.el, idx: current.index };
2769
- }
2770
- }
2771
- if (!matched && selector) {
2772
- matched = ToolsInteract._findFirstMatchingElement(elements, selector);
2773
- }
2774
- if (!matched) {
2775
- lastReason = 'element not found';
2776
- lastFailureCode = 'ELEMENT_NOT_FOUND';
2777
- stableCount = 0;
2778
- recordTraceStep('resolve', 'retry', {
2779
- selector: selector ?? null,
2780
- element_id: lastResolvedElementId,
2781
- matched: false,
2782
- reason: lastReason,
2783
- attempt: attempts
2784
- });
2785
- recordTraceStep('stabilize', 'retry', {
2786
- stabilization_attempts: attempts,
2787
- stable_observation_count: stableCount,
2788
- snapshot_freshness_ms: treeAgeMs,
2789
- reason: lastReason
2790
- });
2791
- await sleep(pollDelay);
2792
- continue;
2793
- }
2794
- const resolvedElement = ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx), matched.el, matched.idx);
2795
- lastResolvedElementId = resolvedElement.elementId;
2796
- lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null };
2797
- if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
2798
- lastReason = 'stale snapshot';
2799
- lastFailureCode = 'UNKNOWN';
2800
- stableCount = 0;
2801
- recordTraceStep('resolve', 'retry', {
2802
- selector: selector ?? null,
2803
- element_id: lastResolvedElementId,
2804
- matched: true,
2805
- reason: lastReason,
2806
- attempt: attempts
2807
- });
2808
- recordTraceStep('stabilize', 'retry', {
2809
- stabilization_attempts: attempts,
2810
- stable_observation_count: stableCount,
2811
- snapshot_freshness_ms: treeAgeMs,
2812
- reason: lastReason
2813
- });
2814
- await sleep(pollDelay);
2815
- continue;
2816
- }
2817
- if (!resolveRecorded) {
2818
- recordTraceStep('resolve', 'success', {
2819
- selector: selector ?? null,
2820
- element_id: lastResolvedElementId,
2821
- matched: true,
2822
- reason: 'element resolved'
2823
- });
2824
- resolveRecorded = true;
2825
- }
2826
- const observedState = matched.el.state ?? null;
2827
- const actual = observedState?.[property] ?? null;
2828
- let success = false;
2829
- let reason = '';
2830
- let rawValue = null;
2831
- let observedValue = actual;
2832
- switch (property) {
2833
- case 'checked':
2834
- case 'focused':
2835
- case 'expanded':
2836
- case 'enabled': {
2837
- const expectedBool = compareBoolean(expected);
2838
- const actualBool = compareBoolean(actual);
2839
- if (expectedBool === null) {
2840
- reason = `expected ${property} must be boolean`;
2841
- }
2842
- else if (actualBool === null) {
2843
- reason = `${property} state unavailable`;
2844
- }
2845
- else {
2846
- rawValue = actualBool;
2847
- success = actualBool === expectedBool;
2848
- reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`;
2849
- }
2850
- observedValue = actualBool;
2851
- break;
2852
- }
2853
- case 'value':
2854
- case 'raw_value': {
2855
- const expectedNumber = compareNumber(expected);
2856
- const actualNumber = compareNumber(actual);
2857
- if (expectedNumber !== null && actualNumber !== null) {
2858
- success = actualNumber === expectedNumber;
2859
- rawValue = actualNumber;
2860
- observedValue = actualNumber;
2861
- reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`;
2862
- break;
2863
- }
2864
- const expectedString = typeof expected === 'string' ? expected : null;
2865
- const actualString = compareString(actual);
2866
- if (expectedString !== null && actualString !== null) {
2867
- success = actualString === expectedString;
2868
- rawValue = actualString;
2869
- observedValue = actualString;
2870
- reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`;
2871
- }
2872
- else {
2873
- reason = 'value state unavailable';
2874
- }
2875
- break;
2876
- }
2877
- case 'selected': {
2878
- const expectedBool = typeof expected === 'boolean' ? expected : null;
2879
- const expectedString = typeof expected === 'string'
2880
- ? expected
2881
- : expected && typeof expected === 'object'
2882
- ? String(expected.id ?? expected.label ?? '')
2883
- : null;
2884
- if (!observedState || observedState.selected === undefined || observedState.selected === null) {
2885
- reason = 'selected state unavailable';
2886
- break;
2887
- }
2888
- if (expectedBool !== null) {
2889
- const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null;
2890
- if (actualBool === null) {
2891
- reason = 'selected state is not boolean';
2892
- break;
2893
- }
2894
- rawValue = actualBool;
2895
- observedValue = actualBool;
2896
- success = actualBool === expectedBool;
2897
- reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`;
2898
- break;
2899
- }
2900
- const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
2901
- ? String(observedState.selected.id ?? observedState.selected.label ?? '')
2902
- : String(observedState.selected);
2903
- const actualString = actualSelected.trim();
2904
- if (!expectedString) {
2905
- reason = 'expected selected must be boolean, string, or object with id/label';
2906
- break;
2907
- }
2908
- rawValue = actualString;
2909
- observedValue = actualString;
2910
- success = actualString === expectedString;
2911
- reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`;
2912
- break;
2913
- }
2914
- case 'text_value': {
2915
- const expectedString = typeof expected === 'string' ? expected : null;
2916
- const actualString = compareString(actual);
2917
- if (!expectedString) {
2918
- reason = 'expected text_value must be string';
2919
- }
2920
- else if (!actualString) {
2921
- reason = 'text_value state unavailable';
2922
- }
2923
- else {
2924
- success = actualString === expectedString;
2925
- rawValue = actualString;
2926
- observedValue = actualString;
2927
- reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`;
2928
- }
2929
- break;
2930
- }
2931
- default: {
2932
- if (actual !== null && actual !== undefined) {
2933
- success = actual === expected;
2934
- observedValue = actual;
2935
- rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null;
2936
- reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`;
2937
- }
2938
- else {
2939
- reason = `unsupported or unavailable state property: ${property}`;
2940
- }
2941
- }
2942
- }
2943
- if (success) {
2944
- stableCount++;
2945
- recordTraceStep('stabilize', 'success', {
2946
- stabilization_attempts: attempts,
2947
- stable_observation_count: stableCount,
2948
- snapshot_freshness_ms: treeAgeMs,
2949
- reason
2950
- });
2951
- recordTraceStep('verify', 'success', {
2952
- property,
2953
- expected,
2954
- observed_value: observedValue,
2955
- reason
2956
- });
2957
- if (stableCount >= stableTarget) {
2958
- return {
2959
- success: true,
2960
- selector,
2961
- element_id: lastResolvedElementId,
2962
- expected_state: { property, expected },
2963
- element: lastObservedElement,
2964
- observed_state: {
2965
- property,
2966
- value: observedValue,
2967
- ...(rawValue !== null ? { raw_value: rawValue } : {})
2968
- },
2969
- reason,
2970
- stabilization_attempts: attempts,
2971
- stabilization_window_ms: Date.now() - start,
2972
- stable_observation_count: stableCount,
2973
- snapshot_freshness_ms: treeAgeMs ?? undefined,
2974
- trace: buildStateTrace('success')
2975
- };
2976
- }
2977
- }
2978
- else {
2979
- stableCount = 0;
2980
- lastReason = reason || lastReason;
2981
- lastFailureCode = 'UNKNOWN';
2982
- recordTraceStep('stabilize', 'retry', {
2983
- stabilization_attempts: attempts,
2984
- stable_observation_count: stableCount,
2985
- snapshot_freshness_ms: treeAgeMs,
2986
- reason: lastReason
2987
- });
2988
- recordTraceStep('verify', 'retry', {
2989
- property,
2990
- expected,
2991
- observed_value: observedValue,
2992
- reason: lastReason
2993
- });
2994
- }
2995
- if (!success) {
2996
- lastObservedValue = observedValue;
2997
- lastRawValue = rawValue;
2998
- }
2999
- await sleep(pollDelay);
3000
- }
3001
- return {
3002
- success: false,
3003
- selector,
3004
- element_id: lastResolvedElementId,
3005
- expected_state: { property, expected },
3006
- element: lastObservedElement,
3007
- observed_state: {
3008
- property,
3009
- value: lastObservedValue,
3010
- ...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
3011
- },
3012
- reason: lastReason,
3013
- failure_code: lastFailureCode,
3014
- retryable: true,
3015
- trace: buildStateTrace('failure')
3016
- };
189
+ static async expectStateHandler(input) {
190
+ return expectState(input, { observe: target => ToolsObserve.getUITreeHandler(target), getHandle: id => ToolsInteract._resolvedUiElements.get(id),
191
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
192
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx) });
3017
193
  }
3018
- static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
3019
- const start = Date.now();
3020
- const deadline = start + (timeoutMs || 0);
3021
- const q = (query === null || query === undefined) ? '' : String(query);
3022
- // Clamp polling interval to 250-500ms for consistent behavior
3023
- const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500));
3024
- // Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
3025
- let initialFingerprint = null;
3026
- let baselineLastLine = null;
3027
- try {
3028
- const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
3029
- const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
3030
- const withTimeout = (p, ms) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))]);
3031
- const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)]);
3032
- if (fpRes && typeof fpRes === 'object')
3033
- initialFingerprint = fpRes.fingerprint ?? null;
3034
- if (gl) {
3035
- const logsArr = Array.isArray(gl.logs) ? gl.logs : [];
3036
- // Normalize to last message string for baseline comparison
3037
- const msgs = ToolsInteract._logsToMessages(logsArr);
3038
- baselineLastLine = msgs.length ? msgs[msgs.length - 1] : null;
3039
- }
3040
- }
3041
- catch (err) {
3042
- try {
3043
- console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err));
3044
- }
3045
- catch { }
3046
- }
3047
- // Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
3048
- let lastChangeAt = Date.now();
3049
- let prevFingerprint = initialFingerprint;
3050
- const sleep = (ms) => new Promise(r => setTimeout(r, ms));
3051
- // Optional initial observation delay requested by caller
3052
- if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
3053
- try {
3054
- console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`);
3055
- }
3056
- catch { }
3057
- await sleep(observationDelayMs);
3058
- }
3059
- // Telemetry
3060
- let pollCount = 0;
3061
- let matchedAt = null;
3062
- let lastObservedState = null;
3063
- let stableDuration = 0;
3064
- let matchSource = null;
3065
- while (Date.now() <= deadline) {
3066
- pollCount++;
3067
- const now = Date.now();
3068
- // Evaluate condition per type
3069
- if (type === 'ui') {
3070
- try {
3071
- // Prefer using the public findElementHandler which tests can override. This avoids relying
3072
- // on resolveObserve/getUITree for unit tests which may not have devices available.
3073
- try {
3074
- const findRes = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId });
3075
- const isPresent = !!(findRes && findRes.found);
3076
- const conditionTrue = (match === 'present') ? isPresent : !isPresent;
3077
- if (conditionTrue) {
3078
- if (matchedAt === null)
3079
- matchedAt = Date.now();
3080
- stableDuration = Date.now() - matchedAt;
3081
- lastObservedState = true;
3082
- if (stableDuration >= stability_ms) {
3083
- matchSource = 'ui-find';
3084
- const element = isPresent ? findRes.element : null;
3085
- const now2 = Date.now();
3086
- return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null };
3087
- }
3088
- }
3089
- else {
3090
- matchedAt = null;
3091
- stableDuration = 0;
3092
- lastObservedState = false;
3093
- }
3094
- }
3095
- catch (err) {
3096
- console.error('waitForUI(ui) find error:', err);
3097
- }
3098
- }
3099
- catch (err) {
3100
- console.error('waitForUI(ui) outer error:', err);
3101
- }
3102
- }
3103
- else if (type === 'log') {
3104
- try {
3105
- // Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
3106
- const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 });
3107
- const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : [];
3108
- for (const ent of entries) {
3109
- const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : '';
3110
- if (q && String(msg).includes(q)) {
3111
- const now2 = Date.now();
3112
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true };
3113
- }
3114
- }
3115
- const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
3116
- const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : [];
3117
- // Normalize to messages for comparison
3118
- const msgs = ToolsInteract._logsToMessages(logsArr);
3119
- let startIndex = 0;
3120
- if (baselineLastLine) {
3121
- const idx = msgs.lastIndexOf(baselineLastLine);
3122
- startIndex = idx >= 0 ? idx + 1 : 0;
3123
- }
3124
- for (let i = startIndex; i < msgs.length; i++) {
3125
- const line = msgs[i];
3126
- if (q && String(line).includes(q)) {
3127
- const now2 = Date.now();
3128
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true };
3129
- }
3130
- }
3131
- }
3132
- catch (err) {
3133
- console.error('waitForUI(log) error:', err);
3134
- }
3135
- }
3136
- else if (type === 'screen') {
3137
- try {
3138
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
3139
- const fp = fpRes?.fingerprint ?? null;
3140
- if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
3141
- // when screen changed, require stability_ms where fingerprint remains the same
3142
- if (matchedAt === null)
3143
- matchedAt = now;
3144
- const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }))?.fingerprint ?? null;
3145
- if (confirmFp === fp) {
3146
- stableDuration = Date.now() - matchedAt;
3147
- lastObservedState = true;
3148
- if (stableDuration >= stability_ms) {
3149
- const now2 = Date.now();
3150
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null };
3151
- }
3152
- }
3153
- else {
3154
- matchedAt = null;
3155
- stableDuration = 0;
3156
- lastObservedState = false;
3157
- }
3158
- }
3159
- }
3160
- catch (err) {
3161
- console.error('waitForUI(screen) error:', err);
3162
- }
3163
- }
3164
- else if (type === 'idle') {
3165
- try {
3166
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
3167
- const fp = fpRes?.fingerprint ?? null;
3168
- if (fp !== prevFingerprint) {
3169
- prevFingerprint = fp;
3170
- lastChangeAt = Date.now();
3171
- matchedAt = null;
3172
- stableDuration = 0;
3173
- lastObservedState = false;
3174
- }
3175
- else {
3176
- const idleMs = Date.now() - lastChangeAt;
3177
- lastObservedState = true;
3178
- if (idleMs >= stability_ms) {
3179
- const now2 = Date.now();
3180
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null };
3181
- }
3182
- }
3183
- }
3184
- catch (err) {
3185
- console.error('waitForUI(idle) error:', err);
3186
- }
3187
- }
3188
- // Respect poll interval and avoid tight loop
3189
- await sleep(pollInterval);
3190
- }
3191
- // On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
3192
- let snapshot = null;
3193
- if (includeSnapshotOnFailure) {
3194
- try {
3195
- // Use dynamic import to avoid circular-initialization issues where the ToolsObserve
3196
- // binding captured earlier may not reflect test-time overrides. Importing at call
3197
- // time ensures the latest exported ToolsObserve object is used.
3198
- const Obs = await import('../observe/index.js');
3199
- snapshot = await Obs.ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId });
3200
- }
3201
- catch (err) {
3202
- snapshot = { error: err instanceof Error ? err.message : String(err) };
3203
- }
3204
- }
3205
- const elapsed = Date.now() - start;
3206
- return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null };
194
+ static async waitForUICore(input) {
195
+ return waitForUICore(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), logs: target => ToolsObserve.getLogsHandler(target),
196
+ stream: target => ToolsObserve.readLogStreamHandler(target), find: input => ToolsInteract.findElementHandler(input),
197
+ snapshot: async (input) => { const { ToolsObserve } = await import('../observe/index.js'); return ToolsObserve.captureDebugSnapshotHandler(input); } });
3207
198
  }
3208
199
  }