mobile-debug-mcp 0.33.0 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (448) hide show
  1. package/.agents/skills/mcp-builder/SKILL.md +44 -64
  2. package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
  3. package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
  4. package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
  5. package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
  6. package/.agents/skills/test-authoring/SKILL.md +7 -6
  7. package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
  8. package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
  9. package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
  10. package/.github/copilot-instructions.md +14 -36
  11. package/.github/workflows/ci.yml +23 -2
  12. package/AGENTS.md +12 -3
  13. package/CONTRIBUTING.md +79 -0
  14. package/README.md +158 -124
  15. package/agent-plugin/.claude-plugin/plugin.json +2 -2
  16. package/agent-plugin/.codex-plugin/plugin.json +1 -1
  17. package/agent-plugin/.cursor-plugin/plugin.json +2 -2
  18. package/agent-plugin/.mcp.json +1 -1
  19. package/agent-plugin/README.md +20 -6
  20. package/agent-plugin/mcp.json +1 -1
  21. package/agent-plugin/plugin.json +1 -1
  22. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +113 -4
  23. package/dist/contracts/execution-controls.js +24 -0
  24. package/dist/contracts/selector.js +8 -0
  25. package/dist/contracts/target.js +15 -0
  26. package/dist/contracts/ui.js +1 -0
  27. package/dist/debugger/android.js +79 -0
  28. package/dist/debugger/diagnostics.js +54 -0
  29. package/dist/debugger/guard.js +30 -0
  30. package/dist/debugger/protocol.js +33 -0
  31. package/dist/debugger/sessions.js +304 -0
  32. package/dist/debugger/worker.js +176 -0
  33. package/dist/interact/adjustment/adjustment-context.js +41 -0
  34. package/dist/interact/adjustment/adjustment-gestures.js +55 -0
  35. package/dist/interact/adjustment/adjustment-plan.js +18 -0
  36. package/dist/interact/adjustment/adjustment-resolution.js +127 -0
  37. package/dist/interact/adjustment/adjustment-result.js +19 -0
  38. package/dist/interact/adjustment/adjustment-semantic.js +64 -0
  39. package/dist/interact/adjustment/adjustment-verification.js +35 -0
  40. package/dist/interact/adjustment/adjustment.js +52 -0
  41. package/dist/interact/adjustment/control-geometry.js +87 -0
  42. package/dist/interact/adjustment/control-values.js +136 -0
  43. package/dist/interact/adjustment/slider-labels.js +84 -0
  44. package/dist/interact/assertions/screen-assertion.js +37 -0
  45. package/dist/interact/assertions/state-assertion-context.js +35 -0
  46. package/dist/interact/assertions/state-assertion.js +76 -0
  47. package/dist/interact/assertions/state-comparison.js +141 -0
  48. package/dist/interact/assertions/visibility-assertion.js +34 -0
  49. package/dist/interact/conditional-action.js +12 -7
  50. package/dist/interact/index.js +65 -3074
  51. package/dist/interact/input-service.js +1 -0
  52. package/dist/interact/interaction-trace.js +21 -0
  53. package/dist/interact/model.js +1 -0
  54. package/dist/interact/observation.js +1 -0
  55. package/dist/interact/targeting/candidate-ancestor.js +58 -0
  56. package/dist/interact/targeting/candidate-result.js +23 -0
  57. package/dist/interact/targeting/candidate-scoring.js +50 -0
  58. package/dist/interact/targeting/element-helpers.js +216 -0
  59. package/dist/interact/targeting/find-element.js +65 -0
  60. package/dist/interact/targeting/nearby-control.js +56 -0
  61. package/dist/interact/targeting/resolved-elements.js +21 -0
  62. package/dist/interact/targeting/tap-target.js +17 -0
  63. package/dist/interact/targeting/target-scope.js +104 -0
  64. package/dist/interact/waits/legacy-log-wait.js +38 -0
  65. package/dist/interact/waits/legacy-wait-observations.js +103 -0
  66. package/dist/interact/waits/legacy-wait-state.js +34 -0
  67. package/dist/interact/waits/legacy-wait.js +48 -0
  68. package/dist/interact/waits/screen-wait.js +49 -0
  69. package/dist/interact/waits/ui-change-helpers.js +157 -0
  70. package/dist/interact/waits/ui-change-state.js +18 -0
  71. package/dist/interact/waits/ui-change-wait.js +77 -0
  72. package/dist/interact/waits/ui-wait-condition.js +56 -0
  73. package/dist/interact/waits/ui-wait-state.js +38 -0
  74. package/dist/interact/waits/ui-wait.js +106 -0
  75. package/dist/manage/android.js +55 -51
  76. package/dist/manage/index.js +38 -75
  77. package/dist/manage/ios/build-types.js +1 -0
  78. package/dist/manage/ios/build.js +210 -0
  79. package/dist/manage/ios/project-discovery.js +29 -0
  80. package/dist/manage/ios/scheme.js +18 -0
  81. package/dist/manage/ios.js +42 -256
  82. package/dist/observe/android/context.js +1 -0
  83. package/dist/observe/android/current-screen.js +55 -0
  84. package/dist/observe/android/fingerprint.js +13 -0
  85. package/dist/observe/android/log-stream.js +132 -0
  86. package/dist/observe/android/logs.js +107 -0
  87. package/dist/observe/android/screenshot.js +70 -0
  88. package/dist/observe/android/ui-tree.js +107 -0
  89. package/dist/observe/android.js +16 -520
  90. package/dist/observe/evidence.js +122 -76
  91. package/dist/observe/index.js +16 -263
  92. package/dist/observe/ios/context.js +16 -0
  93. package/dist/observe/ios/fingerprint.js +12 -0
  94. package/dist/observe/ios/log-stream.js +105 -0
  95. package/dist/observe/ios/logs.js +168 -0
  96. package/dist/observe/ios/screenshot.js +22 -0
  97. package/dist/observe/ios/ui-elements.js +241 -0
  98. package/dist/observe/ios/ui-tree.js +137 -0
  99. package/dist/observe/ios.js +17 -715
  100. package/dist/observe/logs/model.js +1 -0
  101. package/dist/{network → observe/network}/index.js +2 -2
  102. package/dist/observe/readiness-service.js +13 -2
  103. package/dist/observe/readiness.js +44 -18
  104. package/dist/observe/screenshot-encoding.js +41 -0
  105. package/dist/observe/snapshot-collection.js +79 -0
  106. package/dist/observe/snapshot-logs.js +30 -0
  107. package/dist/observe/snapshot-metadata.js +43 -31
  108. package/dist/observe/snapshot-semantic.js +115 -0
  109. package/dist/observe/system-trace-android.js +135 -0
  110. package/dist/observe/system-trace.js +294 -0
  111. package/dist/runtime/action-results.js +270 -0
  112. package/dist/runtime/clock.js +4 -0
  113. package/dist/server/common.js +1 -256
  114. package/dist/server/definitions/assert.js +162 -0
  115. package/dist/server/definitions/diagnostic.js +118 -0
  116. package/dist/server/definitions/input.js +274 -0
  117. package/dist/server/definitions/manage.js +194 -0
  118. package/dist/server/definitions/observe.js +149 -0
  119. package/dist/server/definitions/target.js +138 -0
  120. package/dist/server/definitions/wait.js +105 -0
  121. package/dist/server/handlers/action-execution.js +163 -0
  122. package/dist/server/handlers/correlation.js +23 -0
  123. package/dist/server/handlers/debugger-handlers.js +48 -0
  124. package/dist/server/handlers/debugger-response.js +9 -0
  125. package/dist/server/handlers/diagnostic.js +69 -0
  126. package/dist/server/handlers/handler-types.js +1 -0
  127. package/dist/server/handlers/input.js +154 -0
  128. package/dist/server/handlers/interact.js +113 -0
  129. package/dist/server/handlers/journey-assertions.js +35 -0
  130. package/dist/server/handlers/journey.js +147 -0
  131. package/dist/server/handlers/manage.js +119 -0
  132. package/dist/server/handlers/observe.js +95 -0
  133. package/dist/server/handlers/payload.js +4 -0
  134. package/dist/server/handlers/tool-handlers.js +160 -0
  135. package/dist/server/schemas/condition-schema.js +2 -0
  136. package/dist/server/schemas/debugger-schema.js +47 -0
  137. package/dist/server/{journey-schema.js → schemas/journey-schema.js} +8 -22
  138. package/dist/server/schemas/system-trace-schema.js +17 -0
  139. package/dist/server/tool-definitions.js +62 -1134
  140. package/dist/server-core.js +4 -4
  141. package/dist/server.js +34 -0
  142. package/dist/system/android-status.js +101 -0
  143. package/dist/system/android.js +37 -27
  144. package/dist/system/index.js +13 -438
  145. package/dist/system/ios-status.js +132 -0
  146. package/dist/system/ios.js +11 -8
  147. package/dist/system/model.js +1 -0
  148. package/dist/system/probes.js +105 -0
  149. package/dist/system/status-summary.js +43 -0
  150. package/dist/system/unsupported-status.js +52 -0
  151. package/dist/utils/android/commands.js +89 -0
  152. package/dist/utils/android/gradle.js +139 -0
  153. package/dist/utils/android/logs.js +103 -0
  154. package/dist/utils/android/ui-tree.js +233 -0
  155. package/dist/utils/android/utils.js +6 -564
  156. package/dist/utils/cli/idb/idb-helper.js +4 -6
  157. package/dist/utils/cli/idb/install-idb.js +13 -9
  158. package/dist/utils/cli/ios/preflight-ios.js +42 -38
  159. package/dist/utils/exec.js +2 -0
  160. package/dist/utils/ios/utils.js +45 -46
  161. package/dist/utils/java.js +75 -57
  162. package/dist/utils/operation-budget.js +2 -0
  163. package/dist/utils/resolve-device.js +6 -2
  164. package/dist/utils/ui/index.js +26 -26
  165. package/docs/CHANGELOG.md +48 -0
  166. package/docs/README.md +38 -0
  167. package/docs/benchmarks/README.md +18 -0
  168. package/docs/benchmarks/interaction-latency/README.md +14 -0
  169. package/docs/benchmarks/interaction-latency/final.md +4 -1
  170. package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
  171. package/docs/benchmarks/ui-tree-performance/README.md +24 -16
  172. package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
  173. package/docs/code-structure.md +118 -0
  174. package/docs/specs/001-state-verification/spec.md +16 -14
  175. package/docs/specs/002-richer-element-identity/spec.md +12 -10
  176. package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
  177. package/docs/specs/004-action-verification-routing/spec.md +3 -3
  178. package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
  179. package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
  180. package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
  181. package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
  182. package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
  183. package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
  184. package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
  185. package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
  186. package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
  187. package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
  188. package/docs/specs/014-actionability-resolution/spec.md +20 -18
  189. package/docs/specs/015-adjustable-control-support/spec.md +17 -15
  190. package/docs/specs/016-environment-tools/spec.md +20 -20
  191. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
  192. package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
  193. package/docs/specs/020-conditional-actions/intent.md +5 -3
  194. package/docs/specs/020-conditional-actions/plan.md +8 -9
  195. package/docs/specs/020-conditional-actions/spec.md +4 -2
  196. package/docs/specs/021-app-readiness-signals/intent.md +4 -2
  197. package/docs/specs/021-app-readiness-signals/plan.md +7 -6
  198. package/docs/specs/021-app-readiness-signals/spec.md +3 -1
  199. package/docs/specs/022-correlated-evidence/intent.md +4 -4
  200. package/docs/specs/022-correlated-evidence/plan.md +10 -11
  201. package/docs/specs/022-correlated-evidence/spec.md +3 -1
  202. package/docs/specs/023-android-system-traces/device-evidence.json +38 -0
  203. package/docs/specs/023-android-system-traces/emulator-recheck.json +38 -0
  204. package/docs/specs/023-android-system-traces/intent.md +41 -0
  205. package/docs/specs/023-android-system-traces/plan.md +115 -0
  206. package/docs/specs/023-android-system-traces/spec.md +122 -0
  207. package/docs/specs/024-android-heap-dump-capture/intent.md +41 -0
  208. package/docs/specs/025-android-runtime-debugger/intent.md +41 -0
  209. package/docs/specs/025-android-runtime-debugger/plan.md +106 -0
  210. package/docs/specs/025-android-runtime-debugger/spec.md +183 -0
  211. package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
  212. package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
  213. package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
  214. package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
  215. package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
  216. package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
  217. package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
  218. package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
  219. package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
  220. package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
  221. package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
  222. package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
  223. package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
  224. package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
  225. package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
  226. package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
  227. package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
  228. package/docs/specs/030-android-coroutine-validation/report.md +240 -0
  229. package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
  230. package/docs/tools/TOOLS.md +31 -16
  231. package/docs/tools/{evidence.md → correlated-evidence.md} +14 -3
  232. package/docs/tools/debugger.md +180 -0
  233. package/docs/tools/interact.md +42 -20
  234. package/docs/tools/manage.md +25 -44
  235. package/docs/tools/observe.md +64 -25
  236. package/docs/tools/system-traces.md +134 -0
  237. package/docs/tools/system.md +12 -5
  238. package/package.json +11 -8
  239. package/scripts/architecture-graph.mjs +69 -0
  240. package/scripts/check-architecture.mjs +47 -0
  241. package/scripts/code-quality-baseline.json +1 -0
  242. package/scripts/code-quality.mjs +161 -0
  243. package/src/contracts/execution-controls.ts +26 -0
  244. package/src/contracts/selector.ts +17 -0
  245. package/src/contracts/target.ts +19 -0
  246. package/src/contracts/ui.ts +94 -0
  247. package/src/debugger/android.ts +66 -0
  248. package/src/debugger/diagnostics.ts +43 -0
  249. package/src/debugger/guard.ts +31 -0
  250. package/src/debugger/java/DebuggerWorker.java +166 -0
  251. package/src/debugger/protocol.ts +37 -0
  252. package/src/debugger/sessions.ts +327 -0
  253. package/src/debugger/worker.ts +135 -0
  254. package/src/interact/adjustment/adjustment-context.ts +83 -0
  255. package/src/interact/adjustment/adjustment-gestures.ts +48 -0
  256. package/src/interact/adjustment/adjustment-plan.ts +23 -0
  257. package/src/interact/adjustment/adjustment-resolution.ts +104 -0
  258. package/src/interact/adjustment/adjustment-result.ts +34 -0
  259. package/src/interact/adjustment/adjustment-semantic.ts +53 -0
  260. package/src/interact/adjustment/adjustment-verification.ts +60 -0
  261. package/src/interact/adjustment/adjustment.ts +49 -0
  262. package/src/interact/adjustment/control-geometry.ts +120 -0
  263. package/src/interact/adjustment/control-values.ts +141 -0
  264. package/src/interact/adjustment/slider-labels.ts +84 -0
  265. package/src/interact/assertions/screen-assertion.ts +39 -0
  266. package/src/interact/assertions/state-assertion-context.ts +48 -0
  267. package/src/interact/assertions/state-assertion.ts +64 -0
  268. package/src/interact/assertions/state-comparison.ts +168 -0
  269. package/src/interact/assertions/visibility-assertion.ts +41 -0
  270. package/src/interact/conditional-action.ts +18 -12
  271. package/src/interact/index.ts +76 -3491
  272. package/src/interact/input-service.ts +11 -0
  273. package/src/interact/interaction-trace.ts +36 -0
  274. package/src/interact/model.ts +109 -0
  275. package/src/interact/observation.ts +7 -0
  276. package/src/interact/targeting/candidate-ancestor.ts +49 -0
  277. package/src/interact/targeting/candidate-result.ts +24 -0
  278. package/src/interact/targeting/candidate-scoring.ts +42 -0
  279. package/src/interact/targeting/element-helpers.ts +243 -0
  280. package/src/interact/targeting/find-element.ts +47 -0
  281. package/src/interact/targeting/nearby-control.ts +46 -0
  282. package/src/interact/targeting/resolved-elements.ts +20 -0
  283. package/src/interact/targeting/tap-target.ts +22 -0
  284. package/src/interact/targeting/target-scope.ts +121 -0
  285. package/src/interact/waits/legacy-log-wait.ts +30 -0
  286. package/src/interact/waits/legacy-wait-observations.ts +55 -0
  287. package/src/interact/waits/legacy-wait-state.ts +48 -0
  288. package/src/interact/waits/legacy-wait.ts +34 -0
  289. package/src/interact/waits/screen-wait.ts +37 -0
  290. package/src/interact/waits/ui-change-helpers.ts +184 -0
  291. package/src/interact/waits/ui-change-state.ts +26 -0
  292. package/src/interact/waits/ui-change-wait.ts +59 -0
  293. package/src/interact/waits/ui-wait-condition.ts +52 -0
  294. package/src/interact/waits/ui-wait-state.ts +55 -0
  295. package/src/interact/waits/ui-wait.ts +84 -0
  296. package/src/manage/android.ts +56 -50
  297. package/src/manage/index.ts +63 -93
  298. package/src/manage/ios/build-types.ts +14 -0
  299. package/src/manage/ios/build.ts +230 -0
  300. package/src/manage/ios/project-discovery.ts +26 -0
  301. package/src/manage/ios/scheme.ts +16 -0
  302. package/src/manage/ios.ts +65 -298
  303. package/src/observe/android/context.ts +1 -0
  304. package/src/observe/android/current-screen.ts +63 -0
  305. package/src/observe/android/fingerprint.ts +13 -0
  306. package/src/observe/android/log-stream.ts +127 -0
  307. package/src/observe/android/logs.ts +110 -0
  308. package/src/observe/android/screenshot.ts +79 -0
  309. package/src/observe/android/ui-tree.ts +113 -0
  310. package/src/observe/android.ts +17 -531
  311. package/src/observe/evidence.ts +130 -71
  312. package/src/observe/index.ts +24 -286
  313. package/src/observe/ios/context.ts +22 -0
  314. package/src/observe/ios/fingerprint.ts +12 -0
  315. package/src/observe/ios/log-stream.ts +103 -0
  316. package/src/observe/ios/logs.ts +179 -0
  317. package/src/observe/ios/screenshot.ts +26 -0
  318. package/src/observe/ios/ui-elements.ts +266 -0
  319. package/src/observe/ios/ui-tree.ts +143 -0
  320. package/src/observe/ios.ts +18 -749
  321. package/src/observe/logs/model.ts +11 -0
  322. package/src/{network → observe/network}/index.ts +2 -2
  323. package/src/observe/readiness-service.ts +10 -3
  324. package/src/observe/readiness.ts +49 -22
  325. package/src/observe/screenshot-encoding.ts +32 -0
  326. package/src/observe/snapshot-collection.ts +69 -0
  327. package/src/observe/snapshot-logs.ts +26 -0
  328. package/src/observe/snapshot-metadata.ts +51 -31
  329. package/src/observe/snapshot-semantic.ts +153 -0
  330. package/src/observe/system-trace-android.ts +133 -0
  331. package/src/observe/system-trace.ts +227 -0
  332. package/src/runtime/action-results.ts +376 -0
  333. package/src/runtime/clock.ts +9 -0
  334. package/src/server/common.ts +9 -359
  335. package/src/server/definitions/assert.ts +164 -0
  336. package/src/server/definitions/diagnostic.ts +120 -0
  337. package/src/server/definitions/input.ts +276 -0
  338. package/src/server/definitions/manage.ts +196 -0
  339. package/src/server/definitions/observe.ts +151 -0
  340. package/src/server/definitions/target.ts +140 -0
  341. package/src/server/definitions/wait.ts +107 -0
  342. package/src/server/handlers/action-execution.ts +164 -0
  343. package/src/server/handlers/correlation.ts +20 -0
  344. package/src/server/handlers/debugger-handlers.ts +39 -0
  345. package/src/server/handlers/debugger-response.ts +7 -0
  346. package/src/server/handlers/diagnostic.ts +79 -0
  347. package/src/server/handlers/handler-types.ts +7 -0
  348. package/src/server/handlers/input.ts +173 -0
  349. package/src/server/handlers/interact.ts +135 -0
  350. package/src/server/handlers/journey-assertions.ts +39 -0
  351. package/src/server/handlers/journey.ts +132 -0
  352. package/src/server/handlers/manage.ts +141 -0
  353. package/src/server/handlers/observe.ts +116 -0
  354. package/src/server/handlers/payload.ts +36 -0
  355. package/src/server/handlers/tool-handlers.ts +159 -0
  356. package/src/server/schemas/condition-schema.ts +2 -0
  357. package/src/server/schemas/debugger-schema.ts +43 -0
  358. package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +8 -22
  359. package/src/server/schemas/system-trace-schema.ts +17 -0
  360. package/src/server/tool-definitions.ts +69 -1140
  361. package/src/server-core.ts +4 -4
  362. package/src/server.ts +25 -1
  363. package/src/system/android-status.ts +124 -0
  364. package/src/system/android.ts +33 -21
  365. package/src/system/index.ts +21 -523
  366. package/src/system/ios-status.ts +147 -0
  367. package/src/system/ios.ts +14 -10
  368. package/src/system/model.ts +41 -0
  369. package/src/system/probes.ts +114 -0
  370. package/src/system/status-summary.ts +47 -0
  371. package/src/system/unsupported-status.ts +55 -0
  372. package/src/types.ts +16 -106
  373. package/src/utils/android/commands.ts +81 -0
  374. package/src/utils/android/gradle.ts +132 -0
  375. package/src/utils/android/logs.ts +103 -0
  376. package/src/utils/android/ui-tree.ts +239 -0
  377. package/src/utils/android/utils.ts +16 -556
  378. package/src/utils/cli/idb/check-idb.ts +4 -4
  379. package/src/utils/cli/idb/idb-helper.ts +6 -8
  380. package/src/utils/cli/idb/install-idb.ts +15 -9
  381. package/src/utils/cli/ios/preflight-ios.ts +39 -31
  382. package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
  383. package/src/utils/exec.ts +2 -0
  384. package/src/utils/ios/utils.ts +57 -61
  385. package/src/utils/java.ts +71 -51
  386. package/src/utils/operation-budget.ts +2 -0
  387. package/src/utils/resolve-device.ts +11 -6
  388. package/src/utils/ui/index.ts +28 -28
  389. package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
  390. package/test/device/automated/debugger/coroutine-support.ts +140 -0
  391. package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
  392. package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
  393. package/test/device/automated/debugger/runtime.android.test.ts +219 -0
  394. package/test/device/automated/interact/interaction.android.test.ts +56 -0
  395. package/test/device/automated/observe/system-trace.smoke.ts +104 -0
  396. package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
  397. package/test/fixtures/contracts/tool-definitions.json +3536 -0
  398. package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
  399. package/test/fixtures/coroutine-app/README.md +48 -0
  400. package/test/fixtures/coroutine-app/build.ts +35 -0
  401. package/test/fixtures/debugger-app/MainActivity.java +67 -0
  402. package/test/fixtures/debugger-app/MainActivity.kt +32 -0
  403. package/test/fixtures/debugger-app/NoLocals.java +7 -0
  404. package/test/fixtures/debugger-app/README.md +65 -0
  405. package/test/fixtures/debugger-app/build.ts +39 -0
  406. package/test/fixtures/latency-journey-app/README.md +2 -0
  407. package/test/fixtures/readiness-app/README.md +2 -0
  408. package/test/helpers/perfetto.ts +59 -0
  409. package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
  410. package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
  411. package/test/unit/debugger/diagnostics.test.ts +80 -0
  412. package/test/unit/debugger/guards.test.ts +50 -0
  413. package/test/unit/debugger/native.test.ts +33 -0
  414. package/test/unit/debugger/preparation.test.ts +32 -0
  415. package/test/unit/debugger/protocol.test.ts +15 -0
  416. package/test/unit/debugger/sessions.test.ts +83 -0
  417. package/test/unit/interact/adjustment_service.test.ts +42 -0
  418. package/test/unit/interact/adjustment_verification.test.ts +35 -0
  419. package/test/unit/interact/clock_dependencies.test.ts +63 -0
  420. package/test/unit/interact/element_helpers.test.ts +19 -0
  421. package/test/unit/interact/legacy_wait_service.test.ts +17 -0
  422. package/test/unit/interact/state_comparison.test.ts +20 -0
  423. package/test/unit/interact/target_scope.test.ts +35 -0
  424. package/test/unit/network/get_network_activity.test.ts +1 -1
  425. package/test/unit/observe/readiness.test.ts +3 -1
  426. package/test/unit/observe/screenshot-encoding.test.ts +30 -0
  427. package/test/unit/observe/snapshot-logs.test.ts +14 -0
  428. package/test/unit/observe/system-trace-android.test.ts +52 -0
  429. package/test/unit/observe/system-trace.test.ts +166 -0
  430. package/test/unit/quality/architecture.test.ts +60 -0
  431. package/test/unit/quality/code-quality.test.ts +58 -0
  432. package/test/unit/quality/interaction_boundaries.test.ts +24 -0
  433. package/test/unit/server/conditional_actions.test.ts +1 -1
  434. package/test/unit/server/conditional_schema.test.ts +2 -2
  435. package/test/unit/server/debugger-response.test.ts +63 -0
  436. package/test/unit/server/system-trace.test.ts +64 -0
  437. package/test/unit/server/tool_definitions_parity.test.ts +8 -0
  438. package/dist/server/tool-handlers.js +0 -893
  439. package/docs/ROADMAP.md +0 -646
  440. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
  441. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
  442. package/docs/specs/baseline-spec-v0.md +0 -312
  443. package/docs/specs/mcp-tooling-spec-v1.md +0 -446
  444. package/src/server/tool-handlers.ts +0 -923
  445. /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
  446. /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
  447. /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
  448. /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
@@ -1,18 +1,34 @@
1
- import { currentReadinessWait, observeReadinessWait, isDeviceUnavailable } from '../observe/readiness.js'
2
- import { readinessDelay } from '../observe/readiness-service.js'
1
+ import type { UiTree } from './observation.js'
2
+ import { resolveTap } from './targeting/tap-target.js'
3
+ import { waitForUICore } from './waits/legacy-wait.js'
4
+ import type { LegacyWaitInput } from './waits/legacy-wait-state.js'
5
+ import { expectState } from './assertions/state-assertion.js'
6
+ import type { StateInput } from './assertions/state-assertion-context.js'
7
+ import { waitForUIChange } from './waits/ui-change-wait.js'
8
+ import type { ChangeInput } from './waits/ui-change-state.js'
9
+ import { waitForScreenChange, type ScreenWaitInput } from './waits/screen-wait.js'
10
+ import { expectScreen, type ScreenExpectation } from './assertions/screen-assertion.js'
11
+ import { expectVisible, type VisibilityInput } from './assertions/visibility-assertion.js'
12
+ import { findElement, type FindInput } from './targeting/find-element.js'
13
+ import { waitForUI } from './waits/ui-wait.js'
14
+ import type { UIWaitInput } from './waits/ui-wait-state.js'
15
+ import { adjustControl } from './adjustment/adjustment.js'
16
+ import type { AdjustmentInput } from './adjustment/adjustment-context.js'
17
+ import type { InputService } from './input-service.js'
18
+ import { ResolvedElements } from './targeting/resolved-elements.js'
19
+ import { findScopedElement, resolveUiChangeScope, type ScopeTree } from './targeting/target-scope.js'
20
+ import type { ScreenFingerprintResponse, UiElement, ResolvedUiElementContext } from './model.js'
21
+ import { _normalizeBounds, _computeElementId, _resolvedTargetFromElement } from './targeting/element-helpers.js'
22
+ import { _collectSubtreeIndices } from './waits/ui-change-helpers.js'
3
23
  import { isConditionBlocked } from './conditional-action.js'
4
- import { remainingBudget } from '../utils/operation-budget.js'
5
- import { createHash } from 'crypto'
6
24
  import { AndroidInteract } from './android.js';
7
25
  import { iOSInteract } from './ios.js';
8
26
  export { AndroidInteract, iOSInteract };
9
27
 
10
28
  import { resolveTargetDevice } from '../utils/resolve-device.js'
11
29
  import { ToolsObserve } from '../observe/index.js'
12
- import { computeSnapshotSignature } from '../observe/snapshot-metadata.js'
13
- import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js'
30
+ import { buildActionExecutionResult } from '../runtime/action-results.js'
14
31
  import type {
15
- ActionTrace,
16
32
  ActionFailureCode,
17
33
  ActionTargetResolved,
18
34
  AdjustControlResponse,
@@ -21,318 +37,17 @@ import type {
21
37
  ExpectStateResponse,
22
38
  ExpectScreenResponse,
23
39
  WaitForUIChangeResponse,
24
- UIElementSemanticMetadata,
25
- UIElementState,
26
- TraceStep,
27
40
  TapElementResponse
28
41
  } from '../types.js'
29
42
 
30
- interface ScreenFingerprintResponse { fingerprint: string | null }
31
-
32
- interface UiElement {
33
- text?: string | null
34
- label?: string | null
35
- value?: string | null
36
- contentDescription?: string | null
37
- contentDesc?: string | null
38
- accessibilityLabel?: string | null
39
- resourceId?: string | null
40
- resourceID?: string | null
41
- id?: string | null
42
- type?: string | null
43
- class?: string | null
44
- bounds?: number[] | null
45
- clickable?: boolean
46
- enabled?: boolean
47
- focusable?: boolean
48
- visible?: boolean
49
- parentId?: number | string | null
50
- children?: number[]
51
- _index?: number
52
- _interactable?: boolean
53
- _sliderLike?: boolean
54
- state?: UIElementState | null
55
- stable_id?: string | null
56
- role?: string | null
57
- test_tag?: string | null
58
- selector?: { value: string | null, confidence: { score: number, reason: string } | null } | null
59
- semantic?: UIElementSemanticMetadata | null
60
- actions?: string[] | null
61
- accessibilityActions?: string[] | null
62
- runtimeActions?: string[] | null
63
- platformActions?: string[] | null
64
- }
65
-
66
- interface ResolvedUiElementContext {
67
- elementId: string
68
- platform: 'android' | 'ios'
69
- deviceId?: string
70
- bounds: [number, number, number, number] | null
71
- index: number
72
- stable_id?: string | null
73
- }
74
-
75
- interface UiResolution {
76
- width?: number
77
- height?: number
78
- }
79
-
80
- interface UiChangeSignatureSet {
81
- hierarchy: string | null
82
- text: string | null
83
- state: string | null
84
- }
85
-
86
- interface UiChangeScopeResolution {
87
- scope: 'screen' | 'subtree'
88
- target: string | null
89
- resolved: boolean
90
- resolvedIndex: number | null
91
- resolvedStableId: string | null
92
- reason: string
93
- }
94
-
95
- interface UiChangeScopeResult {
96
- elements: UiElement[]
97
- resolution: UiChangeScopeResolution
98
- error?: {
99
- code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
100
- message: string
101
- }
102
- }
103
-
104
- interface RankedResolutionCandidate {
105
- el: UiElement
106
- idx: number
107
- score: number
108
- reason: string
109
- interactable: boolean
110
- }
111
-
112
- interface InferredLabeledSliderState {
113
- current: number
114
- min: number
115
- max: number
116
- value_range: { min: number; max: number }
117
- touch_bounds: [number, number, number, number]
118
- source: 'nearby_labels'
119
- }
120
-
121
- function buildObservationTrace({
122
- actionType,
123
- stage,
124
- success,
125
- attempts,
126
- metadata
127
- }: {
128
- actionType: string
129
- stage: 'verify' | 'stabilize' | 'resolve'
130
- success: boolean
131
- attempts: number
132
- metadata?: Record<string, unknown>
133
- }): ActionTrace {
134
- const now = Date.now()
135
- const actionId = nextActionId(actionType, now)
136
- const steps: TraceStep[] = [
137
- createTraceStep({
138
- stage,
139
- timestamp: now,
140
- result: success ? 'success' : 'failure',
141
- attemptIndex: 0,
142
- metadata
143
- })
144
- ]
145
-
146
- return {
147
- action_id: actionId,
148
- steps,
149
- final_outcome: success ? 'success' : 'failure',
150
- attempts: Math.max(1, Math.floor(attempts || 1))
151
- }
152
- }
153
-
154
- interface FindElementResolutionSummary {
155
- confidence: number
156
- reason: string
157
- fallback_available: boolean
158
- matched_count: number
159
- alternates: Array<{
160
- text: string | null
161
- resource_id: string | null
162
- accessibility_id: string | null
163
- class: string | null
164
- bounds: { left: number; top: number; right: number; bottom: number } | null
165
- clickable: boolean
166
- enabled: boolean
167
- score: number
168
- reason: string
169
- }>
170
- }
171
-
172
-
173
43
  export class ToolsInteract {
174
44
  private static readonly _maxResolvedUiElements = 256
175
- private static readonly _uiChangeKinds: Array<'hierarchy_diff' | 'text_change' | 'state_change'> = ['hierarchy_diff', 'text_change', 'state_change']
176
- private static readonly _sliderSearchLookahead = 8
177
- private static readonly _sliderNegativeGapTolerancePx = 32
178
- private static readonly _sliderPositiveGapLimitPx = 640
179
- private static readonly _sliderTrackMinLengthPx = 220
180
- private static readonly _sliderTrackMaxThicknessPx = 180
181
- private static readonly _sliderTrackLengthRatio = 0.18
182
- private static readonly _sliderTrackThicknessRatio = 0.08
183
- private static readonly _sliderLabelWidthRatio = 1.5
184
- private static _resolvedUiElements = new Map<string, ResolvedUiElementContext>()
185
-
186
- private static _normalize(s: any): string {
187
- if (s === null || s === undefined) return ''
188
- try { return String(s).toLowerCase().trim() } catch { return '' }
189
- }
190
-
191
- private static _normalizeBounds(bounds: any): [number, number, number, number] | null {
192
- if (!Array.isArray(bounds) || bounds.length < 4) return null
193
- const normalized = bounds.slice(0, 4).map((value: any) => Number(value))
194
- if (normalized.some((value: number) => Number.isNaN(value))) return null
195
- return normalized as [number, number, number, number]
196
- }
197
-
198
- private static _hash(value: unknown): string {
199
- return createHash('sha256').update(JSON.stringify(value)).digest('hex')
200
- }
201
-
202
- private static _matchesSelector(el: UiElement, selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }): boolean {
203
- if (!selector) return false
204
- const normalize = ToolsInteract._normalize
205
- const containsFlag = !!selector.contains
206
- const text = normalize(el.text ?? el.label ?? el.value ?? '')
207
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
208
- const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '')
209
-
210
- if (selector.text !== undefined && selector.text !== null) {
211
- const q = normalize(selector.text)
212
- if (containsFlag ? !text.includes(q) : text !== q) return false
213
- }
214
-
215
- if (selector.resource_id !== undefined && selector.resource_id !== null) {
216
- const q = normalize(selector.resource_id)
217
- if (containsFlag ? !resourceId.includes(q) : resourceId !== q) return false
218
- }
219
-
220
- if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
221
- const q = normalize(selector.accessibility_id)
222
- if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q) return false
223
- }
224
-
225
- return true
226
- }
227
-
228
- private static _findFirstMatchingElement(
229
- elements: UiElement[],
230
- selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }
231
- ): { el: UiElement, idx: number } | null {
232
- if (!selector) return null
233
- for (let i = 0; i < elements.length; i++) {
234
- const el = elements[i]
235
- if (!el) continue
236
- if (ToolsInteract._matchesSelector(el, selector)) return { el, idx: i }
237
- }
238
- return null
239
- }
240
-
241
- private static _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
242
- if (parentId === undefined || parentId === null) return null
243
-
244
- if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
245
- return parentId
246
- }
247
-
248
- if (typeof parentId === 'string') {
249
- const normalized = ToolsInteract._normalize(parentId)
250
- if (!normalized) return null
251
-
252
- if (/^\d+$/.test(normalized)) {
253
- const index = Number(normalized)
254
- if (index >= 0 && index < elements.length) return index
255
- }
256
-
257
- const foundIndex = elements.findIndex((el) => {
258
- if (!el) return false
259
- return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
260
- ToolsInteract._normalize(el.stable_id ?? '') === normalized
261
- })
262
-
263
- return foundIndex >= 0 ? foundIndex : null
264
- }
265
-
266
- return null
267
- }
268
-
269
- private static _isVisibleElement(el: UiElement): boolean {
270
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
271
- return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
272
- }
273
-
274
- private static _isTapActionable(
275
- el: UiElement,
276
- storedStableId?: string | null,
277
- platform?: 'android' | 'ios'
278
- ): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
279
- if (!ToolsInteract._isVisibleElement(el)) {
280
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
281
- }
282
-
283
- if (el.enabled === false) {
284
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
285
- }
286
-
287
- const semanticTapActionable = !!el.semantic && (
288
- el.semantic.is_clickable ||
289
- (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap'))
290
- )
291
-
292
- if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
293
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
294
- }
295
-
296
- if (storedStableId) {
297
- if (!el.stable_id || el.stable_id !== storedStableId) {
298
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
299
- }
300
- }
301
-
302
- return { actionable: true }
303
- }
304
-
305
- private static _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
306
- if (!ToolsInteract._isVisibleElement(el)) {
307
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
308
- }
309
-
310
- if (el.enabled === false) {
311
- return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
312
- }
313
45
 
314
- if (storedStableId) {
315
- if (!el.stable_id || el.stable_id !== storedStableId) {
316
- return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
317
- }
318
- }
46
+ private static _resolvedUiElements = new ResolvedElements(ToolsInteract._maxResolvedUiElements)
319
47
 
320
- return { actionable: true }
321
- }
48
+ private static _normalizeBounds = _normalizeBounds
322
49
 
323
- private static _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
324
- const identity = {
325
- platform,
326
- deviceId: deviceId || '',
327
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
328
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
329
- accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
330
- class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
331
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
332
- index
333
- }
334
- return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`
335
- }
50
+ private static _computeElementId = _computeElementId
336
51
 
337
52
  private static _buildResolvedElement(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number) {
338
53
  const bounds = ToolsInteract._normalizeBounds(el.bounds)
@@ -364,247 +79,18 @@ export class ToolsInteract {
364
79
  }
365
80
  }
366
81
 
367
- private static _resolveUiChangeScope(
368
- tree: any,
369
- scope: 'screen' | 'subtree' | undefined,
370
- target: string | null | undefined
371
- ): UiChangeScopeResult {
372
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
373
- const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen'
374
-
375
- if (normalizedScope === 'screen') {
376
- return {
377
- elements,
378
- resolution: {
379
- scope: 'screen',
380
- target: null,
381
- resolved: true,
382
- resolvedIndex: null,
383
- resolvedStableId: null,
384
- reason: 'screen scope'
385
- }
386
- }
387
- }
388
-
389
- const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null
390
- if (!requestedTarget) {
391
- return {
392
- elements: [],
393
- resolution: {
394
- scope: 'subtree',
395
- target: null,
396
- resolved: false,
397
- resolvedIndex: null,
398
- resolvedStableId: null,
399
- reason: 'subtree scope requires a target element id'
400
- },
401
- error: {
402
- code: 'INVALID_SCOPE',
403
- message: 'scope=subtree requires a target element_id'
404
- }
405
- }
406
- }
407
-
408
- const resolved = ToolsInteract._findScopedElement(tree, requestedTarget)
409
- if (!resolved) {
410
- return {
411
- elements: [],
412
- resolution: {
413
- scope: 'subtree',
414
- target: requestedTarget,
415
- resolved: false,
416
- resolvedIndex: null,
417
- resolvedStableId: null,
418
- reason: 'target element could not be resolved'
419
- },
420
- error: {
421
- code: 'ELEMENT_NOT_FOUND',
422
- message: `Target element ${requestedTarget} could not be resolved for subtree scope`
423
- }
424
- }
425
- }
426
-
427
- const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index)
428
- const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element): element is UiElement => !!element)
429
-
430
- return {
431
- elements: scopedElements,
432
- resolution: {
433
- scope: 'subtree',
434
- target: requestedTarget,
435
- resolved: true,
436
- resolvedIndex: resolved.index,
437
- resolvedStableId: resolved.stableId,
438
- reason: resolved.reason
439
- }
440
- }
82
+ private static _resolveUiChangeScope(tree: ScopeTree | null | undefined, scope: 'screen' | 'subtree' | undefined, target: string | null | undefined) {
83
+ return resolveUiChangeScope(tree, scope, target, (snapshot, id) => ToolsInteract._findScopedElement(snapshot, id))
441
84
  }
442
85
 
443
- private static _findScopedElement(tree: any, targetElementId: string): { index: number, stableId: string | null, reason: string } | null {
444
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
445
- const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android'
446
- const deviceId = tree?.device?.id ?? undefined
447
- const normalizedTarget = ToolsInteract._normalize(targetElementId)
448
-
449
- for (let i = 0; i < elements.length; i++) {
450
- const el = elements[i]
451
- if (!el) continue
452
-
453
- const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i)
454
- if (computedElementId === targetElementId) {
455
- return {
456
- index: i,
457
- stableId: el.stable_id ?? null,
458
- reason: 'element_id_match'
459
- }
460
- }
461
- }
462
-
463
- for (let i = 0; i < elements.length; i++) {
464
- const el = elements[i]
465
- if (!el) continue
466
-
467
- if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
468
- return {
469
- index: i,
470
- stableId: el.stable_id,
471
- reason: 'stable_id_match'
472
- }
473
- }
474
- }
475
-
476
- const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId)
477
- if (storedContext?.stable_id) {
478
- const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id)
479
- for (let i = 0; i < elements.length; i++) {
480
- const el = elements[i]
481
- if (!el?.stable_id) continue
482
- if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
483
- return {
484
- index: i,
485
- stableId: el.stable_id,
486
- reason: 'stored_stable_id_match'
487
- }
488
- }
489
- }
490
- }
491
-
492
- return null
86
+ private static _findScopedElement(tree: ScopeTree | null | undefined, id: string) {
87
+ return findScopedElement(tree, id, ToolsInteract._resolvedUiElements)
493
88
  }
494
89
 
495
- private static _collectSubtreeIndices(elements: UiElement[], rootIndex: number): number[] {
496
- if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length) return []
497
-
498
- const visited = new Set<number>()
499
- const stack = [rootIndex]
500
- const result: number[] = []
501
-
502
- while (stack.length > 0) {
503
- const index = stack.pop()
504
- if (index === undefined || visited.has(index) || index < 0 || index >= elements.length) continue
505
- visited.add(index)
506
- result.push(index)
507
-
508
- const element = elements[index]
509
- if (!element) continue
510
-
511
- const directChildren = new Set<number>()
512
- if (Array.isArray(element.children)) {
513
- for (const childIndex of element.children) {
514
- if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
515
- directChildren.add(childIndex)
516
- }
517
- }
518
- }
519
-
520
- for (let i = 0; i < elements.length; i++) {
521
- if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
522
- directChildren.add(i)
523
- }
524
- }
525
-
526
- for (const childIndex of directChildren) {
527
- if (!visited.has(childIndex)) stack.push(childIndex)
528
- }
529
- }
530
-
531
- return result.sort((left, right) => left - right)
532
- }
533
-
534
- private static _changeIdentityForElement(el: UiElement, index: number): string {
535
- const stableId = ToolsInteract._normalize(el.stable_id)
536
- if (stableId) return `stable:${stableId}`
537
-
538
- return `fallback:${ToolsInteract._hash({
539
- text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
540
- contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
541
- resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
542
- type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
543
- bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
544
- index
545
- })}`
546
- }
547
-
548
- private static _summarizeUiChangeDelta(initialElements: UiElement[], currentElements: UiElement[]) {
549
- const buildMap = (elements: UiElement[]) => {
550
- const map = new Map<string, string>()
551
- for (let i = 0; i < elements.length; i++) {
552
- const element = elements[i]
553
- if (!element) continue
554
- const key = ToolsInteract._changeIdentityForElement(element, i)
555
- map.set(key, ToolsInteract._hash({
556
- text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
557
- contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
558
- resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
559
- type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
560
- bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
561
- state: element.state ?? null,
562
- visible: !!element.visible,
563
- enabled: !!element.enabled,
564
- clickable: !!element.clickable
565
- }))
566
- }
567
- return map
568
- }
569
-
570
- const initialMap = buildMap(initialElements)
571
- const currentMap = buildMap(currentElements)
572
- let added = 0
573
- let removed = 0
574
- let mutated = 0
575
-
576
- for (const [key, value] of currentMap.entries()) {
577
- if (!initialMap.has(key)) {
578
- added++
579
- } else if (initialMap.get(key) !== value) {
580
- mutated++
581
- }
582
- }
583
-
584
- for (const key of initialMap.keys()) {
585
- if (!currentMap.has(key)) removed++
586
- }
587
-
588
- return {
589
- total_elements: currentElements.length,
590
- added_elements: added,
591
- removed_elements: removed,
592
- mutated_elements: mutated
593
- }
594
- }
90
+ private static _collectSubtreeIndices = _collectSubtreeIndices
595
91
 
596
92
  private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
597
- if (ToolsInteract._resolvedUiElements.has(elementId)) {
598
- ToolsInteract._resolvedUiElements.delete(elementId)
599
- }
600
-
601
- ToolsInteract._resolvedUiElements.set(elementId, context)
602
-
603
- while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
604
- const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value
605
- if (!oldestElementId) break
606
- ToolsInteract._resolvedUiElements.delete(oldestElementId)
607
- }
93
+ ToolsInteract._resolvedUiElements.remember(elementId, context)
608
94
  }
609
95
 
610
96
  private static async _captureFingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null> {
@@ -617,454 +103,7 @@ export class ToolsInteract {
617
103
  }
618
104
  }
619
105
 
620
- private static _buildUiChangeSignatures(tree: any): UiChangeSignatureSet {
621
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
622
- const textPayload: Array<{ text: string, contentDescription: string, resourceId: string }> = []
623
- const statePayload: Array<{
624
- checked: boolean | null
625
- selected: boolean | string | { id: string; label?: string } | null
626
- focused: boolean | null
627
- expanded: boolean | null
628
- enabled: boolean | null
629
- text_value: string | null
630
- value: number | string | null
631
- raw_value: number | string | null
632
- value_range: UIElementState['value_range']
633
- }> = []
634
-
635
- for (const el of elements) {
636
- textPayload.push({
637
- text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
638
- contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
639
- resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
640
- })
641
-
642
- statePayload.push({
643
- checked: el?.state?.checked ?? null,
644
- selected: el?.state?.selected ?? null,
645
- focused: el?.state?.focused ?? null,
646
- expanded: el?.state?.expanded ?? null,
647
- enabled: el?.state?.enabled ?? null,
648
- text_value: el?.state?.text_value ?? null,
649
- value: el?.state?.value ?? null,
650
- raw_value: el?.state?.raw_value ?? null,
651
- value_range: el?.state?.value_range ?? null
652
- })
653
- }
654
-
655
- return {
656
- hierarchy: computeSnapshotSignature(tree),
657
- text: ToolsInteract._hash({
658
- screen: ToolsInteract._normalize(tree?.screen),
659
- elements: textPayload
660
- }),
661
- state: ToolsInteract._hash({
662
- screen: ToolsInteract._normalize(tree?.screen),
663
- elements: statePayload
664
- })
665
- }
666
- }
667
-
668
- private static _matchesUiChange(expected: 'hierarchy_diff' | 'text_change' | 'state_change' | undefined, initial: UiChangeSignatureSet, current: UiChangeSignatureSet): 'hierarchy_diff' | 'text_change' | 'state_change' | null {
669
- const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds
670
-
671
- for (const changeKind of candidates) {
672
- if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy) return changeKind
673
- if (changeKind === 'text_change' && initial.text !== current.text) return changeKind
674
- if (changeKind === 'state_change' && initial.state !== current.state) return changeKind
675
- }
676
-
677
- return null
678
- }
679
-
680
- private static _uiChangeSignaturesEqual(left: UiChangeSignatureSet, right: UiChangeSignatureSet): boolean {
681
- return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state
682
- }
683
-
684
- private static _resolvedTargetFromElement(
685
- elementId: string,
686
- element: UiElement,
687
- index: number
688
- ): ActionTargetResolved {
689
- return {
690
- elementId,
691
- text: element.text ?? null,
692
- resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
693
- accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
694
- class: element.type ?? element.class ?? null,
695
- bounds: ToolsInteract._normalizeBounds(element.bounds),
696
- index,
697
- state: element.state ?? null,
698
- stable_id: element.stable_id ?? null,
699
- role: element.role ?? null,
700
- test_tag: element.test_tag ?? null,
701
- selector: element.selector ?? null,
702
- semantic: element.semantic ?? null
703
- }
704
- }
705
-
706
- private static _summarizeResolutionCandidate(candidate: RankedResolutionCandidate): FindElementResolutionSummary['alternates'][number] {
707
- const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds)
708
- return {
709
- text: candidate.el.text ?? null,
710
- resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
711
- accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
712
- class: candidate.el.type ?? candidate.el.class ?? null,
713
- bounds: bounds
714
- ? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
715
- : null,
716
- clickable: !!candidate.el.clickable,
717
- enabled: !!candidate.el.enabled,
718
- score: candidate.score,
719
- reason: candidate.reason
720
- }
721
- }
722
-
723
- private static _isAdjustableControl(el: UiElement | null): boolean {
724
- if (!el) return false
725
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
726
- const role = ToolsInteract._normalize(el.role ?? '')
727
- return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
728
- }
729
-
730
- private static _isSemanticActionable(el: UiElement | null): boolean {
731
- if (!el?.semantic) return false
732
- if (el.semantic.adjustable) return true
733
- return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0
734
- }
735
-
736
- private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
737
- if (!el?.state) return null
738
- const stateValue = el.state[property as keyof UIElementState]
739
- if (typeof stateValue === 'number' && Number.isFinite(stateValue)) return stateValue
740
- const parsedStateValue = ToolsInteract._parseNumericControlValue(stateValue)
741
- if (parsedStateValue !== null) return parsedStateValue
742
- if (property === 'value' || property === 'raw_value') {
743
- const fallback = el.state.raw_value ?? el.state.value
744
- if (typeof fallback === 'number' && Number.isFinite(fallback)) return fallback
745
- return ToolsInteract._parseNumericControlValue(fallback)
746
- }
747
- return null
748
- }
749
-
750
- private static _parseNumericControlValue(value: unknown): number | null {
751
- if (typeof value === 'number' && Number.isFinite(value)) return value
752
- if (typeof value !== 'string') return null
753
- const trimmed = value.trim()
754
- if (!trimmed) return null
755
-
756
- const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/)
757
- if (percent) return Number(percent[1])
758
-
759
- const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i)
760
- if (fraction) return Number(fraction[1])
761
-
762
- const plain = Number(trimmed)
763
- if (Number.isFinite(plain)) return plain
764
-
765
- const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/)
766
- if (trailing) return Number(trailing[1])
767
-
768
- return null
769
- }
770
-
771
- private static _readNumericControlRange(el: UiElement | null): { min: number; max: number } | null {
772
- const min = typeof el?.state?.value_range?.min === 'number' ? el.state.value_range.min : null
773
- const max = typeof el?.state?.value_range?.max === 'number' ? el.state.value_range.max : null
774
- if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
775
- return { min, max }
776
- }
777
-
778
- const value = el?.state?.value ?? el?.state?.raw_value
779
- if (typeof value === 'string') {
780
- const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i)
781
- if (fraction) {
782
- const parsedMax = Number(fraction[1])
783
- if (Number.isFinite(parsedMax) && parsedMax > 0) return { min: 0, max: parsedMax }
784
- }
785
- }
786
-
787
- return null
788
- }
789
-
790
- private static _readControlStep(el: UiElement | null): number | null {
791
- const candidates = [
792
- el?.state?.step,
793
- el?.state?.value_range?.step
794
- ]
795
- for (const candidate of candidates) {
796
- if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0) return candidate
797
- }
798
- return null
799
- }
800
-
801
- private static _quantizeTargetValue(targetValue: number, min: number, max: number, step: number | null): number {
802
- if (!step) return targetValue
803
- const quantized = min + (Math.round((targetValue - min) / step) * step)
804
- const bounded = Math.max(min, Math.min(max, quantized))
805
- const precision = ToolsInteract._decimalPrecisionForStep(step)
806
- return Number(bounded.toFixed(precision))
807
- }
808
-
809
- private static _decimalPrecisionForStep(step: number): number {
810
- const normalized = String(step).toLowerCase()
811
- const [coefficient, exponentPart] = normalized.split('e')
812
- const coefficientDecimals = coefficient.includes('.')
813
- ? coefficient.split('.')[1]?.length ?? 0
814
- : 0
815
- const exponent = exponentPart !== undefined ? Number(exponentPart) : 0
816
- const precision = exponent < 0
817
- ? coefficientDecimals + Math.abs(exponent)
818
- : Math.max(0, coefficientDecimals - exponent)
819
- return Math.max(0, Math.min(12, precision))
820
- }
821
-
822
- private static _defaultAdjustTolerance(range: { min: number; max: number } | null, step: number | null): number {
823
- if (step) return 0
824
- if (!range) return 0
825
- return Math.max(0, 0.01 * Math.abs(range.max - range.min))
826
- }
827
-
828
- private static _runtimeActionSet(el: UiElement | null): Set<string> {
829
- const values = [
830
- ...(Array.isArray(el?.actions) ? el!.actions! : []),
831
- ...(Array.isArray(el?.accessibilityActions) ? el!.accessibilityActions! : []),
832
- ...(Array.isArray(el?.runtimeActions) ? el!.runtimeActions! : []),
833
- ...(Array.isArray(el?.platformActions) ? el!.platformActions! : [])
834
- ]
835
- return new Set(values.map((value) => ToolsInteract._normalize(value)).filter(Boolean))
836
- }
837
-
838
- private static _hasDirectAdjustmentAction(el: UiElement | null): boolean {
839
- const actions = ToolsInteract._runtimeActionSet(el)
840
- return [
841
- 'action_set_progress',
842
- 'set_progress',
843
- 'setprogress',
844
- 'semanticsactions.setprogress',
845
- 'set_value',
846
- 'setvalue',
847
- 'direct_set_value'
848
- ].some((action) => actions.has(action))
849
- }
850
-
851
- private static _adjustmentDirections(el: UiElement | null): { increment: boolean; decrement: boolean } {
852
- const actions = ToolsInteract._runtimeActionSet(el)
853
- return {
854
- increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
855
- decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
856
- }
857
- }
858
-
859
- private static _textFromElement(el: UiElement | null): string {
860
- return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim()
861
- }
862
-
863
- private static _parseLeadingNumber(text: string): number | null {
864
- const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/)
865
- if (!match) return null
866
- const value = Number(match[1])
867
- return Number.isFinite(value) ? value : null
868
- }
869
-
870
- private static _inferLabeledSliderState(elements: UiElement[], track: UiElement | null): InferredLabeledSliderState | null {
871
- const trackBounds = ToolsInteract._normalizeBounds(track?.bounds)
872
- if (!track || !trackBounds) return null
873
- const type = ToolsInteract._normalize(track.type ?? track.class ?? '')
874
- const role = ToolsInteract._normalize(track.role ?? '')
875
- const width = trackBounds[2] - trackBounds[0]
876
- const height = trackBounds[3] - trackBounds[1]
877
- const wideTrackLike = width >= 240 && height <= 220
878
- const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role)
879
- if (!wideTrackLike && !knownTrackLike) return null
880
-
881
- const trackTop = trackBounds[1]
882
- const trackBottom = trackBounds[3]
883
- const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2
884
- const labels = elements
885
- .map((el, index) => {
886
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
887
- const text = ToolsInteract._textFromElement(el)
888
- const value = ToolsInteract._parseLeadingNumber(text)
889
- return bounds && text && value !== null
890
- ? {
891
- index,
892
- el,
893
- text,
894
- value,
895
- bounds,
896
- centerX: (bounds[0] + bounds[2]) / 2,
897
- centerY: (bounds[1] + bounds[3]) / 2,
898
- hasUnit: /[a-zA-Z%]/.test(text)
899
- }
900
- : null
901
- })
902
- .filter((label): label is NonNullable<typeof label> => !!label)
903
-
904
- const rangeLabels = labels
905
- .filter((label) => {
906
- const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260
907
- const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120
908
- return verticallyNearTrack && horizontallyNearTrack && label.hasUnit
909
- })
910
- .sort((a, b) => a.centerX - b.centerX)
911
-
912
- if (rangeLabels.length < 2) return null
913
- const minLabel = rangeLabels[0]
914
- const maxLabel = rangeLabels[rangeLabels.length - 1]
915
- if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value) return null
916
-
917
- const currentCandidates = labels
918
- .filter((label) => {
919
- if (label.index === minLabel.index || label.index === maxLabel.index) return false
920
- if (label.value < minLabel.value || label.value > maxLabel.value) return false
921
- if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20) return false
922
- return label.centerX >= trackCenterX - 80
923
- })
924
- .sort((a, b) => {
925
- const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY)
926
- if (yDistance !== 0) return yDistance
927
- return b.centerX - a.centerX
928
- })
929
-
930
- const currentLabel = currentCandidates[0]
931
- if (!currentLabel) return null
932
-
933
- const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop
934
- const labelDerivedBounds: [number, number, number, number] = [
935
- Math.min(minLabel.bounds[0], trackBounds[0]),
936
- Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
937
- Math.max(maxLabel.bounds[2], trackBounds[2]),
938
- Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
939
- ]
940
-
941
- return {
942
- current: currentLabel.value,
943
- min: minLabel.value,
944
- max: maxLabel.value,
945
- value_range: { min: minLabel.value, max: maxLabel.value },
946
- touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
947
- source: 'nearby_labels'
948
- }
949
- }
950
-
951
- private static _buildControlPoint(bounds: [number, number, number, number], ratio: number, axis: 'horizontal' | 'vertical') {
952
- const clampedRatio = Math.max(0, Math.min(1, ratio))
953
- const [left, top, right, bottom] = bounds
954
- const width = Math.max(1, right - left)
955
- const height = Math.max(1, bottom - top)
956
- const insetX = Math.max(8, Math.floor(width * 0.08))
957
- const insetY = Math.max(8, Math.floor(height * 0.08))
958
- if (axis === 'vertical') {
959
- const usableHeight = Math.max(1, height - (insetY * 2))
960
- return {
961
- x: Math.floor((left + right) / 2),
962
- y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
963
- }
964
- }
965
- const usableWidth = Math.max(1, width - (insetX * 2))
966
- return {
967
- x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
968
- y: Math.floor((top + bottom) / 2)
969
- }
970
- }
971
-
972
- private static _buildConservativeControlPoint(
973
- bounds: [number, number, number, number],
974
- targetValue: number,
975
- currentValue: number | null,
976
- min: number,
977
- max: number,
978
- axis: 'horizontal' | 'vertical'
979
- ) {
980
- const range = Math.max(1, max - min)
981
- const targetRatio = (targetValue - min) / range
982
- const stepRatio = 1 / range
983
- const centerBias = stepRatio / 2
984
- const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue)
985
- const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0])
986
- const edgeWindow = Math.max(3, Math.floor(range * 0.1))
987
- const isNearLowEdge = targetValue - min <= edgeWindow
988
- const isNearHighEdge = max - targetValue <= edgeWindow
989
- const directionBias = direction > 0
990
- ? -stepRatio * 0.15
991
- : direction < 0
992
- ? stepRatio * 0.65
993
- : 0
994
- const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx))
995
- const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin)
996
- const edgeBias = isNearLowEdge
997
- ? endpointMargin
998
- : isNearHighEdge
999
- ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
1000
- : 0
1001
- const safeRatio = Math.min(
1002
- 1 - (endpointMargin * 0.25),
1003
- Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias)
1004
- )
1005
- return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
1006
- }
1007
-
1008
- private static _buildAdjustmentProbePoints(
1009
- bounds: [number, number, number, number],
1010
- targetValue: number,
1011
- currentValue: number | null,
1012
- min: number,
1013
- max: number,
1014
- axis: 'horizontal' | 'vertical'
1015
- ) {
1016
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
1017
- const currentPoint = currentValue !== null
1018
- ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
1019
- : ToolsInteract._buildControlPoint(bounds, 0.5, axis)
1020
-
1021
- const [left, top, right, bottom] = bounds
1022
- const width = Math.max(1, right - left)
1023
- const height = Math.max(1, bottom - top)
1024
- const crossAxisBumps = axis === 'horizontal'
1025
- ? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
1026
- : [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))]
1027
-
1028
- const clampPoint = (point: { x: number, y: number }) => ({
1029
- x: axis === 'horizontal'
1030
- ? Math.max(left, Math.min(right, point.x))
1031
- : Math.max(left, Math.min(right + Math.max(width, height), point.x)),
1032
- y: axis === 'vertical'
1033
- ? Math.max(top, Math.min(bottom, point.y))
1034
- : Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
1035
- })
1036
-
1037
- const probes = [targetPoint, currentPoint]
1038
- for (const bump of crossAxisBumps) {
1039
- if (axis === 'horizontal') {
1040
- probes.push(
1041
- { x: targetPoint.x, y: bottom + bump },
1042
- { x: currentPoint.x, y: bottom + bump }
1043
- )
1044
- } else {
1045
- probes.push(
1046
- { x: right + bump, y: targetPoint.y },
1047
- { x: right + bump, y: currentPoint.y }
1048
- )
1049
- }
1050
- }
1051
-
1052
- return Array.from(
1053
- new Map(
1054
- probes
1055
- .map(clampPoint)
1056
- .map((point) => [`${point.x}:${point.y}`, point] as const)
1057
- ).values()
1058
- )
1059
- }
1060
-
1061
- private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
1062
- const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
1063
- const role = ToolsInteract._normalize(el.role ?? '')
1064
- if (/vertical/.test(type) || /vertical/.test(role)) return 'vertical'
1065
- if (/horizontal/.test(type) || /horizontal/.test(role)) return 'horizontal'
1066
- return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal'
1067
- }
106
+ private static _resolvedTargetFromElement = _resolvedTargetFromElement
1068
107
 
1069
108
  private static _actionFailure(
1070
109
  actionType: string,
@@ -1106,129 +145,11 @@ export class ToolsInteract {
1106
145
  return null
1107
146
  }
1108
147
 
1109
- private static _resolveActionableAncestor(elements: UiElement[], chosen: { el: UiElement, idx: number } | null): { el: UiElement, idx: number } | null {
1110
- if (!chosen) return null
1111
- if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el)) return chosen
1112
-
1113
- let current = chosen
1114
- let safety = 0
1115
-
1116
- while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
1117
- const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId)
1118
-
1119
- if (parentIndex !== null && elements[parentIndex]) {
1120
- current = { el: elements[parentIndex], idx: parentIndex }
1121
- if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
1122
- } else {
1123
- break
1124
- }
1125
-
1126
- safety++
1127
- }
1128
-
1129
- const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
1130
- if (!childBounds) return null
1131
- const [cl, ct, cr, cb] = childBounds
1132
-
1133
- let best: { el: UiElement, idx: number } | null = null
1134
- let bestArea = Infinity
1135
-
1136
- for (let i = 0; i < elements.length; i++) {
1137
- const el = elements[i]
1138
- if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el))) continue
1139
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
1140
- if (!bounds) continue
1141
- const [pl, pt, pr, pb] = bounds
1142
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
1143
- const area = (pr - pl) * (pb - pt)
1144
- if (area < bestArea) {
1145
- bestArea = area
1146
- best = { el, idx: i }
1147
- }
1148
- }
1149
- }
1150
-
1151
- return best
1152
- }
1153
-
1154
- private static _resolveNearbyActionableControl(
1155
- elements: UiElement[],
1156
- chosen: { el: UiElement, idx: number } | null,
1157
- screen?: UiResolution | null
1158
- ): { el: UiElement, idx: number, sliderLike?: boolean } | null {
1159
- if (!chosen) return null
1160
-
1161
- const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
1162
- if (!labelBounds) return null
1163
-
1164
- const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds
1165
- const labelWidth = labelRight - labelLeft
1166
- const labelHeight = labelBottom - labelTop
1167
- const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0
1168
- const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0
1169
- const minTrackLengthPx = Math.max(
1170
- ToolsInteract._sliderTrackMinLengthPx,
1171
- screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0,
1172
- screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0
1173
- )
1174
- const maxTrackThicknessPx = Math.max(
1175
- ToolsInteract._sliderTrackMaxThicknessPx,
1176
- screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0,
1177
- screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0
1178
- )
1179
-
1180
- let best: { el: UiElement, idx: number, sliderLike?: boolean } | null = null
1181
- let bestScore = Infinity
1182
-
1183
- for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
1184
- const candidate = elements[i]
1185
- if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false) continue
1186
-
1187
- const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds)
1188
- if (!candidateBounds) continue
1189
-
1190
- const [left, top, right] = candidateBounds
1191
- const width = right - left
1192
- const height = candidateBounds[3] - top
1193
- const verticalGap = top - labelBottom
1194
- if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx) continue
1195
-
1196
- const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left)
1197
- if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx) continue
1198
-
1199
- const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '')
1200
- const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '')
1201
- const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '')
1202
-
1203
- let score = verticalGap
1204
- const horizontalTrackLike =
1205
- width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
1206
- height <= maxTrackThicknessPx
1207
- const verticalTrackLike =
1208
- height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
1209
- width <= maxTrackThicknessPx
1210
- const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike
1211
- if (!candidateText && !candidateContent) score -= 18
1212
- if (trackLike) score -= 30
1213
- if (/view|layout|group|frame/i.test(candidateClass)) score -= 10
1214
- if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio) score -= 8
1215
- if (candidateText || candidateContent) score += 20
1216
-
1217
- if (score < bestScore) {
1218
- bestScore = score
1219
- best = { el: candidate, idx: i, sliderLike: trackLike }
1220
- }
1221
- }
1222
-
1223
- return best
1224
- }
1225
-
1226
-
1227
148
  private static async getInteractionService(platform?: 'android' | 'ios', deviceId?: string) {
1228
149
  const effectivePlatform = platform || 'android'
1229
150
  const resolved = await resolveTargetDevice({ platform: effectivePlatform as 'android' | 'ios', deviceId })
1230
- const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract()
1231
- return { interact: interact as any, resolved, platform: effectivePlatform }
151
+ const interact: InputService = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract()
152
+ return { interact, resolved, platform: effectivePlatform }
1232
153
  }
1233
154
 
1234
155
  static async tapHandler({ platform, x, y, deviceId, timeoutMs }: { platform?: 'android' | 'ios', x: number, y: number, deviceId?: string, timeoutMs?: number }) {
@@ -1236,7 +157,7 @@ export class ToolsInteract {
1236
157
  return await interact.tap(x, y, resolved.id, timeoutMs)
1237
158
  }
1238
159
 
1239
- static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }: { elementId: string, verificationMode?: 'none' | 'light' | 'full', timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string, freshTree?: any }): Promise<TapElementResponse> {
160
+ static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }: { elementId: string, verificationMode?: 'none' | 'light' | 'full', timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string, freshTree?: UiTree }): Promise<TapElementResponse> {
1240
161
  const actionType = 'tap_element'
1241
162
  const selector = { elementId }
1242
163
  const resolved = ToolsInteract._resolvedUiElements.get(elementId)
@@ -1246,37 +167,14 @@ export class ToolsInteract {
1246
167
 
1247
168
  const fingerprintBefore = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null
1248
169
 
1249
- const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId }) as any
1250
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform
1251
- const treeDeviceId = tree?.device?.id || resolved.deviceId
1252
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1253
- const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
1254
-
1255
- if (!currentMatch) {
1256
- return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore)
1257
- }
1258
-
1259
- const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
1260
-
1261
- const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform)
1262
- if (!tapActionability.actionable) {
1263
- return ToolsInteract._actionFailure(
1264
- actionType,
1265
- selector,
1266
- resolvedTarget,
1267
- tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
1268
- true,
1269
- fingerprintBefore
1270
- )
1271
- }
1272
-
1273
- const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
1274
- if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
1275
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
1276
- }
1277
-
1278
- const x = Math.floor((bounds[0] + bounds[2]) / 2)
1279
- const y = Math.floor((bounds[1] + bounds[3]) / 2)
170
+ const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId })
171
+ const tap = resolveTap(tree, resolved, {
172
+ current: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
173
+ target: (element, index) => ToolsInteract._resolvedTargetFromElement(resolved.elementId, element, index)
174
+ })
175
+ if ('failure' in tap) return ToolsInteract._actionFailure(actionType, selector, tap.target, tap.failure, true, fingerprintBefore)
176
+ const resolvedTarget = tap.target
177
+ const { x, y } = tap.point
1280
178
  const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId, timeoutMs })
1281
179
 
1282
180
  if (!tapResult.success) {
@@ -1297,916 +195,17 @@ export class ToolsInteract {
1297
195
  })
1298
196
  }
1299
197
 
1300
- static async adjustControlHandler({
1301
- selector,
1302
- element_id,
1303
- property = 'value',
1304
- targetValue,
1305
- tolerance,
1306
- maxAttempts,
1307
- platform,
1308
- deviceId
1309
- }: {
1310
- selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
1311
- element_id?: string,
1312
- property?: string,
1313
- targetValue: number,
1314
- tolerance?: number,
1315
- maxAttempts?: number,
1316
- platform?: 'android' | 'ios',
1317
- deviceId?: string
1318
- }): Promise<AdjustControlResponse> {
1319
- const actionType = 'adjust_control'
1320
- const targetSelector = selector ?? (element_id ? { elementId: element_id } : null)
1321
- const requestedTolerance = typeof tolerance === 'number' && Number.isFinite(tolerance) ? Math.max(0, tolerance) : null
1322
- const attemptsLimit = maxAttempts === undefined || maxAttempts === null
1323
- ? 5
1324
- : Math.max(1, Math.floor(Number(maxAttempts) || 1))
1325
- const sourcePlatform: 'android' | 'ios' = platform || 'android'
1326
- let resolvedPlatform = sourcePlatform
1327
- let resolvedDeviceId = deviceId
1328
- const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null
1329
- const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1330
- let semanticFallbackElement: FindElementResponse['element'] | null = null
1331
- const traceSteps: TraceStep[] = []
1332
- let traceAttemptIndex = 0
1333
-
1334
- const recordTraceStep = (
1335
- stage: TraceStep['stage'],
1336
- result: TraceStep['result'],
1337
- metadata?: Record<string, unknown>
1338
- ) => {
1339
- traceSteps.push(createTraceStep({
1340
- stage,
1341
- timestamp: Date.now(),
1342
- result,
1343
- attemptIndex: traceAttemptIndex++,
1344
- metadata
1345
- }))
1346
- }
1347
-
1348
- const buildFailure = (
1349
- failureCode: ActionFailureCode,
1350
- reason: string,
1351
- resolved: ActionTargetResolved | null,
1352
- device: any,
1353
- actualState: { property: string; value: number | null; raw_value?: number | null } | null,
1354
- attempts: number,
1355
- adjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture',
1356
- retryable = false,
1357
- uiFingerprintAfter: string | null = null
1358
- ): AdjustControlResponse => {
1359
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1360
- recordTraceStep('resolve', 'failure',
1361
- {
1362
- reason,
1363
- failure_code: failureCode
1364
- })
1365
- }
1366
- if (!traceSteps.some((step) => step.stage === 'recover')) {
1367
- recordTraceStep('recover', retryable ? 'retry' : 'failure', {
1368
- reason,
1369
- failure_code: failureCode,
1370
- retry_allowed: retryable,
1371
- recovery_attempts: attempts,
1372
- retry_depth: attempts
1373
- })
1374
- }
1375
- const responseTolerance = requestedTolerance ?? 0
1376
- const base = buildActionExecutionResult({
1377
- actionType,
1378
- sourceModule: 'interact',
1379
- device,
1380
- selector: targetSelector,
1381
- resolved,
1382
- success: false,
1383
- uiFingerprintBefore: fingerprintBefore,
1384
- uiFingerprintAfter,
1385
- failure: { failureCode, retryable },
1386
- details: {
1387
- target_value: targetValue,
1388
- tolerance: responseTolerance,
1389
- property,
1390
- attempts,
1391
- adjustment_mode: adjustmentMode,
1392
- actual_state: actualState,
1393
- converged: false,
1394
- within_tolerance: false,
1395
- reason
1396
- },
1397
- traceSteps
1398
- }) as AdjustControlResponse
1399
-
1400
- return {
1401
- ...base,
1402
- target_state: {
1403
- property,
1404
- target_value: targetValue,
1405
- tolerance: responseTolerance
1406
- },
1407
- actual_state: actualState,
1408
- within_tolerance: false,
1409
- converged: false,
1410
- attempts,
1411
- adjustment_mode: adjustmentMode
1412
- }
1413
- }
1414
-
1415
- const resolveCurrentMatch = async (): Promise<{
1416
- tree: any
1417
- device: any
1418
- match: { el: UiElement, idx: number } | null
1419
- resolvedTarget: ActionTargetResolved | null
1420
- } | null> => {
1421
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
1422
- resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform
1423
- resolvedDeviceId = tree?.device?.id || resolvedDeviceId
1424
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1425
-
1426
- if (element_id) {
1427
- const stored = ToolsInteract._resolvedUiElements.get(element_id)
1428
- if (!stored) {
1429
- return null
1430
- }
1431
- const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored)
1432
- if (!current) {
1433
- return null
1434
- }
1435
- return {
1436
- tree,
1437
- device: tree?.device,
1438
- match: { el: current.el, idx: current.index },
1439
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1440
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index),
1441
- current.el,
1442
- current.index
1443
- )
1444
- }
1445
- }
1446
-
1447
- if (semanticFallbackElement) {
1448
- const fallbackBounds = ToolsInteract._normalizeBounds(
1449
- Array.isArray(semanticFallbackElement.bounds)
1450
- ? semanticFallbackElement.bounds
1451
- : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
1452
- ? [
1453
- Number((semanticFallbackElement.bounds as any).left),
1454
- Number((semanticFallbackElement.bounds as any).top),
1455
- Number((semanticFallbackElement.bounds as any).right),
1456
- Number((semanticFallbackElement.bounds as any).bottom)
1457
- ]
1458
- : null
1459
- )
1460
-
1461
- let matchedIndex = -1
1462
- if (fallbackBounds) {
1463
- matchedIndex = elements.findIndex((el) => {
1464
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
1465
- return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3]
1466
- })
1467
- }
1468
-
1469
- if (matchedIndex === -1 && fallbackBounds) {
1470
- const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2)
1471
- const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2)
1472
- let bestDistance = Infinity
1473
- for (let i = 0; i < elements.length; i++) {
1474
- const el = elements[i]
1475
- if (!ToolsInteract._isAdjustableControl(el)) continue
1476
- const bounds = ToolsInteract._normalizeBounds(el.bounds)
1477
- if (!bounds) continue
1478
- const centerX = Math.floor((bounds[0] + bounds[2]) / 2)
1479
- const centerY = Math.floor((bounds[1] + bounds[3]) / 2)
1480
- const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY)
1481
- if (distance < bestDistance) {
1482
- bestDistance = distance
1483
- matchedIndex = i
1484
- }
1485
- }
1486
- }
1487
-
1488
- if (matchedIndex >= 0 && elements[matchedIndex]) {
1489
- const matched = { el: elements[matchedIndex], idx: matchedIndex }
1490
- return {
1491
- tree,
1492
- device: tree?.device,
1493
- match: matched,
1494
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1495
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
1496
- matched.el,
1497
- matched.idx
1498
- )
1499
- }
1500
- }
1501
- }
1502
-
1503
- if (selector) {
1504
- const matched = ToolsInteract._findFirstMatchingElement(elements, selector)
1505
- if (!matched) {
1506
- return null
1507
- }
1508
- return {
1509
- tree,
1510
- device: tree?.device,
1511
- match: matched,
1512
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1513
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
1514
- matched.el,
1515
- matched.idx
1516
- )
1517
- }
1518
- }
1519
-
1520
- return null
1521
- }
1522
-
1523
- if ((!selector && !element_id) || (selector && element_id)) {
1524
- return buildFailure('ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', null, undefined, null, 0, 'gesture', false)
1525
- }
1526
-
1527
- if (selector && !element_id) {
1528
- const waitResult = await ToolsInteract.waitForUIHandler({
1529
- selector,
1530
- condition: 'clickable',
1531
- timeout_ms: 5000,
1532
- poll_interval_ms: 300,
1533
- platform: resolvedPlatform,
1534
- deviceId: resolvedDeviceId
1535
- }) as any
1536
-
1537
- if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
1538
- const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
1539
- if (!semanticQuery) {
1540
- return buildFailure(
1541
- waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
1542
- waitResult?.error?.message ?? 'adjustable control not found',
1543
- null,
1544
- waitResult?.device,
1545
- null,
1546
- 0,
1547
- 'gesture',
1548
- waitResult?.error?.code === 'ELEMENT_NOT_FOUND'
1549
- )
1550
- }
1551
-
1552
- const fallback = await ToolsInteract.findElementHandler({
1553
- query: semanticQuery,
1554
- exact: false,
1555
- timeoutMs: 3000,
1556
- platform: resolvedPlatform,
1557
- deviceId: resolvedDeviceId
1558
- })
1559
-
1560
- if (!fallback.found || !fallback.element) {
1561
- return buildFailure(
1562
- 'ELEMENT_NOT_FOUND',
1563
- waitResult?.error?.message ?? 'adjustable control not found',
1564
- null,
1565
- waitResult?.device,
1566
- null,
1567
- 0,
1568
- 'gesture',
1569
- true
1570
- )
1571
- }
1572
-
1573
- semanticFallbackElement = fallback.element
1574
- } else {
1575
- element_id = waitResult.element.elementId
1576
- semanticFallbackElement = null
1577
- }
1578
- }
1579
-
1580
- let lastObservedState: { property: string; value: number | null; raw_value?: number | null } | null = null
1581
- let lastAdjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture'
1582
- let resolvedTarget: ActionTargetResolved | null = null
1583
- let currentDevice: any = undefined
1584
- let attemptCount = 0
1585
- let cachedResolvedMatch: { el: UiElement, idx: number } | null = null
1586
- let coordinateTapsAwaitingFreshGesture = false
1587
-
1588
- adjustmentAttempts:
1589
- for (let attempt = 0; attempt < attemptsLimit && attemptCount < attemptsLimit; attempt++) {
1590
- const resolved: {
1591
- tree: any
1592
- device: any
1593
- match: { el: UiElement, idx: number } | null
1594
- resolvedTarget: ActionTargetResolved | null
1595
- } | null = cachedResolvedMatch
1596
- ? {
1597
- tree: null,
1598
- device: currentDevice,
1599
- match: cachedResolvedMatch,
1600
- resolvedTarget: ToolsInteract._resolvedTargetFromElement(
1601
- ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx),
1602
- cachedResolvedMatch.el,
1603
- cachedResolvedMatch.idx
1604
- )
1605
- }
1606
- : await resolveCurrentMatch()
1607
- if (!resolved || !resolved.match || !resolved.resolvedTarget) {
1608
- return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true)
1609
- }
1610
-
1611
- currentDevice = resolved.device
1612
- resolvedTarget = resolved.resolvedTarget
1613
- const currentEl: UiElement = resolved.match.el
1614
- cachedResolvedMatch = resolved.match
1615
-
1616
- const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id)
1617
- if (!adjustableActionability.actionable) {
1618
- return buildFailure(
1619
- adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
1620
- adjustableActionability.reason ?? 'adjustable control is not actionable',
1621
- resolvedTarget,
1622
- currentDevice,
1623
- lastObservedState,
1624
- attemptCount,
1625
- lastAdjustmentMode,
1626
- true
1627
- )
1628
- }
1629
-
1630
- const elementsForInference = Array.isArray(resolved.tree?.elements)
1631
- ? resolved.tree.elements as UiElement[]
1632
- : cachedResolvedMatch
1633
- ? [cachedResolvedMatch.el]
1634
- : [currentEl]
1635
- const inferredSliderState = ToolsInteract._inferLabeledSliderState(elementsForInference, currentEl)
1636
- const nativeBounds = ToolsInteract._normalizeBounds(currentEl.bounds)
1637
- const bounds = inferredSliderState?.touch_bounds ?? nativeBounds
1638
- const valueRange = ToolsInteract._readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null
1639
- const step = ToolsInteract._readControlStep(currentEl)
1640
- const normalizedTargetValue = valueRange
1641
- ? ToolsInteract._quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step)
1642
- : targetValue
1643
- const effectiveTolerance = requestedTolerance ?? ToolsInteract._defaultAdjustTolerance(valueRange, step)
1644
- const currentValue = ToolsInteract._readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null
1645
- const actualState = currentValue !== null
1646
- ? {
1647
- property,
1648
- value: currentValue,
1649
- raw_value: typeof currentEl.state?.raw_value === 'number'
1650
- ? currentEl.state.raw_value
1651
- : inferredSliderState?.current
1652
- }
1653
- : null
1654
-
1655
- lastObservedState = actualState
1656
-
1657
- if (!traceSteps.some((step) => step.stage === 'resolve')) {
1658
- recordTraceStep('resolve', 'success', {
1659
- resolved_target: resolvedTarget,
1660
- current_value: currentValue,
1661
- adjustment_mode: lastAdjustmentMode
1662
- })
1663
- }
1664
-
1665
- if (property !== 'value' && property !== 'raw_value') {
1666
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1667
- }
1668
-
1669
- const isAdjustableControl = ToolsInteract._isAdjustableControl(currentEl) || !!inferredSliderState
1670
- if (!isAdjustableControl) {
1671
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1672
- }
1673
-
1674
- if (currentValue === null) {
1675
- return buildFailure('UNKNOWN', 'current numeric value is unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1676
- }
1677
-
1678
- if (valueRange && (targetValue < valueRange.min || targetValue > valueRange.max)) {
1679
- return buildFailure('CONTROL_CONVERGENCE_FAILED', `targetValue ${targetValue} is outside the control range ${valueRange.min}..${valueRange.max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1680
- }
1681
-
1682
- if (currentValue !== null && Math.abs(currentValue - normalizedTargetValue) <= effectiveTolerance) {
1683
- recordTraceStep('verify', 'success', {
1684
- property,
1685
- target_value: normalizedTargetValue,
1686
- actual_state: actualState,
1687
- reason: 'control already within tolerance'
1688
- })
1689
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1690
- const base = buildActionExecutionResult({
1691
- actionType,
1692
- sourceModule: 'interact',
1693
- device: currentDevice,
1694
- selector: targetSelector,
1695
- resolved: resolvedTarget,
1696
- success: true,
1697
- uiFingerprintBefore: fingerprintBefore,
1698
- uiFingerprintAfter,
1699
- details: {
1700
- target_value: normalizedTargetValue,
1701
- tolerance: effectiveTolerance,
1702
- property,
1703
- attempts: attemptCount,
1704
- adjustment_mode: 'semantic',
1705
- actual_state: actualState,
1706
- converged: true,
1707
- within_tolerance: true,
1708
- reason: 'control already within tolerance'
1709
- },
1710
- traceSteps
1711
- }) as AdjustControlResponse
1712
-
1713
- return {
1714
- ...base,
1715
- target_state: {
1716
- property,
1717
- target_value: normalizedTargetValue,
1718
- tolerance: effectiveTolerance
1719
- },
1720
- actual_state: actualState,
1721
- within_tolerance: true,
1722
- converged: true,
1723
- attempts: attemptCount,
1724
- adjustment_mode: 'semantic'
1725
- }
1726
- }
1727
-
1728
- if (!bounds) {
1729
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1730
- }
1731
-
1732
- const runVerification = async (): Promise<{
1733
- verification: any
1734
- observedState: { property: string; value: number | null; raw_value?: number | null } | null
1735
- withinTolerance: boolean
1736
- }> => {
1737
- if (inferredSliderState) {
1738
- const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
1739
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1740
- const refreshed = ToolsInteract._inferLabeledSliderState(elements, currentEl)
1741
- const observedValue = refreshed?.current ?? null
1742
- const observedState = observedValue !== null
1743
- ? { property, value: observedValue, raw_value: observedValue }
1744
- : actualState
1745
-
1746
- return {
1747
- verification: {
1748
- success: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance,
1749
- observed_state: observedState,
1750
- reason: observedValue !== null
1751
- ? 'visible labeled value read from nearby slider labels'
1752
- : 'visible labeled value unavailable'
1753
- },
1754
- observedState,
1755
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1756
- }
1757
- }
1758
-
1759
- const verification = await ToolsInteract.expectStateHandler({
1760
- element_id: resolvedTarget?.elementId ?? element_id,
1761
- selector: selector ?? undefined,
1762
- property,
1763
- expected: normalizedTargetValue,
1764
- platform: resolvedPlatform,
1765
- deviceId: resolvedDeviceId
1766
- }) as any
1767
-
1768
- const observedValue = ToolsInteract._parseNumericControlValue(verification?.observed_state?.value) ??
1769
- ToolsInteract._parseNumericControlValue(verification?.observed_state?.raw_value)
1770
- const observedState = observedValue !== null
1771
- ? {
1772
- property,
1773
- value: observedValue,
1774
- raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
1775
- }
1776
- : actualState
1777
-
1778
- return {
1779
- verification,
1780
- observedState,
1781
- withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
1782
- }
1783
- }
1784
-
1785
- let actionDevice: any = currentDevice
1786
- let observedState: { property: string; value: number | null; raw_value?: number | null } | null = actualState
1787
- let verification: any = null
1788
- let verificationResult: any = { verification: null, observedState: actualState, withinTolerance: false }
1789
-
1790
- const directActionSupported = ToolsInteract._hasDirectAdjustmentAction(currentEl)
1791
- if (directActionSupported && attemptCount < attemptsLimit) {
1792
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId)
1793
- if (typeof interact.setAdjustableValue === 'function') {
1794
- lastAdjustmentMode = 'semantic'
1795
- recordTraceStep('execute', 'retry', {
1796
- attempt: attemptCount + 1,
1797
- mode: 'semantic',
1798
- strategy: 'direct_set_value',
1799
- target_value: normalizedTargetValue
1800
- })
1801
- const actionResult = await interact.setAdjustableValue({
1802
- deviceId: actionResolved.id,
1803
- target: resolvedTarget,
1804
- property,
1805
- value: normalizedTargetValue
1806
- })
1807
- attemptCount++
1808
- actionDevice = actionResult?.device ?? actionDevice
1809
-
1810
- if (actionResult?.success !== false) {
1811
- verificationResult = await runVerification()
1812
- observedState = verificationResult.observedState
1813
- lastObservedState = observedState
1814
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1815
- attempt: attemptCount,
1816
- property,
1817
- target_value: normalizedTargetValue,
1818
- actual_state: observedState,
1819
- reason: verificationResult.verification?.reason ?? 'direct adjustment did not converge yet'
1820
- })
1821
-
1822
- if (verificationResult.withinTolerance) {
1823
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1824
- const base = buildActionExecutionResult({
1825
- actionType,
1826
- sourceModule: 'interact',
1827
- device: actionDevice ?? currentDevice,
1828
- selector: targetSelector,
1829
- resolved: resolvedTarget,
1830
- success: true,
1831
- uiFingerprintBefore: fingerprintBefore,
1832
- uiFingerprintAfter,
1833
- details: {
1834
- target_value: normalizedTargetValue,
1835
- tolerance: effectiveTolerance,
1836
- property,
1837
- attempts: attemptCount,
1838
- adjustment_mode: lastAdjustmentMode,
1839
- actual_state: observedState,
1840
- converged: true,
1841
- within_tolerance: true,
1842
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1843
- },
1844
- traceSteps
1845
- }) as AdjustControlResponse
1846
-
1847
- return {
1848
- ...base,
1849
- target_state: {
1850
- property,
1851
- target_value: normalizedTargetValue,
1852
- tolerance: effectiveTolerance
1853
- },
1854
- actual_state: observedState,
1855
- within_tolerance: true,
1856
- converged: true,
1857
- attempts: attemptCount,
1858
- adjustment_mode: lastAdjustmentMode
1859
- }
1860
- }
1861
-
1862
- cachedResolvedMatch = null
1863
- if (attemptCount < attemptsLimit) {
1864
- continue
1865
- }
1866
- }
1867
- }
1868
- }
1869
-
1870
- const directions = ToolsInteract._adjustmentDirections(currentEl)
1871
- if ((directions.increment || directions.decrement) && attemptCount < attemptsLimit) {
1872
- const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId)
1873
- const adjustByDirection = typeof interact.adjustAccessibleValue === 'function'
1874
- ? interact.adjustAccessibleValue.bind(interact)
1875
- : null
1876
- while (adjustByDirection && attemptCount < attemptsLimit) {
1877
- const direction = normalizedTargetValue > (observedState?.value ?? currentValue) ? 'increment' : 'decrement'
1878
- if ((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement)) break
1879
- lastAdjustmentMode = 'semantic'
1880
- recordTraceStep('execute', 'retry', {
1881
- attempt: attemptCount + 1,
1882
- mode: 'semantic',
1883
- strategy: 'accessibility_increment_decrement',
1884
- direction
1885
- })
1886
- const actionResult = await adjustByDirection({
1887
- deviceId: actionResolved.id,
1888
- target: resolvedTarget,
1889
- property,
1890
- direction
1891
- })
1892
- attemptCount++
1893
- actionDevice = actionResult?.device ?? actionDevice
1894
- if (actionResult?.success === false) break
1895
-
1896
- verificationResult = await runVerification()
1897
- observedState = verificationResult.observedState
1898
- lastObservedState = observedState
1899
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1900
- attempt: attemptCount,
1901
- property,
1902
- target_value: normalizedTargetValue,
1903
- actual_state: observedState,
1904
- reason: verificationResult.verification?.reason ?? 'incremental adjustment did not converge yet'
1905
- })
1906
-
1907
- if (verificationResult.withinTolerance) {
1908
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1909
- const base = buildActionExecutionResult({
1910
- actionType,
1911
- sourceModule: 'interact',
1912
- device: actionDevice ?? currentDevice,
1913
- selector: targetSelector,
1914
- resolved: resolvedTarget,
1915
- success: true,
1916
- uiFingerprintBefore: fingerprintBefore,
1917
- uiFingerprintAfter,
1918
- details: {
1919
- target_value: normalizedTargetValue,
1920
- tolerance: effectiveTolerance,
1921
- property,
1922
- attempts: attemptCount,
1923
- adjustment_mode: lastAdjustmentMode,
1924
- actual_state: observedState,
1925
- converged: true,
1926
- within_tolerance: true,
1927
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
1928
- },
1929
- traceSteps
1930
- }) as AdjustControlResponse
1931
-
1932
- return {
1933
- ...base,
1934
- target_state: {
1935
- property,
1936
- target_value: normalizedTargetValue,
1937
- tolerance: effectiveTolerance
1938
- },
1939
- actual_state: observedState,
1940
- within_tolerance: true,
1941
- converged: true,
1942
- attempts: attemptCount,
1943
- adjustment_mode: lastAdjustmentMode
1944
- }
1945
- }
1946
-
1947
- cachedResolvedMatch = null
1948
- if (attemptCount < attemptsLimit) {
1949
- continue adjustmentAttempts
1950
- }
1951
- }
1952
- }
1953
-
1954
- if (!bounds) {
1955
- return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1956
- }
1957
-
1958
- if (!valueRange) {
1959
- return buildFailure('UNKNOWN', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1960
- }
1961
-
1962
- const min = valueRange.min
1963
- const max = valueRange.max
1964
- const axis = ToolsInteract._controlAxis(currentEl, bounds)
1965
- const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, normalizedTargetValue, currentValue, min, max, axis)
1966
- const currentPoint = ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
1967
- if (!coordinateTapsAwaitingFreshGesture) {
1968
- const probePoints = inferredSliderState
1969
- ? [targetPoint]
1970
- : ToolsInteract._buildAdjustmentProbePoints(bounds, normalizedTargetValue, currentValue, min, max, axis)
1971
-
1972
- for (let i = 0; i < probePoints.length && attemptCount < attemptsLimit; i++) {
1973
- const probePoint = probePoints[i]
1974
- lastAdjustmentMode = 'coordinate'
1975
- recordTraceStep('execute', 'retry', {
1976
- attempt: attemptCount + 1,
1977
- mode: 'coordinate',
1978
- point: probePoint
1979
- })
1980
- const actionResult = await ToolsInteract.tapHandler({
1981
- platform: resolvedPlatform,
1982
- x: probePoint.x,
1983
- y: probePoint.y,
1984
- deviceId: resolvedDeviceId
1985
- })
1986
- attemptCount++
1987
- actionDevice = actionResult.device ?? actionDevice
1988
-
1989
- if (!actionResult.success) {
1990
- recordTraceStep('execute', 'retry', {
1991
- attempt: attemptCount,
1992
- mode: 'coordinate',
1993
- point: probePoint,
1994
- success: false
1995
- })
1996
- continue
1997
- }
1998
-
1999
- verificationResult = await runVerification()
2000
- observedState = verificationResult.observedState
2001
- lastObservedState = observedState
2002
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
2003
- attempt: attemptCount,
2004
- property,
2005
- target_value: normalizedTargetValue,
2006
- actual_state: observedState,
2007
- reason: verificationResult.verification?.reason ?? 'control did not converge yet'
2008
- })
2009
-
2010
- if (verificationResult.withinTolerance) {
2011
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2012
- const base = buildActionExecutionResult({
2013
- actionType,
2014
- sourceModule: 'interact',
2015
- device: actionDevice ?? currentDevice,
2016
- selector: targetSelector,
2017
- resolved: resolvedTarget,
2018
- success: true,
2019
- uiFingerprintBefore: fingerprintBefore,
2020
- uiFingerprintAfter,
2021
- details: {
2022
- target_value: normalizedTargetValue,
2023
- tolerance: effectiveTolerance,
2024
- property,
2025
- attempts: attemptCount,
2026
- adjustment_mode: lastAdjustmentMode,
2027
- actual_state: observedState,
2028
- converged: true,
2029
- within_tolerance: true,
2030
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
2031
- },
2032
- traceSteps
2033
- }) as AdjustControlResponse
2034
-
2035
- return {
2036
- ...base,
2037
- target_state: {
2038
- property,
2039
- target_value: normalizedTargetValue,
2040
- tolerance: effectiveTolerance
2041
- },
2042
- actual_state: observedState,
2043
- within_tolerance: true,
2044
- converged: true,
2045
- attempts: attemptCount,
2046
- adjustment_mode: lastAdjustmentMode
2047
- }
2048
- }
2049
-
2050
- coordinateTapsAwaitingFreshGesture = true
2051
- cachedResolvedMatch = null
2052
- if (attemptCount < attemptsLimit) {
2053
- continue adjustmentAttempts
2054
- }
2055
- }
2056
- }
2057
-
2058
- if (attemptCount < attemptsLimit) {
2059
- coordinateTapsAwaitingFreshGesture = false
2060
- lastAdjustmentMode = 'gesture'
2061
- recordTraceStep('execute', 'retry', {
2062
- attempt: attemptCount + 1,
2063
- mode: 'gesture',
2064
- start: currentPoint,
2065
- end: targetPoint
2066
- })
2067
- const fallbackActionResult = await ToolsInteract.swipeHandler({
2068
- platform: resolvedPlatform,
2069
- x1: currentPoint.x,
2070
- y1: currentPoint.y,
2071
- x2: targetPoint.x,
2072
- y2: targetPoint.y,
2073
- duration: inferredSliderState ? 520 : 220,
2074
- deviceId: resolvedDeviceId
2075
- })
2076
- attemptCount++
2077
- if (!fallbackActionResult.success) {
2078
- recordTraceStep('execute', 'failure', {
2079
- attempt: attemptCount,
2080
- mode: 'gesture',
2081
- start: currentPoint,
2082
- end: targetPoint,
2083
- success: false
2084
- })
2085
- return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
2086
- }
2087
-
2088
- actionDevice = fallbackActionResult.device ?? actionDevice
2089
- verificationResult = await runVerification()
2090
- observedState = verificationResult.observedState
2091
- lastObservedState = observedState
2092
- recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
2093
- attempt: attemptCount,
2094
- property,
2095
- target_value: normalizedTargetValue,
2096
- actual_state: observedState,
2097
- reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
2098
- })
2099
-
2100
- if (verificationResult.withinTolerance) {
2101
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2102
- const base = buildActionExecutionResult({
2103
- actionType,
2104
- sourceModule: 'interact',
2105
- device: actionDevice ?? currentDevice,
2106
- selector: targetSelector,
2107
- resolved: resolvedTarget,
2108
- success: true,
2109
- uiFingerprintBefore: fingerprintBefore,
2110
- uiFingerprintAfter,
2111
- details: {
2112
- target_value: normalizedTargetValue,
2113
- tolerance: effectiveTolerance,
2114
- property,
2115
- attempts: attemptCount,
2116
- adjustment_mode: lastAdjustmentMode,
2117
- actual_state: observedState,
2118
- converged: true,
2119
- within_tolerance: true,
2120
- reason: verificationResult.verification?.reason ?? 'control converged to target value'
2121
- },
2122
- traceSteps
2123
- }) as AdjustControlResponse
2124
-
2125
- return {
2126
- ...base,
2127
- target_state: {
2128
- property,
2129
- target_value: normalizedTargetValue,
2130
- tolerance: effectiveTolerance
2131
- },
2132
- actual_state: observedState,
2133
- within_tolerance: true,
2134
- converged: true,
2135
- attempts: attemptCount,
2136
- adjustment_mode: lastAdjustmentMode
2137
- }
2138
- }
2139
- }
2140
-
2141
- verification = verificationResult.verification
2142
- lastObservedState = observedState
2143
-
2144
- if (verificationResult.withinTolerance) {
2145
- recordTraceStep('verify', 'success', {
2146
- attempt: attemptCount,
2147
- property,
2148
- target_value: normalizedTargetValue,
2149
- actual_state: observedState,
2150
- reason: verification?.reason ?? 'control converged to target value'
2151
- })
2152
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2153
- const base = buildActionExecutionResult({
2154
- actionType,
2155
- sourceModule: 'interact',
2156
- device: actionDevice ?? currentDevice,
2157
- selector: targetSelector,
2158
- resolved: resolvedTarget,
2159
- success: true,
2160
- uiFingerprintBefore: fingerprintBefore,
2161
- uiFingerprintAfter,
2162
- details: {
2163
- target_value: normalizedTargetValue,
2164
- tolerance: effectiveTolerance,
2165
- property,
2166
- attempts: attemptCount,
2167
- adjustment_mode: lastAdjustmentMode,
2168
- actual_state: observedState,
2169
- converged: true,
2170
- within_tolerance: true,
2171
- reason: verification?.reason ?? 'control converged to target value'
2172
- },
2173
- traceSteps
2174
- }) as AdjustControlResponse
2175
-
2176
- return {
2177
- ...base,
2178
- target_state: {
2179
- property,
2180
- target_value: normalizedTargetValue,
2181
- tolerance: effectiveTolerance
2182
- },
2183
- actual_state: observedState,
2184
- within_tolerance: true,
2185
- converged: true,
2186
- attempts: attemptCount,
2187
- adjustment_mode: lastAdjustmentMode
2188
- }
2189
- }
2190
-
2191
- cachedResolvedMatch = inferredSliderState
2192
- ? null
2193
- : {
2194
- el: {
2195
- ...currentEl,
2196
- state: {
2197
- ...(currentEl.state ?? null),
2198
- ...(observedState ? {
2199
- [observedState.property]: observedState.value,
2200
- raw_value: observedState.raw_value ?? observedState.value
2201
- } : {})
2202
- }
2203
- },
2204
- idx: resolved.match.idx
2205
- }
2206
- }
2207
-
2208
- const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
2209
- return buildFailure('CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter)
198
+ static async adjustControlHandler(input: AdjustmentInput): Promise<AdjustControlResponse> {
199
+ return adjustControl(input, {
200
+ observe: (platform, deviceId) => ToolsObserve.getUITreeHandler({ platform, deviceId }),
201
+ fingerprint: (platform, deviceId) => ToolsInteract._captureFingerprint(platform, deviceId),
202
+ getHandle: id => ToolsInteract._resolvedUiElements.get(id),
203
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
204
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx),
205
+ wait: input => ToolsInteract.waitForUIHandler(input), find: input => ToolsInteract.findElementHandler(input),
206
+ expectState: input => ToolsInteract.expectStateHandler(input), service: (platform, deviceId) => ToolsInteract.getInteractionService(platform, deviceId),
207
+ tap: input => ToolsInteract.tapHandler(input), swipe: input => ToolsInteract.swipeHandler(input)
208
+ })
2210
209
  }
2211
210
 
2212
211
  static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId, timeoutMs }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string, timeoutMs?: number }) {
@@ -2229,1454 +228,40 @@ export class ToolsInteract {
2229
228
  return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id)
2230
229
  }
2231
230
 
2232
- static async findElementHandler({ query, exact = false, timeoutMs = 3000, platform, deviceId }: { query: string, exact?: boolean, timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string }): Promise<FindElementResponse> {
2233
- // Try to use observe layer to fetch the current UI tree and perform a fast semantic search
2234
- const start = Date.now()
2235
- const deadline = start + timeoutMs
2236
- const normalize = ToolsInteract._normalize
2237
-
2238
- const q = normalize(query)
2239
- if (!q) return { found: false, error: 'Empty query' }
2240
-
2241
- let best: RankedResolutionCandidate | null = null
2242
- let bestTree: any = null
2243
- let bestIterationCandidates: RankedResolutionCandidate[] = []
2244
- let shouldStop = false
2245
-
2246
- const scoreElement = (el: UiElement | null, idx: number): RankedResolutionCandidate | null => {
2247
- if (!el || !el.visible) return null
2248
- const bounds = el.bounds || [0,0,0,0]
2249
- if (!Array.isArray(bounds) || bounds.length < 4) return null
2250
- const [l,t,r,b] = bounds
2251
- if (r <= l || b <= t) return null
2252
- // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
2253
- const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el))
2254
-
2255
- const text = normalize(el.text ?? el.label ?? el.value ?? '')
2256
- const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '')
2257
- const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
2258
- const className = normalize(el.type ?? el.class ?? '')
2259
- const semanticRole = normalize(el.semantic?.semantic_role ?? '')
2260
- const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : []
2261
-
2262
- let score = 0
2263
- let reason = 'best_scoring_candidate'
2264
- if (exact) {
2265
- if (text && text === q) {
2266
- score = 1.0
2267
- reason = 'exact_text_match'
2268
- } else if (content && content === q) {
2269
- score = 0.95
2270
- reason = 'exact_content_desc_match'
2271
- } else if (resourceId && resourceId === q) {
2272
- score = 0.92
2273
- reason = 'exact_resource_id_match'
2274
- } else if (className && className === q) {
2275
- score = 0.3
2276
- reason = 'exact_class_match'
2277
- }
2278
- } else {
2279
- if (text && text === q) {
2280
- score = 1.0
2281
- reason = 'exact_text_match'
2282
- } else if (content && content === q) {
2283
- score = 0.95
2284
- reason = 'exact_content_desc_match'
2285
- } else if (resourceId && resourceId === q) {
2286
- score = 0.92
2287
- reason = 'exact_resource_id_match'
2288
- } else if (text && text.includes(q)) {
2289
- score = 0.6
2290
- reason = 'partial_text_match'
2291
- } else if (content && content.includes(q)) {
2292
- score = 0.55
2293
- reason = 'partial_content_desc_match'
2294
- } else if (resourceId && resourceId.includes(q)) {
2295
- score = 0.7
2296
- reason = 'partial_resource_id_match'
2297
- } else if (className && className.includes(q)) {
2298
- score = 0.3
2299
- reason = 'partial_class_match'
2300
- }
2301
- }
2302
- if (!exact) {
2303
- if (!score && semanticRole && semanticRole.includes(q)) {
2304
- score = 0.5
2305
- reason = 'semantic_role_match'
2306
- }
2307
- if (semanticActions.some((action) => action.includes(q))) {
2308
- score = Math.max(score, score > 0 ? 0.65 : 0.6)
2309
- reason = 'semantic_action_match'
2310
- }
2311
- if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
2312
- score = 0.45
2313
- reason = 'semantic_control_match'
2314
- }
2315
- } else {
2316
- if (!score && semanticRole && semanticRole === q) {
2317
- score = 0.5
2318
- reason = 'semantic_role_match'
2319
- }
2320
- if (semanticActions.some((action) => action === q)) {
2321
- score = Math.max(score, score > 0 ? 0.65 : 0.6)
2322
- reason = 'semantic_action_match'
2323
- }
2324
- }
2325
- if (score > 0 && interactable) score += 0.05
2326
- if (score <= 0) return null
2327
- return { el, idx, score, reason, interactable }
2328
- }
2329
-
2330
- while (Date.now() <= deadline) {
2331
- try {
2332
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
2333
- if (tree && Array.isArray((tree as any).elements)) {
2334
- const elements = ((tree as any).elements as UiElement[])
2335
- const iterationCandidates: RankedResolutionCandidate[] = []
2336
- let iterationImprovedBest = false
2337
- for (let i = 0; i < elements.length; i++) {
2338
- const el = elements[i]
2339
- try {
2340
- const candidate = scoreElement(el, i)
2341
- if (!candidate) continue
2342
- iterationCandidates.push(candidate)
2343
- if (!best || candidate.score > best.score) {
2344
- best = candidate
2345
- bestTree = tree
2346
- iterationImprovedBest = true
2347
- if (best.score >= 0.95) {
2348
- shouldStop = true
2349
- break
2350
- }
2351
- }
2352
- } catch (e) { console.error('Error scoring element:', e) }
2353
- }
2354
- if (iterationImprovedBest) {
2355
- bestIterationCandidates = iterationCandidates.slice()
2356
- }
2357
- }
2358
- } catch (e) { console.error('Error fetching UI tree:', e) }
2359
- if (shouldStop || Date.now() > deadline) break
2360
- await new Promise(r => setTimeout(r, 100))
2361
- }
2362
-
2363
- if (!best) return { found: false, error: 'Element not found' }
2364
-
2365
- // If the best match is not interactable, try to resolve an actionable ancestor.
2366
- try {
2367
- const elements = (bestTree && Array.isArray(bestTree.elements)) ? (bestTree.elements as UiElement[]) : []
2368
- const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution as UiResolution : null
2369
- let chosen = best as { el: UiElement, idx: number }
2370
- const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null
2371
-
2372
- // Strategy 1: if parentId references an index, climb that chain
2373
- let resolvedAncestor: { el: UiElement, idx: number } | null = null
2374
- if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
2375
- let cur = chosen
2376
- let safety = 0
2377
- while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
2378
- let pid = cur.el.parentId
2379
- let idx: number | null = null
2380
- if (typeof pid === 'number') idx = pid
2381
- else if (typeof pid === 'string' && /^\d+$/.test(pid)) idx = Number(pid)
2382
- // If parentId is not an index, try to find by matching resourceId or id field
2383
- if (idx !== null && elements[idx]) {
2384
- cur = { el: elements[idx], idx }
2385
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
2386
- } else if (typeof pid === 'string') {
2387
- // fallback: search elements for matching resourceId or id
2388
- const foundIndex = elements.findIndex((el: UiElement)=> (el.resourceId === pid || el.id === pid))
2389
- const found = foundIndex >= 0 ? elements[foundIndex] : null
2390
- if (found) {
2391
- cur = { el: found, idx: foundIndex }
2392
- if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
2393
- // otherwise continue climbing if this found element has its own parentId
2394
- } else {
2395
- break
2396
- }
2397
- } else {
2398
- break
2399
- }
2400
- safety++
2401
- }
2402
- }
2403
-
2404
- // Strategy 2: fallback - find a clickable element whose bounds fully contain the child's bounds
2405
- if (!resolvedAncestor && childBounds) {
2406
- const [cl,ct,cr,cb] = childBounds
2407
- // find candidates that are clickable and contain the child bounds
2408
- const candidates = elements
2409
- .map((el: UiElement, idx: number) => ({ el, idx }))
2410
- .filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds!.length >= 4)
2411
- let bestCandidate: { el: UiElement, idx: number } | null = null
2412
- let bestCandidateArea = Infinity
2413
- for (const c of candidates) {
2414
- const bounds = c.el.bounds as number[]
2415
- const [pl,pt,pr,pb] = bounds
2416
- if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
2417
- const area = (pr-pl) * (pb-pt)
2418
- if (area < bestCandidateArea) { bestCandidateArea = area; bestCandidate = c }
2419
- }
2420
- }
2421
- if (bestCandidate) resolvedAncestor = bestCandidate
2422
- }
2423
-
2424
- if (resolvedAncestor) {
2425
- best = {
2426
- el: resolvedAncestor.el,
2427
- idx: resolvedAncestor.idx,
2428
- score: Math.min(1, best.score + 0.02),
2429
- reason: 'clickable_parent_preferred',
2430
- interactable: true
2431
- }
2432
- }
2433
-
2434
- if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
2435
- const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen)
2436
- if (nearbyActionable) {
2437
- best = {
2438
- el: nearbyActionable.el,
2439
- idx: nearbyActionable.idx,
2440
- score: Math.min(1, best.score + 0.02),
2441
- reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
2442
- interactable: true
2443
- }
2444
- }
2445
- }
2446
- } catch (e) { console.error('Error resolving ancestor:', e) }
2447
-
2448
- if (!best) return { found: false, error: 'Element not found' }
2449
-
2450
- const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null
2451
- const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null
2452
- const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el))
2453
- const alternateCandidates = uniqueRanked
2454
- .filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
2455
- .slice(0, 3)
2456
- .map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate))
2457
-
2458
- const outEl = {
2459
- text: best.el.text ?? null,
2460
- resourceId: best.el.resourceId ?? null,
2461
- contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
2462
- class: best.el.type ?? best.el.class ?? null,
2463
- bounds: boundsObj,
2464
- clickable: !!best.el.clickable,
2465
- enabled: !!best.el.enabled,
2466
- stable_id: best.el.stable_id ?? null,
2467
- role: best.el.role ?? null,
2468
- test_tag: best.el.test_tag ?? null,
2469
- selector: best.el.selector ?? null,
2470
- semantic: best.el.semantic ?? null,
2471
- tapCoordinates,
2472
- telemetry: {
2473
- matchedIndex: best.idx ?? null,
2474
- matchedInteractable: !!best.interactable,
2475
- sliderLike: best.reason === 'slider_track_preferred'
2476
- }
2477
- }
2478
- if (best.reason === 'slider_track_preferred') {
2479
- const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left)
2480
- const interactionHint = {
2481
- kind: 'slider',
2482
- axis: isVertical ? 'vertical' : 'horizontal',
2483
- trackBounds: boundsObj
2484
- }
2485
- ;(outEl as any).interactionHint = interactionHint
2486
- }
2487
- const scoreVal = Math.min(1, Number(best.score.toFixed(3)))
2488
- const resolution: FindElementResolutionSummary = {
2489
- confidence: scoreVal,
2490
- reason: best.reason,
2491
- fallback_available: alternateCandidates.length > 0,
2492
- matched_count: uniqueRanked.length,
2493
- alternates: alternateCandidates
2494
- }
2495
- return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution }
231
+ static async findElementHandler(input: FindInput): Promise<FindElementResponse> {
232
+ return findElement(input, target => ToolsObserve.getUITreeHandler(target))
2496
233
  }
2497
234
 
2498
- static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false, resolveTapTarget = false }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string, singleObservation?: boolean, singleAttempt?: boolean, rejectAmbiguous?: boolean, resolveTapTarget?: boolean }) {
2499
- const readinessTracker = currentReadinessWait()
2500
- const overallStart = Date.now()
2501
- const requestedIndex = typeof match?.index === 'number' ? match.index : null
2502
- const requested = {
2503
- selector: selector ?? {},
2504
- condition,
2505
- match: requestedIndex === null ? null : { index: requestedIndex }
2506
- }
2507
-
2508
- // Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
2509
- const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
2510
- const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
2511
- const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
2512
-
2513
- if (!hasText && !hasResId && !hasAccId) {
2514
- return {
2515
- status: 'timeout',
2516
- error: {
2517
- code: 'INVALID_SELECTOR',
2518
- message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
2519
- },
2520
- metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
2521
- requested,
2522
- observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
2523
- };
2524
- }
2525
-
2526
- // Validate condition
2527
- if (!['exists','not_exists','visible','clickable'].includes(condition)) {
2528
- return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
2529
- }
2530
-
2531
- // Platform check
2532
- if (platform && !['android','ios'].includes(platform)) {
2533
- return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
2534
- }
2535
-
2536
- const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000))
2537
- const maxAttempts = (retry && retry.max_attempts) ? Math.max(1, retry.max_attempts) : 1
2538
- const backoff = (retry && retry.backoff_ms) ? Math.max(0, retry.backoff_ms) : 0
2539
-
2540
- let attempts = 0
2541
- let totalPollCount = 0
2542
- let lastMatchedCount = 0
2543
- let lastMatchedElement: ActionTargetResolved | null = null
2544
- let lastConditionSatisfied = false
2545
- let matchedAt: number | null = null
2546
- let stableMatchCount = 0
2547
- const stableObservationCount = singleObservation ? 1 : 2
2548
- const snapshotStaleThresholdMs = 500
2549
-
2550
- // Precompute normalized selector values and helpers (constant across polls)
2551
- const normalize = ToolsInteract._normalize
2552
- const containsFlag = !!selector?.contains
2553
- const selText = normalize(selector?.text)
2554
- const selRid = normalize(selector?.resource_id)
2555
- const selAid = normalize(selector?.accessibility_id)
2556
-
2557
- try {
2558
- while (attempts < maxAttempts) {
2559
- attempts++
2560
- const attemptStart = Date.now()
2561
- const deadline = attemptStart + (timeout_ms || 0)
2562
-
2563
- while (Date.now() <= deadline || (singleAttempt && totalPollCount === 0)) {
2564
- remainingBudget()
2565
- totalPollCount++
2566
- try {
2567
- const acquisitionStarted = Date.now()
2568
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
2569
- observeReadinessWait(tree, acquisitionStarted)
2570
- // A failed acquisition cannot establish that the requested tap target is absent.
2571
- if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error))) throw new Error('ACTION_TIMEOUT')
2572
- const elements = (tree && Array.isArray(tree.elements)) ? tree.elements as any[] : []
2573
-
2574
- const matches: { el: any, idx: number }[] = []
2575
-
2576
- for (let i = 0; i < elements.length; i++) {
2577
- const el = elements[i]
2578
- let ok = true
2579
-
2580
- // text
2581
- if (selector.text !== undefined && selector.text !== null) {
2582
- const val = normalize(el.text || el.label || el.value || '')
2583
- if (containsFlag) {
2584
- if (!val.includes(selText)) ok = false
2585
- } else {
2586
- if (val !== selText) ok = false
2587
- }
2588
- }
2589
-
2590
- // resource_id
2591
- if (ok && selector.resource_id !== undefined && selector.resource_id !== null) {
2592
- const rid = normalize(el.resourceId || el.resourceID || el.id || '')
2593
- if (containsFlag) {
2594
- if (!rid.includes(selRid)) ok = false
2595
- } else {
2596
- if (rid !== selRid) ok = false
2597
- }
2598
- }
2599
-
2600
- // accessibility_id
2601
- if (ok && selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
2602
- const aid = normalize(el.contentDescription || el.contentDesc || el.accessibilityLabel || el.label || '')
2603
- if (containsFlag) {
2604
- if (!aid.includes(selAid)) ok = false
2605
- } else {
2606
- if (aid !== selAid) ok = false
2607
- }
2608
- }
2609
-
2610
- if (ok) matches.push({ el, idx: i })
2611
- }
2612
-
2613
- // Evaluate condition
2614
- const matchedCount = matches.length
2615
- if (rejectAmbiguous && matchedCount > 1 && match?.index === undefined) return { status: 'timeout', error: { code: 'AMBIGUOUS_TARGET' } }
2616
- const pickIndex = (typeof match?.index === 'number') ? match!.index as number : undefined
2617
- let chosen: { el: any, idx: number } | null = null
2618
- if (matches.length > 0) {
2619
- if (pickIndex !== undefined) {
2620
- // If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
2621
- if (pickIndex >= 0 && pickIndex < matches.length) chosen = matches[pickIndex]
2622
- else chosen = null
2623
- } else {
2624
- chosen = matches[0]
2625
- }
2626
- } else {
2627
- chosen = null
2628
- }
2629
-
2630
- let conditionMet = false
2631
- let matchedElement = chosen
2632
- if (condition === 'exists') {
2633
- // when an index is specified, existence requires that specific index be present
2634
- conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1)
2635
- } else if (condition === 'not_exists') {
2636
- // when an index is specified, not_exists is true if that index is absent
2637
- conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0)
2638
- } else if (condition === 'visible') {
2639
- if (chosen) {
2640
- const b = chosen.el.bounds
2641
- const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
2642
- conditionMet = visibleFlag
2643
- } else conditionMet = false
2644
- } else if (condition === 'clickable') {
2645
- matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
2646
- ? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen as { el: UiElement, idx: number }) || chosen) : null
2647
- if (matchedElement) {
2648
- const b = matchedElement.el.bounds
2649
- const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
2650
- const enabled = !!matchedElement.el.enabled
2651
- const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable
2652
- conditionMet = visibleFlag && enabled && clickable
2653
- } else conditionMet = false
2654
- }
2655
-
2656
- // Waiting for a label to exist/be visible is distinct from resolving its
2657
- // executable control. Reuse the clickable-wait ancestor policy without
2658
- // promoting hidden or disabled labels into an unrelated enabled target.
2659
- if (resolveTapTarget && conditionMet && matchedElement &&
2660
- ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
2661
- matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement
2662
- }
2663
- const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
2664
- const resolvedDeviceId = tree?.device?.id || deviceId
2665
- lastMatchedCount = matchedCount
2666
- lastConditionSatisfied = conditionMet
2667
- lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
2668
- const now = Date.now()
2669
-
2670
- const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null
2671
- const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs
2672
-
2673
- if (conditionMet && snapshotFresh) {
2674
- if (matchedAt === null) matchedAt = now
2675
- stableMatchCount++
2676
- if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2677
- const latency_ms = now - overallStart
2678
- const outEl = lastMatchedElement
2679
-
2680
- return {
2681
- status: 'success',
2682
- ...(singleObservation ? { _tree: tree } : {}),
2683
- matched: matchedCount,
2684
- element: outEl,
2685
- metrics: { latency_ms, poll_count: totalPollCount, attempts },
2686
- requested,
2687
- observed: {
2688
- matched_count: matchedCount,
2689
- condition_satisfied: true,
2690
- selected_index: outEl?.index ?? null,
2691
- last_matched_element: outEl
2692
- }
2693
- }
2694
- }
2695
- } else {
2696
- stableMatchCount = 0
2697
- matchedAt = null
2698
- }
2699
-
2700
- } catch (e) {
2701
- // Deadline expiry does not invalidate the last successfully observed state.
2702
- if (e instanceof Error && e.message === 'ACTION_TIMEOUT') throw e
2703
- if (readinessTracker && !String(e).includes('READINESS_')) readinessTracker.unavailable()
2704
- if (readinessTracker && isDeviceUnavailable(e)) throw e
2705
- // Non-fatal per-poll error; record and continue
2706
- if (!readinessTracker) console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e))
2707
- }
2708
-
2709
- if (singleAttempt) break
2710
- // Sleep until next poll
2711
- await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)))
2712
- }
2713
-
2714
- // Attempt timed out; if more attempts allowed, backoff then retry
2715
- if (attempts < maxAttempts) {
2716
- if (backoff > 0) await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)))
2717
- continue
2718
- }
2719
-
2720
- // Final failure for this call
2721
- const elapsed = Date.now() - overallStart
2722
- const observed = {
2723
- matched_count: lastMatchedCount,
2724
- condition_satisfied: lastConditionSatisfied,
2725
- selected_index: lastMatchedElement?.index ?? null,
2726
- last_matched_element: lastMatchedElement
2727
- }
2728
- const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
2729
- ? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
2730
- : ` observed ${lastMatchedCount} match(es)`
2731
- return {
2732
- status: 'timeout',
2733
- error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
2734
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2735
- requested,
2736
- observed
2737
- }
2738
- }
2739
-
2740
- } catch (err) {
2741
- if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT') throw err
2742
- if (readinessTracker && isDeviceUnavailable(err)) throw err
2743
- const elapsed = Date.now() - overallStart
2744
- return {
2745
- status: 'timeout',
2746
- error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
2747
- metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
2748
- requested,
2749
- observed: {
2750
- matched_count: lastMatchedCount,
2751
- condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
2752
- selected_index: lastMatchedElement?.index ?? null,
2753
- last_matched_element: lastMatchedElement
2754
- }
2755
- }
2756
- }
235
+ static async waitForUIHandler(input: UIWaitInput) {
236
+ return waitForUI(input, { observe: target => ToolsObserve.getUITreeHandler(target),
237
+ resolve: (platform, deviceId, element, index) => ToolsInteract._buildResolvedElement(platform, deviceId, element, index) })
2757
238
  }
2758
239
 
2759
- // Helper: normalize various log objects into plain message strings for comparison
2760
- private static _logsToMessages(logsArr: any[]): string[] {
2761
- if (!Array.isArray(logsArr)) return []
2762
- return logsArr.map((l: any) => {
2763
- if (typeof l === 'string') return l
2764
- if (l && (l.message || l.msg)) return l.message || l.msg
2765
- try { return JSON.stringify(l) } catch { return String(l) }
2766
- })
240
+ static async waitForScreenChangeHandler(input: ScreenWaitInput) {
241
+ return waitForScreenChange(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target) })
2767
242
  }
2768
243
 
2769
- static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }: { platform?: 'android' | 'ios', previousFingerprint: string, timeoutMs?: number, pollIntervalMs?: number, deviceId?: string }) {
2770
- const start = Date.now()
2771
- let lastFingerprint: string | null = null
2772
- let lastActivity: string | null = null
2773
-
2774
- while (Date.now() - start < timeoutMs) {
2775
- try {
2776
- const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
2777
- const fp = res?.fingerprint ?? null
2778
- lastActivity = (res as any)?.activity ?? lastActivity
2779
- if (fp === null || fp === undefined) {
2780
- lastFingerprint = null
2781
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
2782
- continue
2783
- }
2784
-
2785
- lastFingerprint = fp
2786
-
2787
- if (fp !== previousFingerprint) {
2788
- // Stability confirmation
2789
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
2790
- try {
2791
- const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
2792
- const confirmFp = confirmRes?.fingerprint ?? null
2793
- lastActivity = (confirmRes as any)?.activity ?? lastActivity
2794
- if (confirmFp === fp) {
2795
- return {
2796
- success: true,
2797
- previousFingerprint,
2798
- newFingerprint: fp,
2799
- elapsedMs: Date.now() - start,
2800
- observed_screen: {
2801
- fingerprint: fp,
2802
- activity: lastActivity
2803
- }
2804
- }
2805
- }
2806
- lastFingerprint = confirmFp
2807
- continue
2808
- } catch (e) { console.error('Error confirming fingerprint:', e); continue }
2809
- }
2810
- } catch (e) { console.error('Error getting screen fingerprint:', e) }
2811
-
2812
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
2813
- }
2814
-
2815
- return {
2816
- success: false,
2817
- reason: 'timeout',
2818
- previousFingerprint,
2819
- lastFingerprint,
2820
- elapsedMs: Date.now() - start,
2821
- observed_screen: {
2822
- fingerprint: lastFingerprint,
2823
- activity: lastActivity
2824
- }
2825
- }
244
+ static async waitForUIChangeHandler(input: ChangeInput): Promise<WaitForUIChangeResponse> {
245
+ return waitForUIChange(input, { observe: target => ToolsObserve.getUITreeHandler(target), scope: (tree, scope, target) => ToolsInteract._resolveUiChangeScope(tree, scope, target) })
2826
246
  }
2827
247
 
2828
- static async waitForUIChangeHandler({
2829
- platform,
2830
- deviceId,
2831
- timeout_ms = 60000,
2832
- stability_window_ms = 300,
2833
- expected_change,
2834
- scope = 'screen',
2835
- target = null
2836
- }: {
2837
- platform?: 'android' | 'ios',
2838
- deviceId?: string,
2839
- timeout_ms?: number,
2840
- stability_window_ms?: number,
2841
- expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change',
2842
- scope?: 'screen' | 'subtree',
2843
- target?: string | null
2844
- }): Promise<WaitForUIChangeResponse> {
2845
- const readinessTracker = currentReadinessWait()
2846
- const start = Date.now()
2847
- const pollIntervalMs = 300
2848
- const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
2849
- let baseline: UiChangeSignatureSet | null = null
2850
- let baselineScope: UiChangeScopeResult | null = null
2851
- let lastObservedRevision: number | null = null
2852
- let lastLoadingState: any = null
2853
- let lastSnapshotFreshnessMs: number | null = null
2854
- let candidateSignatures: UiChangeSignatureSet | null = null
2855
- let candidateObservedChange: 'hierarchy_diff' | 'text_change' | 'state_change' | null = null
2856
- let candidateSinceMs: number | null = null
2857
- let lastChangeSummary: ReturnType<typeof ToolsInteract._summarizeUiChangeDelta> | null = null
2858
- let lastScopeResolution: UiChangeScopeResolution = {
2859
- scope: scope === 'subtree' ? 'subtree' : 'screen',
2860
- target: target && typeof target === 'string' ? target : null,
2861
- resolved: scope !== 'subtree',
2862
- resolvedIndex: null,
2863
- resolvedStableId: null,
2864
- reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
2865
- }
2866
-
2867
- while (Date.now() - start < timeout_ms) {
2868
- try {
2869
- if (readinessTracker) remainingBudget()
2870
- const acquisitionStarted = Date.now()
2871
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
2872
- observeReadinessWait(tree, acquisitionStarted)
2873
- const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
2874
- if (scopedTree.error) {
2875
- lastScopeResolution = scopedTree.resolution
2876
- return {
2877
- success: false,
2878
- observed_change: null,
2879
- snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
2880
- snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
2881
- timeout: true,
2882
- elapsed_ms: Date.now() - start,
2883
- expected_change,
2884
- loading_state: tree?.loading_state ?? lastLoadingState ?? null,
2885
- scope: scopedTree.resolution.scope,
2886
- target: scopedTree.resolution.target,
2887
- stability_state: 'transient',
2888
- change_summary: lastChangeSummary,
2889
- reason: scopedTree.error.message,
2890
- error: scopedTree.error
2891
- }
2892
- }
2893
-
2894
- const scopedElements = scopedTree.elements
2895
- const scopedSignatureTree = {
2896
- ...tree,
2897
- elements: scopedElements
2898
- }
2899
- const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree)
2900
- lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
2901
- lastLoadingState = tree?.loading_state ?? lastLoadingState
2902
- lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs
2903
- lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary
2904
- lastScopeResolution = scopedTree.resolution
2905
- baselineScope = baselineScope ?? scopedTree
2906
-
2907
- if (!baseline) {
2908
- baseline = signatures
2909
- baselineScope = scopedTree
2910
- } else {
2911
- const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
2912
- if (observedChange) {
2913
- if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
2914
- candidateSignatures = signatures
2915
- candidateObservedChange = observedChange
2916
- candidateSinceMs = Date.now()
2917
- }
2918
-
2919
- const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
2920
- if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
2921
- return {
2922
- success: true,
2923
- observed_change: candidateObservedChange ?? observedChange,
2924
- snapshot_revision: lastObservedRevision ?? undefined,
2925
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2926
- timeout: false,
2927
- elapsed_ms: Date.now() - start,
2928
- expected_change,
2929
- loading_state: lastLoadingState ?? null,
2930
- scope: lastScopeResolution.scope,
2931
- target: lastScopeResolution.target,
2932
- stability_state: 'stable',
2933
- change_summary: lastChangeSummary,
2934
- reason: 'UI change observed'
2935
- }
2936
- }
2937
- } else {
2938
- candidateSignatures = null
2939
- candidateObservedChange = null
2940
- candidateSinceMs = null
2941
- }
2942
- }
2943
- } catch (error) {
2944
- if (readinessTracker && !String(error).includes('READINESS_')) readinessTracker.unavailable()
2945
- if (readinessTracker && isDeviceUnavailable(error)) throw error
2946
- if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) { readinessTracker.deadline(); break }
2947
- // Default waits retain best-effort polling.
2948
- }
2949
-
2950
- await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)))
2951
- }
2952
-
2953
- return {
2954
- success: false,
2955
- observed_change: null,
2956
- snapshot_revision: lastObservedRevision ?? undefined,
2957
- snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2958
- timeout: true,
2959
- elapsed_ms: Date.now() - start,
2960
- expected_change,
2961
- loading_state: lastLoadingState ?? null,
2962
- scope: lastScopeResolution.scope,
2963
- target: lastScopeResolution.target,
2964
- stability_state: 'transient',
2965
- change_summary: lastChangeSummary,
2966
- reason: 'timeout'
2967
- }
248
+ static async expectScreenHandler(input: ScreenExpectation): Promise<ExpectScreenResponse> {
249
+ return expectScreen(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), current: target => ToolsObserve.getCurrentScreenHandler(target) })
2968
250
  }
2969
251
 
2970
- static async expectScreenHandler({
2971
- platform,
2972
- fingerprint,
2973
- screen,
2974
- deviceId
2975
- }: {
2976
- platform?: 'android' | 'ios',
2977
- fingerprint?: string,
2978
- screen?: string,
2979
- deviceId?: string
2980
- }): Promise<ExpectScreenResponse> {
2981
- const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
2982
- const observedScreen = {
2983
- fingerprint: observedFingerprint?.fingerprint ?? null,
2984
- screen: observedFingerprint?.activity ?? null
2985
- }
2986
-
2987
- let observedScreenLabel = observedScreen.screen
2988
- if (!fingerprint && screen && platform !== 'ios') {
2989
- try {
2990
- const current = await ToolsObserve.getCurrentScreenHandler({ deviceId }) as any
2991
- observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel
2992
- } catch {
2993
- // Keep fingerprint-derived activity when current-screen lookup is unavailable.
2994
- }
2995
- }
2996
-
2997
- const expectedScreen = {
2998
- fingerprint: fingerprint ?? null,
2999
- screen: screen ?? null
3000
- }
3001
-
3002
- let success = false
3003
- let basis: 'fingerprint' | 'screen' | 'none' = 'none'
3004
- let reason = 'No fingerprint or screen expectation provided'
3005
- if (fingerprint) {
3006
- basis = 'fingerprint'
3007
- success = observedScreen.fingerprint === fingerprint
3008
- reason = success
3009
- ? `observed fingerprint matches expected fingerprint ${fingerprint}`
3010
- : `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`
3011
- } else if (screen) {
3012
- basis = 'screen'
3013
- const candidates = new Set<string>()
3014
- if (observedScreen.screen) candidates.add(observedScreen.screen)
3015
- if (observedScreenLabel) candidates.add(observedScreenLabel)
3016
- success = candidates.has(screen)
3017
- reason = success
3018
- ? `observed screen matches expected screen ${screen}`
3019
- : `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`
3020
- }
3021
-
3022
- return {
3023
- success,
3024
- observed_screen: {
3025
- fingerprint: observedScreen.fingerprint,
3026
- screen: observedScreenLabel
3027
- },
3028
- expected_screen: expectedScreen,
3029
- confidence: success ? 1 : 0,
3030
- comparison: {
3031
- basis,
3032
- matched: success,
3033
- reason
3034
- },
3035
- trace: buildObservationTrace({
3036
- actionType: 'expect_screen',
3037
- stage: 'verify',
3038
- success,
3039
- attempts: 1,
3040
- metadata: {
3041
- expected_screen: expectedScreen,
3042
- observed_screen: {
3043
- fingerprint: observedScreen.fingerprint,
3044
- screen: observedScreenLabel
3045
- },
3046
- comparison: {
3047
- basis,
3048
- matched: success
3049
- }
3050
- }
3051
- })
3052
- }
252
+ static async expectElementVisibleHandler(input: VisibilityInput): Promise<ExpectElementVisibleResponse> {
253
+ return expectVisible(input, request => ToolsInteract.waitForUIHandler(request))
3053
254
  }
3054
255
 
3055
- static async expectElementVisibleHandler({
3056
- selector,
3057
- element_id,
3058
- timeout_ms = 5000,
3059
- poll_interval_ms = 300,
3060
- platform,
3061
- deviceId
3062
- }: {
3063
- selector: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
3064
- element_id?: string,
3065
- timeout_ms?: number,
3066
- poll_interval_ms?: number,
3067
- platform?: 'android' | 'ios',
3068
- deviceId?: string
3069
- }): Promise<ExpectElementVisibleResponse> {
3070
- const result = await ToolsInteract.waitForUIHandler({
3071
- selector,
3072
- condition: 'visible',
3073
- timeout_ms,
3074
- poll_interval_ms,
3075
- platform,
3076
- deviceId
3077
- }) as any
3078
-
3079
- if (result?.status === 'success' && result?.element) {
3080
- return {
3081
- success: true,
3082
- selector,
3083
- element_id: result.element.elementId ?? element_id ?? null,
3084
- expected_condition: 'visible',
3085
- element: {
3086
- elementId: result.element.elementId ?? null,
3087
- text: result.element.text ?? null,
3088
- resource_id: result.element.resource_id ?? null,
3089
- accessibility_id: result.element.accessibility_id ?? null,
3090
- class: result.element.class ?? null,
3091
- bounds: result.element.bounds ?? null,
3092
- index: typeof result.element.index === 'number' ? result.element.index : null,
3093
- state: (result.element as any).state ?? null,
3094
- stable_id: (result.element as any).stable_id ?? null,
3095
- role: (result.element as any).role ?? null,
3096
- test_tag: (result.element as any).test_tag ?? null,
3097
- selector: (result.element as any).selector ?? null,
3098
- semantic: (result.element as any).semantic ?? null
3099
- },
3100
- observed: {
3101
- status: result.status,
3102
- matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
3103
- condition_satisfied: true,
3104
- selected_index: typeof result.element.index === 'number' ? result.element.index : null,
3105
- last_matched_element: {
3106
- elementId: result.element.elementId ?? null,
3107
- text: result.element.text ?? null,
3108
- resource_id: result.element.resource_id ?? null,
3109
- accessibility_id: result.element.accessibility_id ?? null,
3110
- class: result.element.class ?? null,
3111
- bounds: result.element.bounds ?? null,
3112
- index: typeof result.element.index === 'number' ? result.element.index : null,
3113
- state: (result.element as any).state ?? null,
3114
- stable_id: (result.element as any).stable_id ?? null,
3115
- role: (result.element as any).role ?? null,
3116
- test_tag: (result.element as any).test_tag ?? null,
3117
- selector: (result.element as any).selector ?? null,
3118
- semantic: (result.element as any).semantic ?? null
3119
- }
3120
- },
3121
- reason: 'selector is visible',
3122
- trace: buildObservationTrace({
3123
- actionType: 'expect_element_visible',
3124
- stage: 'verify',
3125
- success: true,
3126
- attempts: 1,
3127
- metadata: {
3128
- selector,
3129
- element_id: result.element.elementId ?? element_id ?? null,
3130
- status: result.status
3131
- }
3132
- })
3133
- }
3134
- }
3135
-
3136
- const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT'
3137
- return {
3138
- success: false,
3139
- selector,
3140
- element_id: element_id ?? null,
3141
- expected_condition: 'visible',
3142
- observed: {
3143
- status: result?.status,
3144
- matched_count: result?.observed?.matched_count,
3145
- condition_satisfied: result?.observed?.condition_satisfied ?? false,
3146
- selected_index: result?.observed?.selected_index ?? null,
3147
- last_matched_element: result?.observed?.last_matched_element ?? null
3148
- },
3149
- reason: result?.error?.message ?? 'selector is not visible',
3150
- failure_code: errorCode,
3151
- retryable: errorCode === 'TIMEOUT',
3152
- trace: buildObservationTrace({
3153
- actionType: 'expect_element_visible',
3154
- stage: 'verify',
3155
- success: false,
3156
- attempts: 1,
3157
- metadata: {
3158
- selector,
3159
- element_id: element_id ?? null,
3160
- status: result?.status ?? null,
3161
- reason: result?.error?.message ?? 'selector is not visible'
3162
- }
3163
- })
3164
- }
3165
- }
3166
-
3167
- static async expectStateHandler({
3168
- selector,
3169
- element_id,
3170
- property,
3171
- expected,
3172
- platform,
3173
- deviceId,
3174
- stabilization_window_ms = 1000,
3175
- stable_observation_count = 2,
3176
- snapshot_stale_threshold_ms = 500,
3177
- poll_interval_ms = 150,
3178
- exact_budget = false
3179
- }: {
3180
- selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
3181
- element_id?: string,
3182
- property: string,
3183
- expected: boolean | number | string | Record<string, unknown>,
3184
- platform?: 'android' | 'ios',
3185
- deviceId?: string,
3186
- stabilization_window_ms?: number,
3187
- stable_observation_count?: number,
3188
- snapshot_stale_threshold_ms?: number,
3189
- poll_interval_ms?: number,
3190
- exact_budget?: boolean
3191
- }): Promise<ExpectStateResponse> {
3192
- const compareBoolean = (value: unknown) => typeof value === 'boolean' ? value : null
3193
- const compareString = (value: unknown) => typeof value === 'string' ? value : null
3194
- const compareNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value) ? value : null
3195
- const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
3196
- const start = Date.now()
3197
- const deadline = start + (exact_budget ? stabilization_window_ms : Math.max(500, stabilization_window_ms))
3198
- const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2))
3199
- const pollDelay = exact_budget ? Math.max(50, Math.min(poll_interval_ms || 150, 1000)) : Math.max(100, Math.min(poll_interval_ms || 150, 200))
3200
- const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800))
3201
-
3202
- let attempts = 0
3203
- let stableCount = 0
3204
- let lastReason = 'element not found'
3205
- let lastFailureCode: 'ELEMENT_NOT_FOUND' | 'UNKNOWN' = 'ELEMENT_NOT_FOUND'
3206
- let lastObservedElement: (ActionTargetResolved & { state?: UIElementState | null }) | null = null
3207
- let lastObservedValue: boolean | number | string | Record<string, unknown> | null = null
3208
- let lastRawValue: boolean | number | string | null = null
3209
- let lastResolvedElementId: string | null = element_id ?? null
3210
- const traceSteps: TraceStep[] = []
3211
- let traceAttemptIndex = 0
3212
- let resolveRecorded = false
3213
-
3214
- const recordTraceStep = (
3215
- stage: TraceStep['stage'],
3216
- result: TraceStep['result'],
3217
- metadata?: Record<string, unknown>
3218
- ) => {
3219
- traceSteps.push(createTraceStep({
3220
- stage,
3221
- timestamp: Date.now(),
3222
- result,
3223
- attemptIndex: traceAttemptIndex++,
3224
- metadata
3225
- }))
3226
- }
3227
-
3228
- const buildStateTrace = (outcome: 'success' | 'failure'): ActionTrace => ({
3229
- action_id: nextActionId('expect_state', Date.now()),
3230
- steps: traceSteps,
3231
- final_outcome: outcome,
3232
- attempts
3233
- })
3234
-
3235
- while (Date.now() <= deadline) {
3236
- attempts++
3237
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
3238
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
3239
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
3240
- const treeDeviceId = tree?.device?.id || deviceId
3241
- const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null
3242
- let matched: { el: UiElement, idx: number } | null = null
3243
-
3244
- if (element_id) {
3245
- const resolved = ToolsInteract._resolvedUiElements.get(element_id)
3246
- if (resolved) {
3247
- const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
3248
- if (current) matched = { el: current.el, idx: current.index }
3249
- }
3250
- }
3251
-
3252
- if (!matched && selector) {
3253
- matched = ToolsInteract._findFirstMatchingElement(elements, selector)
3254
- }
3255
-
3256
- if (!matched) {
3257
- lastReason = 'element not found'
3258
- lastFailureCode = 'ELEMENT_NOT_FOUND'
3259
- stableCount = 0
3260
- recordTraceStep('resolve', 'retry', {
3261
- selector: selector ?? null,
3262
- element_id: lastResolvedElementId,
3263
- matched: false,
3264
- reason: lastReason,
3265
- attempt: attempts
3266
- })
3267
- recordTraceStep('stabilize', 'retry', {
3268
- stabilization_attempts: attempts,
3269
- stable_observation_count: stableCount,
3270
- snapshot_freshness_ms: treeAgeMs,
3271
- reason: lastReason
3272
- })
3273
- await sleep(pollDelay)
3274
- continue
3275
- }
3276
-
3277
- const resolvedElement = ToolsInteract._resolvedTargetFromElement(
3278
- ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx),
3279
- matched.el,
3280
- matched.idx
3281
- )
3282
- lastResolvedElementId = resolvedElement.elementId
3283
- lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null }
3284
-
3285
- if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
3286
- lastReason = 'stale snapshot'
3287
- lastFailureCode = 'UNKNOWN'
3288
- stableCount = 0
3289
- recordTraceStep('resolve', 'retry', {
3290
- selector: selector ?? null,
3291
- element_id: lastResolvedElementId,
3292
- matched: true,
3293
- reason: lastReason,
3294
- attempt: attempts
3295
- })
3296
- recordTraceStep('stabilize', 'retry', {
3297
- stabilization_attempts: attempts,
3298
- stable_observation_count: stableCount,
3299
- snapshot_freshness_ms: treeAgeMs,
3300
- reason: lastReason
3301
- })
3302
- await sleep(pollDelay)
3303
- continue
3304
- }
3305
-
3306
- if (!resolveRecorded) {
3307
- recordTraceStep('resolve', 'success', {
3308
- selector: selector ?? null,
3309
- element_id: lastResolvedElementId,
3310
- matched: true,
3311
- reason: 'element resolved'
3312
- })
3313
- resolveRecorded = true
3314
- }
3315
-
3316
- const observedState = matched.el.state ?? null
3317
- const actual = observedState?.[property as keyof UIElementState] ?? null
3318
-
3319
- let success = false
3320
- let reason = ''
3321
- let rawValue: boolean | number | string | null = null
3322
- let observedValue: boolean | number | string | Record<string, unknown> | null = actual as any
3323
-
3324
- switch (property) {
3325
- case 'checked':
3326
- case 'focused':
3327
- case 'expanded':
3328
- case 'enabled': {
3329
- const expectedBool = compareBoolean(expected)
3330
- const actualBool = compareBoolean(actual)
3331
- if (expectedBool === null) {
3332
- reason = `expected ${property} must be boolean`
3333
- } else if (actualBool === null) {
3334
- reason = `${property} state unavailable`
3335
- } else {
3336
- rawValue = actualBool
3337
- success = actualBool === expectedBool
3338
- reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`
3339
- }
3340
- observedValue = actualBool
3341
- break
3342
- }
3343
- case 'value':
3344
- case 'raw_value': {
3345
- const expectedNumber = compareNumber(expected)
3346
- const actualNumber = compareNumber(actual)
3347
- if (expectedNumber !== null && actualNumber !== null) {
3348
- success = actualNumber === expectedNumber
3349
- rawValue = actualNumber
3350
- observedValue = actualNumber
3351
- reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`
3352
- break
3353
- }
3354
- const expectedString = typeof expected === 'string' ? expected : null
3355
- const actualString = compareString(actual)
3356
- if (expectedString !== null && actualString !== null) {
3357
- success = actualString === expectedString
3358
- rawValue = actualString
3359
- observedValue = actualString
3360
- reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`
3361
- } else {
3362
- reason = 'value state unavailable'
3363
- }
3364
- break
3365
- }
3366
- case 'selected': {
3367
- const expectedBool = typeof expected === 'boolean' ? expected : null
3368
- const expectedString = typeof expected === 'string'
3369
- ? expected
3370
- : expected && typeof expected === 'object'
3371
- ? String((expected as { id?: unknown; label?: unknown }).id ?? (expected as { id?: unknown; label?: unknown }).label ?? '')
3372
- : null
3373
- if (!observedState || observedState.selected === undefined || observedState.selected === null) {
3374
- reason = 'selected state unavailable'
3375
- break
3376
- }
3377
- if (expectedBool !== null) {
3378
- const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null
3379
- if (actualBool === null) {
3380
- reason = 'selected state is not boolean'
3381
- break
3382
- }
3383
- rawValue = actualBool
3384
- observedValue = actualBool
3385
- success = actualBool === expectedBool
3386
- reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`
3387
- break
3388
- }
3389
- const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
3390
- ? String((observedState.selected as { id?: unknown; label?: unknown }).id ?? (observedState.selected as { id?: unknown; label?: unknown }).label ?? '')
3391
- : String(observedState.selected)
3392
- const actualString = actualSelected.trim()
3393
- if (!expectedString) {
3394
- reason = 'expected selected must be boolean, string, or object with id/label'
3395
- break
3396
- }
3397
- rawValue = actualString
3398
- observedValue = actualString
3399
- success = actualString === expectedString
3400
- reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`
3401
- break
3402
- }
3403
- case 'text_value': {
3404
- const expectedString = typeof expected === 'string' ? expected : null
3405
- const actualString = compareString(actual)
3406
- if (!expectedString) {
3407
- reason = 'expected text_value must be string'
3408
- } else if (!actualString) {
3409
- reason = 'text_value state unavailable'
3410
- } else {
3411
- success = actualString === expectedString
3412
- rawValue = actualString
3413
- observedValue = actualString
3414
- reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`
3415
- }
3416
- break
3417
- }
3418
- default: {
3419
- if (actual !== null && actual !== undefined) {
3420
- success = actual === expected
3421
- observedValue = actual as any
3422
- rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null
3423
- reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`
3424
- } else {
3425
- reason = `unsupported or unavailable state property: ${property}`
3426
- }
3427
- }
3428
- }
3429
-
3430
- if (success) {
3431
- stableCount++
3432
- recordTraceStep('stabilize', 'success', {
3433
- stabilization_attempts: attempts,
3434
- stable_observation_count: stableCount,
3435
- snapshot_freshness_ms: treeAgeMs,
3436
- reason
3437
- })
3438
- recordTraceStep('verify', 'success', {
3439
- property,
3440
- expected,
3441
- observed_value: observedValue,
3442
- reason
3443
- })
3444
- if (stableCount >= stableTarget) {
3445
- return {
3446
- success: true,
3447
- selector,
3448
- element_id: lastResolvedElementId,
3449
- expected_state: { property, expected },
3450
- element: lastObservedElement,
3451
- observed_state: {
3452
- property,
3453
- value: observedValue,
3454
- ...(rawValue !== null ? { raw_value: rawValue } : {})
3455
- },
3456
- reason,
3457
- stabilization_attempts: attempts,
3458
- stabilization_window_ms: Date.now() - start,
3459
- stable_observation_count: stableCount,
3460
- snapshot_freshness_ms: treeAgeMs ?? undefined,
3461
- trace: buildStateTrace('success')
3462
- } as ExpectStateResponse & {
3463
- stabilization_attempts?: number;
3464
- stabilization_window_ms?: number;
3465
- stable_observation_count?: number;
3466
- snapshot_freshness_ms?: number;
3467
- }
3468
- }
3469
- } else {
3470
- stableCount = 0
3471
- lastReason = reason || lastReason
3472
- lastFailureCode = 'UNKNOWN'
3473
- recordTraceStep('stabilize', 'retry', {
3474
- stabilization_attempts: attempts,
3475
- stable_observation_count: stableCount,
3476
- snapshot_freshness_ms: treeAgeMs,
3477
- reason: lastReason
3478
- })
3479
- recordTraceStep('verify', 'retry', {
3480
- property,
3481
- expected,
3482
- observed_value: observedValue,
3483
- reason: lastReason
3484
- })
3485
- }
3486
-
3487
- if (!success) {
3488
- lastObservedValue = observedValue
3489
- lastRawValue = rawValue
3490
- }
3491
-
3492
- await sleep(pollDelay)
3493
- }
3494
-
3495
- return {
3496
- success: false,
3497
- selector,
3498
- element_id: lastResolvedElementId,
3499
- expected_state: { property, expected },
3500
- element: lastObservedElement,
3501
- observed_state: {
3502
- property,
3503
- value: lastObservedValue,
3504
- ...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
3505
- },
3506
- reason: lastReason,
3507
- failure_code: lastFailureCode,
3508
- retryable: true,
3509
- trace: buildStateTrace('failure')
3510
- }
256
+ static async expectStateHandler(input: StateInput): Promise<ExpectStateResponse> {
257
+ return expectState(input, { observe: target => ToolsObserve.getUITreeHandler(target), getHandle: id => ToolsInteract._resolvedUiElements.get(id),
258
+ currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
259
+ target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx) })
3511
260
  }
3512
261
 
3513
- static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }: { type?: 'ui' | 'log' | 'screen' | 'idle', query?: string, timeoutMs?: number, pollIntervalMs?: number, includeSnapshotOnFailure?: boolean, match?: 'present'|'absent', stability_ms?: number, observationDelayMs?: number, platform?: 'android' | 'ios', deviceId?: string }) {
3514
- const start = Date.now()
3515
- const deadline = start + (timeoutMs || 0)
3516
- const q = (query === null || query === undefined) ? '' : String(query)
3517
-
3518
- // Clamp polling interval to 250-500ms for consistent behavior
3519
- const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500))
3520
-
3521
- // Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
3522
- let initialFingerprint: string | null = null
3523
- let baselineLastLine: string | null = null
3524
- try {
3525
- const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as Promise<ScreenFingerprintResponse | null>
3526
- const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as Promise<any>
3527
- const withTimeout = (p: Promise<any>, ms: number) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))])
3528
- const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)])
3529
- if (fpRes && typeof fpRes === 'object') initialFingerprint = (fpRes as ScreenFingerprintResponse).fingerprint ?? null
3530
- if (gl) {
3531
- const logsArr = Array.isArray((gl as any).logs) ? (gl as any).logs : []
3532
- // Normalize to last message string for baseline comparison
3533
- const msgs = ToolsInteract._logsToMessages(logsArr)
3534
- baselineLastLine = msgs.length ? msgs[msgs.length - 1] : null
3535
- }
3536
- } catch (err) {
3537
- try { console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err)) } catch { }
3538
- }
3539
-
3540
- // Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
3541
- let lastChangeAt = Date.now()
3542
- let prevFingerprint = initialFingerprint
3543
-
3544
- const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))
3545
-
3546
- // Optional initial observation delay requested by caller
3547
- if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
3548
- try { console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`) } catch { }
3549
- await sleep(observationDelayMs)
3550
- }
3551
-
3552
- // Telemetry
3553
- let pollCount = 0
3554
- let matchedAt: number | null = null
3555
- let lastObservedState: boolean | null = null
3556
- let stableDuration = 0
3557
- let matchSource: string | null = null
3558
-
3559
- while (Date.now() <= deadline) {
3560
- pollCount++
3561
- const now = Date.now()
3562
- // Evaluate condition per type
3563
- if (type === 'ui') {
3564
- try {
3565
- // Prefer using the public findElementHandler which tests can override. This avoids relying
3566
- // on resolveObserve/getUITree for unit tests which may not have devices available.
3567
- try {
3568
- const findRes = await (ToolsInteract as any).findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId })
3569
- const isPresent = !!(findRes && (findRes as any).found)
3570
- const conditionTrue = (match === 'present') ? isPresent : !isPresent
3571
- if (conditionTrue) {
3572
- if (matchedAt === null) matchedAt = Date.now()
3573
- stableDuration = Date.now() - (matchedAt as number)
3574
- lastObservedState = true
3575
- if (stableDuration >= stability_ms) {
3576
- matchSource = 'ui-find'
3577
- const element = isPresent ? (findRes as any).element : null
3578
- const now2 = Date.now()
3579
- return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null }
3580
- }
3581
- } else {
3582
- matchedAt = null
3583
- stableDuration = 0
3584
- lastObservedState = false
3585
- }
3586
- } catch (err) { console.error('waitForUI(ui) find error:', err) }
3587
- } catch (err) { console.error('waitForUI(ui) outer error:', err) }
3588
- } else if (type === 'log') {
3589
- try {
3590
- // Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
3591
- const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 }) as any
3592
- const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : []
3593
- for (const ent of entries) {
3594
- const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : ''
3595
- if (q && String(msg).includes(q)) {
3596
- const now2 = Date.now()
3597
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true }
3598
- }
3599
- }
3600
-
3601
- const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as any
3602
- const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : []
3603
- // Normalize to messages for comparison
3604
- const msgs = ToolsInteract._logsToMessages(logsArr)
3605
- let startIndex = 0
3606
- if (baselineLastLine) {
3607
- const idx = msgs.lastIndexOf(baselineLastLine)
3608
- startIndex = idx >= 0 ? idx + 1 : 0
3609
- }
3610
- for (let i = startIndex; i < msgs.length; i++) {
3611
- const line = msgs[i]
3612
- if (q && String(line).includes(q)) {
3613
- const now2 = Date.now()
3614
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true }
3615
- }
3616
- }
3617
- } catch (err) { console.error('waitForUI(log) error:', err) }
3618
- } else if (type === 'screen') {
3619
- try {
3620
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
3621
- const fp = fpRes?.fingerprint ?? null
3622
- if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
3623
- // when screen changed, require stability_ms where fingerprint remains the same
3624
- if (matchedAt === null) matchedAt = now
3625
- const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null)?.fingerprint ?? null
3626
- if (confirmFp === fp) {
3627
- stableDuration = Date.now() - (matchedAt as number)
3628
- lastObservedState = true
3629
- if (stableDuration >= stability_ms) {
3630
- const now2 = Date.now()
3631
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null }
3632
- }
3633
- } else {
3634
- matchedAt = null
3635
- stableDuration = 0
3636
- lastObservedState = false
3637
- }
3638
- }
3639
- } catch (err) { console.error('waitForUI(screen) error:', err) }
3640
- } else if (type === 'idle') {
3641
- try {
3642
- const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
3643
- const fp = fpRes?.fingerprint ?? null
3644
- if (fp !== prevFingerprint) {
3645
- prevFingerprint = fp
3646
- lastChangeAt = Date.now()
3647
- matchedAt = null
3648
- stableDuration = 0
3649
- lastObservedState = false
3650
- } else {
3651
- const idleMs = Date.now() - lastChangeAt
3652
- lastObservedState = true
3653
- if (idleMs >= stability_ms) {
3654
- const now2 = Date.now()
3655
- return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null }
3656
- }
3657
- }
3658
- } catch (err) { console.error('waitForUI(idle) error:', err) }
3659
- }
3660
-
3661
- // Respect poll interval and avoid tight loop
3662
- await sleep(pollInterval)
3663
- }
3664
-
3665
- // On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
3666
- let snapshot: any = null
3667
- if (includeSnapshotOnFailure) {
3668
- try {
3669
- // Use dynamic import to avoid circular-initialization issues where the ToolsObserve
3670
- // binding captured earlier may not reflect test-time overrides. Importing at call
3671
- // time ensures the latest exported ToolsObserve object is used.
3672
- const Obs = await import('../observe/index.js')
3673
- snapshot = await (Obs as any).ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId })
3674
- } catch (err) {
3675
- snapshot = { error: err instanceof Error ? err.message : String(err) }
3676
- }
3677
- }
3678
-
3679
- const elapsed = Date.now() - start
3680
- return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null }
262
+ static async waitForUICore(input: LegacyWaitInput) {
263
+ return waitForUICore(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), logs: target => ToolsObserve.getLogsHandler(target),
264
+ stream: target => ToolsObserve.readLogStreamHandler(target), find: input => ToolsInteract.findElementHandler(input),
265
+ snapshot: async input => { const { ToolsObserve } = await import('../observe/index.js'); return ToolsObserve.captureDebugSnapshotHandler(input) } })
3681
266
  }
3682
267
  }