mobile-debug-mcp 0.33.0 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (448) hide show
  1. package/.agents/skills/mcp-builder/SKILL.md +44 -64
  2. package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
  3. package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
  4. package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
  5. package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
  6. package/.agents/skills/test-authoring/SKILL.md +7 -6
  7. package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
  8. package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
  9. package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
  10. package/.github/copilot-instructions.md +14 -36
  11. package/.github/workflows/ci.yml +23 -2
  12. package/AGENTS.md +12 -3
  13. package/CONTRIBUTING.md +79 -0
  14. package/README.md +158 -124
  15. package/agent-plugin/.claude-plugin/plugin.json +2 -2
  16. package/agent-plugin/.codex-plugin/plugin.json +1 -1
  17. package/agent-plugin/.cursor-plugin/plugin.json +2 -2
  18. package/agent-plugin/.mcp.json +1 -1
  19. package/agent-plugin/README.md +20 -6
  20. package/agent-plugin/mcp.json +1 -1
  21. package/agent-plugin/plugin.json +1 -1
  22. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +113 -4
  23. package/dist/contracts/execution-controls.js +24 -0
  24. package/dist/contracts/selector.js +8 -0
  25. package/dist/contracts/target.js +15 -0
  26. package/dist/contracts/ui.js +1 -0
  27. package/dist/debugger/android.js +79 -0
  28. package/dist/debugger/diagnostics.js +54 -0
  29. package/dist/debugger/guard.js +30 -0
  30. package/dist/debugger/protocol.js +33 -0
  31. package/dist/debugger/sessions.js +304 -0
  32. package/dist/debugger/worker.js +176 -0
  33. package/dist/interact/adjustment/adjustment-context.js +41 -0
  34. package/dist/interact/adjustment/adjustment-gestures.js +55 -0
  35. package/dist/interact/adjustment/adjustment-plan.js +18 -0
  36. package/dist/interact/adjustment/adjustment-resolution.js +127 -0
  37. package/dist/interact/adjustment/adjustment-result.js +19 -0
  38. package/dist/interact/adjustment/adjustment-semantic.js +64 -0
  39. package/dist/interact/adjustment/adjustment-verification.js +35 -0
  40. package/dist/interact/adjustment/adjustment.js +52 -0
  41. package/dist/interact/adjustment/control-geometry.js +87 -0
  42. package/dist/interact/adjustment/control-values.js +136 -0
  43. package/dist/interact/adjustment/slider-labels.js +84 -0
  44. package/dist/interact/assertions/screen-assertion.js +37 -0
  45. package/dist/interact/assertions/state-assertion-context.js +35 -0
  46. package/dist/interact/assertions/state-assertion.js +76 -0
  47. package/dist/interact/assertions/state-comparison.js +141 -0
  48. package/dist/interact/assertions/visibility-assertion.js +34 -0
  49. package/dist/interact/conditional-action.js +12 -7
  50. package/dist/interact/index.js +65 -3074
  51. package/dist/interact/input-service.js +1 -0
  52. package/dist/interact/interaction-trace.js +21 -0
  53. package/dist/interact/model.js +1 -0
  54. package/dist/interact/observation.js +1 -0
  55. package/dist/interact/targeting/candidate-ancestor.js +58 -0
  56. package/dist/interact/targeting/candidate-result.js +23 -0
  57. package/dist/interact/targeting/candidate-scoring.js +50 -0
  58. package/dist/interact/targeting/element-helpers.js +216 -0
  59. package/dist/interact/targeting/find-element.js +65 -0
  60. package/dist/interact/targeting/nearby-control.js +56 -0
  61. package/dist/interact/targeting/resolved-elements.js +21 -0
  62. package/dist/interact/targeting/tap-target.js +17 -0
  63. package/dist/interact/targeting/target-scope.js +104 -0
  64. package/dist/interact/waits/legacy-log-wait.js +38 -0
  65. package/dist/interact/waits/legacy-wait-observations.js +103 -0
  66. package/dist/interact/waits/legacy-wait-state.js +34 -0
  67. package/dist/interact/waits/legacy-wait.js +48 -0
  68. package/dist/interact/waits/screen-wait.js +49 -0
  69. package/dist/interact/waits/ui-change-helpers.js +157 -0
  70. package/dist/interact/waits/ui-change-state.js +18 -0
  71. package/dist/interact/waits/ui-change-wait.js +77 -0
  72. package/dist/interact/waits/ui-wait-condition.js +56 -0
  73. package/dist/interact/waits/ui-wait-state.js +38 -0
  74. package/dist/interact/waits/ui-wait.js +106 -0
  75. package/dist/manage/android.js +55 -51
  76. package/dist/manage/index.js +38 -75
  77. package/dist/manage/ios/build-types.js +1 -0
  78. package/dist/manage/ios/build.js +210 -0
  79. package/dist/manage/ios/project-discovery.js +29 -0
  80. package/dist/manage/ios/scheme.js +18 -0
  81. package/dist/manage/ios.js +42 -256
  82. package/dist/observe/android/context.js +1 -0
  83. package/dist/observe/android/current-screen.js +55 -0
  84. package/dist/observe/android/fingerprint.js +13 -0
  85. package/dist/observe/android/log-stream.js +132 -0
  86. package/dist/observe/android/logs.js +107 -0
  87. package/dist/observe/android/screenshot.js +70 -0
  88. package/dist/observe/android/ui-tree.js +107 -0
  89. package/dist/observe/android.js +16 -520
  90. package/dist/observe/evidence.js +122 -76
  91. package/dist/observe/index.js +16 -263
  92. package/dist/observe/ios/context.js +16 -0
  93. package/dist/observe/ios/fingerprint.js +12 -0
  94. package/dist/observe/ios/log-stream.js +105 -0
  95. package/dist/observe/ios/logs.js +168 -0
  96. package/dist/observe/ios/screenshot.js +22 -0
  97. package/dist/observe/ios/ui-elements.js +241 -0
  98. package/dist/observe/ios/ui-tree.js +137 -0
  99. package/dist/observe/ios.js +17 -715
  100. package/dist/observe/logs/model.js +1 -0
  101. package/dist/{network → observe/network}/index.js +2 -2
  102. package/dist/observe/readiness-service.js +13 -2
  103. package/dist/observe/readiness.js +44 -18
  104. package/dist/observe/screenshot-encoding.js +41 -0
  105. package/dist/observe/snapshot-collection.js +79 -0
  106. package/dist/observe/snapshot-logs.js +30 -0
  107. package/dist/observe/snapshot-metadata.js +43 -31
  108. package/dist/observe/snapshot-semantic.js +115 -0
  109. package/dist/observe/system-trace-android.js +135 -0
  110. package/dist/observe/system-trace.js +294 -0
  111. package/dist/runtime/action-results.js +270 -0
  112. package/dist/runtime/clock.js +4 -0
  113. package/dist/server/common.js +1 -256
  114. package/dist/server/definitions/assert.js +162 -0
  115. package/dist/server/definitions/diagnostic.js +118 -0
  116. package/dist/server/definitions/input.js +274 -0
  117. package/dist/server/definitions/manage.js +194 -0
  118. package/dist/server/definitions/observe.js +149 -0
  119. package/dist/server/definitions/target.js +138 -0
  120. package/dist/server/definitions/wait.js +105 -0
  121. package/dist/server/handlers/action-execution.js +163 -0
  122. package/dist/server/handlers/correlation.js +23 -0
  123. package/dist/server/handlers/debugger-handlers.js +48 -0
  124. package/dist/server/handlers/debugger-response.js +9 -0
  125. package/dist/server/handlers/diagnostic.js +69 -0
  126. package/dist/server/handlers/handler-types.js +1 -0
  127. package/dist/server/handlers/input.js +154 -0
  128. package/dist/server/handlers/interact.js +113 -0
  129. package/dist/server/handlers/journey-assertions.js +35 -0
  130. package/dist/server/handlers/journey.js +147 -0
  131. package/dist/server/handlers/manage.js +119 -0
  132. package/dist/server/handlers/observe.js +95 -0
  133. package/dist/server/handlers/payload.js +4 -0
  134. package/dist/server/handlers/tool-handlers.js +160 -0
  135. package/dist/server/schemas/condition-schema.js +2 -0
  136. package/dist/server/schemas/debugger-schema.js +47 -0
  137. package/dist/server/{journey-schema.js → schemas/journey-schema.js} +8 -22
  138. package/dist/server/schemas/system-trace-schema.js +17 -0
  139. package/dist/server/tool-definitions.js +62 -1134
  140. package/dist/server-core.js +4 -4
  141. package/dist/server.js +34 -0
  142. package/dist/system/android-status.js +101 -0
  143. package/dist/system/android.js +37 -27
  144. package/dist/system/index.js +13 -438
  145. package/dist/system/ios-status.js +132 -0
  146. package/dist/system/ios.js +11 -8
  147. package/dist/system/model.js +1 -0
  148. package/dist/system/probes.js +105 -0
  149. package/dist/system/status-summary.js +43 -0
  150. package/dist/system/unsupported-status.js +52 -0
  151. package/dist/utils/android/commands.js +89 -0
  152. package/dist/utils/android/gradle.js +139 -0
  153. package/dist/utils/android/logs.js +103 -0
  154. package/dist/utils/android/ui-tree.js +233 -0
  155. package/dist/utils/android/utils.js +6 -564
  156. package/dist/utils/cli/idb/idb-helper.js +4 -6
  157. package/dist/utils/cli/idb/install-idb.js +13 -9
  158. package/dist/utils/cli/ios/preflight-ios.js +42 -38
  159. package/dist/utils/exec.js +2 -0
  160. package/dist/utils/ios/utils.js +45 -46
  161. package/dist/utils/java.js +75 -57
  162. package/dist/utils/operation-budget.js +2 -0
  163. package/dist/utils/resolve-device.js +6 -2
  164. package/dist/utils/ui/index.js +26 -26
  165. package/docs/CHANGELOG.md +48 -0
  166. package/docs/README.md +38 -0
  167. package/docs/benchmarks/README.md +18 -0
  168. package/docs/benchmarks/interaction-latency/README.md +14 -0
  169. package/docs/benchmarks/interaction-latency/final.md +4 -1
  170. package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
  171. package/docs/benchmarks/ui-tree-performance/README.md +24 -16
  172. package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
  173. package/docs/code-structure.md +118 -0
  174. package/docs/specs/001-state-verification/spec.md +16 -14
  175. package/docs/specs/002-richer-element-identity/spec.md +12 -10
  176. package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
  177. package/docs/specs/004-action-verification-routing/spec.md +3 -3
  178. package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
  179. package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
  180. package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
  181. package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
  182. package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
  183. package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
  184. package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
  185. package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
  186. package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
  187. package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
  188. package/docs/specs/014-actionability-resolution/spec.md +20 -18
  189. package/docs/specs/015-adjustable-control-support/spec.md +17 -15
  190. package/docs/specs/016-environment-tools/spec.md +20 -20
  191. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
  192. package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
  193. package/docs/specs/020-conditional-actions/intent.md +5 -3
  194. package/docs/specs/020-conditional-actions/plan.md +8 -9
  195. package/docs/specs/020-conditional-actions/spec.md +4 -2
  196. package/docs/specs/021-app-readiness-signals/intent.md +4 -2
  197. package/docs/specs/021-app-readiness-signals/plan.md +7 -6
  198. package/docs/specs/021-app-readiness-signals/spec.md +3 -1
  199. package/docs/specs/022-correlated-evidence/intent.md +4 -4
  200. package/docs/specs/022-correlated-evidence/plan.md +10 -11
  201. package/docs/specs/022-correlated-evidence/spec.md +3 -1
  202. package/docs/specs/023-android-system-traces/device-evidence.json +38 -0
  203. package/docs/specs/023-android-system-traces/emulator-recheck.json +38 -0
  204. package/docs/specs/023-android-system-traces/intent.md +41 -0
  205. package/docs/specs/023-android-system-traces/plan.md +115 -0
  206. package/docs/specs/023-android-system-traces/spec.md +122 -0
  207. package/docs/specs/024-android-heap-dump-capture/intent.md +41 -0
  208. package/docs/specs/025-android-runtime-debugger/intent.md +41 -0
  209. package/docs/specs/025-android-runtime-debugger/plan.md +106 -0
  210. package/docs/specs/025-android-runtime-debugger/spec.md +183 -0
  211. package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
  212. package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
  213. package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
  214. package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
  215. package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
  216. package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
  217. package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
  218. package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
  219. package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
  220. package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
  221. package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
  222. package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
  223. package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
  224. package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
  225. package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
  226. package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
  227. package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
  228. package/docs/specs/030-android-coroutine-validation/report.md +240 -0
  229. package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
  230. package/docs/tools/TOOLS.md +31 -16
  231. package/docs/tools/{evidence.md → correlated-evidence.md} +14 -3
  232. package/docs/tools/debugger.md +180 -0
  233. package/docs/tools/interact.md +42 -20
  234. package/docs/tools/manage.md +25 -44
  235. package/docs/tools/observe.md +64 -25
  236. package/docs/tools/system-traces.md +134 -0
  237. package/docs/tools/system.md +12 -5
  238. package/package.json +11 -8
  239. package/scripts/architecture-graph.mjs +69 -0
  240. package/scripts/check-architecture.mjs +47 -0
  241. package/scripts/code-quality-baseline.json +1 -0
  242. package/scripts/code-quality.mjs +161 -0
  243. package/src/contracts/execution-controls.ts +26 -0
  244. package/src/contracts/selector.ts +17 -0
  245. package/src/contracts/target.ts +19 -0
  246. package/src/contracts/ui.ts +94 -0
  247. package/src/debugger/android.ts +66 -0
  248. package/src/debugger/diagnostics.ts +43 -0
  249. package/src/debugger/guard.ts +31 -0
  250. package/src/debugger/java/DebuggerWorker.java +166 -0
  251. package/src/debugger/protocol.ts +37 -0
  252. package/src/debugger/sessions.ts +327 -0
  253. package/src/debugger/worker.ts +135 -0
  254. package/src/interact/adjustment/adjustment-context.ts +83 -0
  255. package/src/interact/adjustment/adjustment-gestures.ts +48 -0
  256. package/src/interact/adjustment/adjustment-plan.ts +23 -0
  257. package/src/interact/adjustment/adjustment-resolution.ts +104 -0
  258. package/src/interact/adjustment/adjustment-result.ts +34 -0
  259. package/src/interact/adjustment/adjustment-semantic.ts +53 -0
  260. package/src/interact/adjustment/adjustment-verification.ts +60 -0
  261. package/src/interact/adjustment/adjustment.ts +49 -0
  262. package/src/interact/adjustment/control-geometry.ts +120 -0
  263. package/src/interact/adjustment/control-values.ts +141 -0
  264. package/src/interact/adjustment/slider-labels.ts +84 -0
  265. package/src/interact/assertions/screen-assertion.ts +39 -0
  266. package/src/interact/assertions/state-assertion-context.ts +48 -0
  267. package/src/interact/assertions/state-assertion.ts +64 -0
  268. package/src/interact/assertions/state-comparison.ts +168 -0
  269. package/src/interact/assertions/visibility-assertion.ts +41 -0
  270. package/src/interact/conditional-action.ts +18 -12
  271. package/src/interact/index.ts +76 -3491
  272. package/src/interact/input-service.ts +11 -0
  273. package/src/interact/interaction-trace.ts +36 -0
  274. package/src/interact/model.ts +109 -0
  275. package/src/interact/observation.ts +7 -0
  276. package/src/interact/targeting/candidate-ancestor.ts +49 -0
  277. package/src/interact/targeting/candidate-result.ts +24 -0
  278. package/src/interact/targeting/candidate-scoring.ts +42 -0
  279. package/src/interact/targeting/element-helpers.ts +243 -0
  280. package/src/interact/targeting/find-element.ts +47 -0
  281. package/src/interact/targeting/nearby-control.ts +46 -0
  282. package/src/interact/targeting/resolved-elements.ts +20 -0
  283. package/src/interact/targeting/tap-target.ts +22 -0
  284. package/src/interact/targeting/target-scope.ts +121 -0
  285. package/src/interact/waits/legacy-log-wait.ts +30 -0
  286. package/src/interact/waits/legacy-wait-observations.ts +55 -0
  287. package/src/interact/waits/legacy-wait-state.ts +48 -0
  288. package/src/interact/waits/legacy-wait.ts +34 -0
  289. package/src/interact/waits/screen-wait.ts +37 -0
  290. package/src/interact/waits/ui-change-helpers.ts +184 -0
  291. package/src/interact/waits/ui-change-state.ts +26 -0
  292. package/src/interact/waits/ui-change-wait.ts +59 -0
  293. package/src/interact/waits/ui-wait-condition.ts +52 -0
  294. package/src/interact/waits/ui-wait-state.ts +55 -0
  295. package/src/interact/waits/ui-wait.ts +84 -0
  296. package/src/manage/android.ts +56 -50
  297. package/src/manage/index.ts +63 -93
  298. package/src/manage/ios/build-types.ts +14 -0
  299. package/src/manage/ios/build.ts +230 -0
  300. package/src/manage/ios/project-discovery.ts +26 -0
  301. package/src/manage/ios/scheme.ts +16 -0
  302. package/src/manage/ios.ts +65 -298
  303. package/src/observe/android/context.ts +1 -0
  304. package/src/observe/android/current-screen.ts +63 -0
  305. package/src/observe/android/fingerprint.ts +13 -0
  306. package/src/observe/android/log-stream.ts +127 -0
  307. package/src/observe/android/logs.ts +110 -0
  308. package/src/observe/android/screenshot.ts +79 -0
  309. package/src/observe/android/ui-tree.ts +113 -0
  310. package/src/observe/android.ts +17 -531
  311. package/src/observe/evidence.ts +130 -71
  312. package/src/observe/index.ts +24 -286
  313. package/src/observe/ios/context.ts +22 -0
  314. package/src/observe/ios/fingerprint.ts +12 -0
  315. package/src/observe/ios/log-stream.ts +103 -0
  316. package/src/observe/ios/logs.ts +179 -0
  317. package/src/observe/ios/screenshot.ts +26 -0
  318. package/src/observe/ios/ui-elements.ts +266 -0
  319. package/src/observe/ios/ui-tree.ts +143 -0
  320. package/src/observe/ios.ts +18 -749
  321. package/src/observe/logs/model.ts +11 -0
  322. package/src/{network → observe/network}/index.ts +2 -2
  323. package/src/observe/readiness-service.ts +10 -3
  324. package/src/observe/readiness.ts +49 -22
  325. package/src/observe/screenshot-encoding.ts +32 -0
  326. package/src/observe/snapshot-collection.ts +69 -0
  327. package/src/observe/snapshot-logs.ts +26 -0
  328. package/src/observe/snapshot-metadata.ts +51 -31
  329. package/src/observe/snapshot-semantic.ts +153 -0
  330. package/src/observe/system-trace-android.ts +133 -0
  331. package/src/observe/system-trace.ts +227 -0
  332. package/src/runtime/action-results.ts +376 -0
  333. package/src/runtime/clock.ts +9 -0
  334. package/src/server/common.ts +9 -359
  335. package/src/server/definitions/assert.ts +164 -0
  336. package/src/server/definitions/diagnostic.ts +120 -0
  337. package/src/server/definitions/input.ts +276 -0
  338. package/src/server/definitions/manage.ts +196 -0
  339. package/src/server/definitions/observe.ts +151 -0
  340. package/src/server/definitions/target.ts +140 -0
  341. package/src/server/definitions/wait.ts +107 -0
  342. package/src/server/handlers/action-execution.ts +164 -0
  343. package/src/server/handlers/correlation.ts +20 -0
  344. package/src/server/handlers/debugger-handlers.ts +39 -0
  345. package/src/server/handlers/debugger-response.ts +7 -0
  346. package/src/server/handlers/diagnostic.ts +79 -0
  347. package/src/server/handlers/handler-types.ts +7 -0
  348. package/src/server/handlers/input.ts +173 -0
  349. package/src/server/handlers/interact.ts +135 -0
  350. package/src/server/handlers/journey-assertions.ts +39 -0
  351. package/src/server/handlers/journey.ts +132 -0
  352. package/src/server/handlers/manage.ts +141 -0
  353. package/src/server/handlers/observe.ts +116 -0
  354. package/src/server/handlers/payload.ts +36 -0
  355. package/src/server/handlers/tool-handlers.ts +159 -0
  356. package/src/server/schemas/condition-schema.ts +2 -0
  357. package/src/server/schemas/debugger-schema.ts +43 -0
  358. package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +8 -22
  359. package/src/server/schemas/system-trace-schema.ts +17 -0
  360. package/src/server/tool-definitions.ts +69 -1140
  361. package/src/server-core.ts +4 -4
  362. package/src/server.ts +25 -1
  363. package/src/system/android-status.ts +124 -0
  364. package/src/system/android.ts +33 -21
  365. package/src/system/index.ts +21 -523
  366. package/src/system/ios-status.ts +147 -0
  367. package/src/system/ios.ts +14 -10
  368. package/src/system/model.ts +41 -0
  369. package/src/system/probes.ts +114 -0
  370. package/src/system/status-summary.ts +47 -0
  371. package/src/system/unsupported-status.ts +55 -0
  372. package/src/types.ts +16 -106
  373. package/src/utils/android/commands.ts +81 -0
  374. package/src/utils/android/gradle.ts +132 -0
  375. package/src/utils/android/logs.ts +103 -0
  376. package/src/utils/android/ui-tree.ts +239 -0
  377. package/src/utils/android/utils.ts +16 -556
  378. package/src/utils/cli/idb/check-idb.ts +4 -4
  379. package/src/utils/cli/idb/idb-helper.ts +6 -8
  380. package/src/utils/cli/idb/install-idb.ts +15 -9
  381. package/src/utils/cli/ios/preflight-ios.ts +39 -31
  382. package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
  383. package/src/utils/exec.ts +2 -0
  384. package/src/utils/ios/utils.ts +57 -61
  385. package/src/utils/java.ts +71 -51
  386. package/src/utils/operation-budget.ts +2 -0
  387. package/src/utils/resolve-device.ts +11 -6
  388. package/src/utils/ui/index.ts +28 -28
  389. package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
  390. package/test/device/automated/debugger/coroutine-support.ts +140 -0
  391. package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
  392. package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
  393. package/test/device/automated/debugger/runtime.android.test.ts +219 -0
  394. package/test/device/automated/interact/interaction.android.test.ts +56 -0
  395. package/test/device/automated/observe/system-trace.smoke.ts +104 -0
  396. package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
  397. package/test/fixtures/contracts/tool-definitions.json +3536 -0
  398. package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
  399. package/test/fixtures/coroutine-app/README.md +48 -0
  400. package/test/fixtures/coroutine-app/build.ts +35 -0
  401. package/test/fixtures/debugger-app/MainActivity.java +67 -0
  402. package/test/fixtures/debugger-app/MainActivity.kt +32 -0
  403. package/test/fixtures/debugger-app/NoLocals.java +7 -0
  404. package/test/fixtures/debugger-app/README.md +65 -0
  405. package/test/fixtures/debugger-app/build.ts +39 -0
  406. package/test/fixtures/latency-journey-app/README.md +2 -0
  407. package/test/fixtures/readiness-app/README.md +2 -0
  408. package/test/helpers/perfetto.ts +59 -0
  409. package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
  410. package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
  411. package/test/unit/debugger/diagnostics.test.ts +80 -0
  412. package/test/unit/debugger/guards.test.ts +50 -0
  413. package/test/unit/debugger/native.test.ts +33 -0
  414. package/test/unit/debugger/preparation.test.ts +32 -0
  415. package/test/unit/debugger/protocol.test.ts +15 -0
  416. package/test/unit/debugger/sessions.test.ts +83 -0
  417. package/test/unit/interact/adjustment_service.test.ts +42 -0
  418. package/test/unit/interact/adjustment_verification.test.ts +35 -0
  419. package/test/unit/interact/clock_dependencies.test.ts +63 -0
  420. package/test/unit/interact/element_helpers.test.ts +19 -0
  421. package/test/unit/interact/legacy_wait_service.test.ts +17 -0
  422. package/test/unit/interact/state_comparison.test.ts +20 -0
  423. package/test/unit/interact/target_scope.test.ts +35 -0
  424. package/test/unit/network/get_network_activity.test.ts +1 -1
  425. package/test/unit/observe/readiness.test.ts +3 -1
  426. package/test/unit/observe/screenshot-encoding.test.ts +30 -0
  427. package/test/unit/observe/snapshot-logs.test.ts +14 -0
  428. package/test/unit/observe/system-trace-android.test.ts +52 -0
  429. package/test/unit/observe/system-trace.test.ts +166 -0
  430. package/test/unit/quality/architecture.test.ts +60 -0
  431. package/test/unit/quality/code-quality.test.ts +58 -0
  432. package/test/unit/quality/interaction_boundaries.test.ts +24 -0
  433. package/test/unit/server/conditional_actions.test.ts +1 -1
  434. package/test/unit/server/conditional_schema.test.ts +2 -2
  435. package/test/unit/server/debugger-response.test.ts +63 -0
  436. package/test/unit/server/system-trace.test.ts +64 -0
  437. package/test/unit/server/tool_definitions_parity.test.ts +8 -0
  438. package/dist/server/tool-handlers.js +0 -893
  439. package/docs/ROADMAP.md +0 -646
  440. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
  441. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
  442. package/docs/specs/baseline-spec-v0.md +0 -312
  443. package/docs/specs/mcp-tooling-spec-v1.md +0 -446
  444. package/src/server/tool-handlers.ts +0 -923
  445. /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
  446. /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
  447. /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
  448. /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
@@ -0,0 +1,105 @@
1
+ import { captureEvidence } from '../evidence.js';
2
+ import { spawn } from "child_process";
3
+ import { getXcrunCmd } from "../../utils/ios/utils.js";
4
+ import { createWriteStream, promises as fsPromises } from 'fs';
5
+ import path from 'path';
6
+ import { parseLogLine } from '../../utils/android/utils.js';
7
+ import { iosActiveLogStreams } from './context.js';
8
+ export async function startLogStream(bundleId, deviceId = 'booted', sessionId = 'default') {
9
+ try {
10
+ const simple = bundleId.split('.').pop() || bundleId;
11
+ const predicate = `process == "${simple}" or process == "${bundleId}" or subsystem contains "${bundleId}"`;
12
+ if (iosActiveLogStreams.has(sessionId)) {
13
+ try {
14
+ iosActiveLogStreams.get(sessionId).proc.kill();
15
+ }
16
+ catch { }
17
+ iosActiveLogStreams.delete(sessionId);
18
+ }
19
+ const args = ['simctl', 'spawn', deviceId, 'log', 'stream', '--style', 'syslog', '--predicate', predicate];
20
+ const proc = spawn(getXcrunCmd(), args);
21
+ const tmpDir = process.env.TMPDIR || '/tmp';
22
+ const file = path.join(tmpDir, `mobile-debug-ios-log-${sessionId}.ndjson`);
23
+ const stream = createWriteStream(file, { flags: 'a' });
24
+ proc.stdout.on('data', (chunk) => {
25
+ const text = chunk.toString();
26
+ const lines = text.split(/\r?\n/).filter(Boolean);
27
+ for (const l of lines) {
28
+ const entry = parseLogLine(l);
29
+ stream.write(JSON.stringify(entry) + '\n');
30
+ }
31
+ });
32
+ proc.stderr.on('data', (chunk) => {
33
+ const text = chunk.toString();
34
+ const lines = text.split(/\r?\n/).filter(Boolean);
35
+ for (const l of lines) {
36
+ const entry = { timestamp: '', level: 'E', tag: 'xcrun', message: l };
37
+ stream.write(JSON.stringify(entry) + '\n');
38
+ }
39
+ });
40
+ proc.on('close', () => {
41
+ stream.end();
42
+ if (iosActiveLogStreams.get(sessionId)?.proc === proc)
43
+ iosActiveLogStreams.delete(sessionId);
44
+ });
45
+ iosActiveLogStreams.set(sessionId, { proc, file, device: { platform: 'ios', id: deviceId ?? '' } });
46
+ return { success: true, stream_started: true };
47
+ }
48
+ catch {
49
+ return { success: false, error: 'log_stream_start_failed' };
50
+ }
51
+ }
52
+ export async function stopLogStream(sessionId = 'default') {
53
+ const entry = iosActiveLogStreams.get(sessionId);
54
+ if (!entry)
55
+ return { success: true };
56
+ try {
57
+ entry.proc.kill();
58
+ }
59
+ catch { }
60
+ iosActiveLogStreams.delete(sessionId);
61
+ return { success: true };
62
+ }
63
+ export async function readLogStream(sessionId = 'default', limit = 100, since) {
64
+ return captureEvidence('logs', iosActiveLogStreams.get(sessionId)?.device ?? null, () => collectReadLogStream(sessionId, limit, since));
65
+ }
66
+ export async function collectReadLogStream(sessionId = 'default', limit = 100, since) {
67
+ const entry = iosActiveLogStreams.get(sessionId);
68
+ if (!entry)
69
+ return { entries: [] };
70
+ try {
71
+ const data = await fsPromises.readFile(entry.file, 'utf8').catch(() => '');
72
+ if (!data)
73
+ return { entries: [], crash_summary: { crash_detected: false } };
74
+ const lines = data.split(/\r?\n/).filter(Boolean);
75
+ const parsed = lines.map(l => {
76
+ try {
77
+ return JSON.parse(l);
78
+ }
79
+ catch {
80
+ return { message: l, _iso: null, crash: false };
81
+ }
82
+ });
83
+ let filtered = parsed;
84
+ if (since) {
85
+ let sinceMs = null;
86
+ if (/^\d+$/.test(since))
87
+ sinceMs = Number(since);
88
+ else {
89
+ const sDate = new Date(since);
90
+ if (!isNaN(sDate.getTime()))
91
+ sinceMs = sDate.getTime();
92
+ }
93
+ if (sinceMs !== null) {
94
+ filtered = parsed.filter(p => p._iso && (new Date(p._iso).getTime() >= sinceMs));
95
+ }
96
+ }
97
+ const entries = filtered.slice(-Math.max(0, limit));
98
+ const crashEntry = entries.find(e => e.crash);
99
+ const crash_summary = crashEntry ? { crash_detected: true, exception: crashEntry.exception, sample: crashEntry.message } : { crash_detected: false };
100
+ return { entries, crash_summary };
101
+ }
102
+ catch {
103
+ return { entries: [], crash_summary: { crash_detected: false } };
104
+ }
105
+ }
@@ -0,0 +1,168 @@
1
+ import { getIOSDeviceMetadata, validateBundleId } from "../../utils/ios/utils.js";
2
+ import { iosExecCommand } from './context.js';
3
+ export async function collectGetLogs(filters = {}) {
4
+ const { appId, deviceId = 'booted', pid, tag, level, contains, since_seconds, limit } = filters;
5
+ const effectiveLimit = typeof limit === 'number' && limit > 0 ? limit : 50;
6
+ const { args, processNameUsed } = logArguments(filters, deviceId);
7
+ try {
8
+ const detectedPid = await detectLogPid(appId, pid, deviceId);
9
+ const effectivePid = pid || detectedPid || null;
10
+ const result = await iosExecCommand(args, deviceId);
11
+ const device = await getIOSDeviceMetadata(deviceId);
12
+ const rawLines = result.output ? result.output.split(/\r?\n/).filter(Boolean) : [];
13
+ // Parse lines into structured entries. iOS log format: timestamp [PID:tid] <level> subsystem:category: message
14
+ const parsed = rawLines.map(line => parseIOSLog(line));
15
+ const filtered = filterLogs(parsed, filters, effectivePid, processNameUsed);
16
+ // Order oldest -> newest
17
+ filtered.sort((a, b) => {
18
+ const ta = a.timestamp ? new Date(a.timestamp).getTime() : 0;
19
+ const tb = b.timestamp ? new Date(b.timestamp).getTime() : 0;
20
+ return ta - tb;
21
+ });
22
+ const source = effectivePid ? 'pid' : (appId ? 'process' : 'broad');
23
+ const meta = { appIdProvided: !!appId, processNameUsed: processNameUsed || null, detectedPid: detectedPid || null, filters: { tag, level, contains, since_seconds, limit: effectiveLimit }, pidExplicit: !!pid };
24
+ const limited = filtered.slice(-Math.max(0, effectiveLimit));
25
+ return { device, logs: limited, logCount: limited.length, source, meta };
26
+ }
27
+ catch (err) {
28
+ console.error('iOS getLogs failed:', err);
29
+ const device = await getIOSDeviceMetadata(deviceId);
30
+ return { device, logs: [], logCount: 0, source: 'broad', meta: { error: err instanceof Error ? err.message : String(err) } };
31
+ }
32
+ }
33
+ function parseIOSLog(line) {
34
+ // Example: 2023-08-12 12:34:56.789012+0000 pid <Debug> MyApp[123:456] <info> MySubsystem: MyCategory: Message here
35
+ // Simpler approach: try to extract ISO timestamp at start
36
+ let ts = null;
37
+ const tsMatch = line.match(/^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d+)?)/);
38
+ if (tsMatch) {
39
+ const d = new Date(tsMatch[1]);
40
+ if (!isNaN(d.getTime()))
41
+ ts = d.toISOString();
42
+ }
43
+ // level mapping
44
+ let lvl = 'INFO';
45
+ const lvlMatch = line.match(/\b(Debug|Info|Default|Error|Fault|Warning)\b/i);
46
+ if (lvlMatch) {
47
+ const map = { 'debug': 'DEBUG', 'info': 'INFO', 'default': 'DEBUG', 'error': 'ERROR', 'fault': 'ERROR', 'warning': 'WARN' };
48
+ lvl = map[(lvlMatch[1] || '').toLowerCase()] || 'INFO';
49
+ }
50
+ // subsystem/category -> tag
51
+ let tagVal = '';
52
+ const tagMatch = line.match(/\s([A-Za-z0-9_./-]+):\s/);
53
+ if (tagMatch)
54
+ tagVal = tagMatch[1];
55
+ // pid extraction
56
+ let pidNum = null;
57
+ const pidMatch = line.match(/\[(\d+):\d+\]/);
58
+ if (pidMatch)
59
+ pidNum = Number(pidMatch[1]);
60
+ // message: extract robustly after the subsystem/category token when available
61
+ let message = line;
62
+ if (tagMatch) {
63
+ // tagMatch[0] includes the delimiter (e.g. " MySubsystem: ") — use it to find the message start
64
+ const marker = tagMatch[0];
65
+ const idx = line.indexOf(marker);
66
+ if (idx !== -1) {
67
+ message = line.slice(idx + marker.length).trim();
68
+ }
69
+ else {
70
+ // fallback: try to trim off common prefixes (timestamp, pid, level) and keep the rest
71
+ const afterPidMatch = line.match(/\]\s+/);
72
+ if (afterPidMatch) {
73
+ const afterPidIdx = line.indexOf(afterPidMatch[0]) + afterPidMatch[0].length;
74
+ message = line.slice(afterPidIdx).trim();
75
+ }
76
+ else {
77
+ // remove leading level tokens like <Debug> and keep remainder
78
+ message = line.replace(/^.*?<.*?>\s*/, '').trim();
79
+ }
80
+ }
81
+ }
82
+ else {
83
+ // No tag found — strip obvious prefixes and keep remainder (preserve colons in message)
84
+ message = line.replace(/^.*?<.*?>\s*/, '').trim();
85
+ }
86
+ return { timestamp: ts, level: lvl, tag: tagVal, pid: pidNum, message };
87
+ }
88
+ function logArguments(filters, deviceId) {
89
+ const { appId, tag, since_seconds } = filters;
90
+ const args = ['simctl', 'spawn', deviceId, 'log', 'show', '--style', 'syslog'];
91
+ // Default to last N seconds if no since_seconds provided; limit lines handled after parsing
92
+ if (since_seconds) {
93
+ // log show accepts --last <time>
94
+ args.push('--last', `${since_seconds}s`);
95
+ }
96
+ else {
97
+ // default to last 60s to keep quick
98
+ args.push('--last', '60s');
99
+ }
100
+ let processNameUsed = undefined;
101
+ if (appId) {
102
+ validateBundleId(appId);
103
+ // prefer matching the simple process name (last segment of bundle id), but also match full bundle id in subsystem
104
+ const parts = appId.split('.');
105
+ const simpleName = parts[parts.length - 1];
106
+ processNameUsed = simpleName;
107
+ // predicate: match process by simple name or full bundle id, or subsystem contains bundle id
108
+ args.push('--predicate', `process == "${simpleName}" or process == "${appId}" or subsystem contains "${appId}"`);
109
+ }
110
+ else if (tag) {
111
+ // predicate by subsystem/category
112
+ args.push('--predicate', `subsystem contains "${tag}"`);
113
+ }
114
+ return { args, processNameUsed };
115
+ }
116
+ async function detectLogPid(appId, pid, deviceId) {
117
+ // Attempt pid detection if appId provided and no explicit pid supplied — prefer process name derived from bundle id
118
+ let detectedPid = null;
119
+ if (appId && !pid) {
120
+ const parts = appId.split('.');
121
+ const simpleName = parts[parts.length - 1];
122
+ try {
123
+ const pgrepRes = await iosExecCommand(['simctl', 'spawn', deviceId, 'pgrep', '-f', simpleName], deviceId);
124
+ const out = pgrepRes && pgrepRes.output ? pgrepRes.output.trim() : '';
125
+ const firstLine = out.split(/\r?\n/).find(Boolean);
126
+ if (firstLine) {
127
+ const n = Number(firstLine.trim());
128
+ if (!isNaN(n) && n > 0)
129
+ detectedPid = n;
130
+ }
131
+ }
132
+ catch {
133
+ // ignore pgrep failures — we'll fall back to process/bundle matching
134
+ }
135
+ }
136
+ return detectedPid;
137
+ }
138
+ function filterLogs(parsed, filters, effectivePid, processNameUsed) {
139
+ const { contains, since_seconds, level, tag, appId } = filters;
140
+ // Apply contains filter
141
+ let filtered = parsed;
142
+ if (contains)
143
+ filtered = filtered.filter(e => e.message && e.message.includes(contains));
144
+ // Apply since_seconds already applied by log show, but double-check timestamps
145
+ if (since_seconds) {
146
+ const sinceMs = Date.now() - (since_seconds * 1000);
147
+ filtered = filtered.filter(e => e.timestamp && (new Date(e.timestamp).getTime() >= sinceMs));
148
+ }
149
+ // level filter
150
+ if (level) {
151
+ const L = level.toUpperCase();
152
+ filtered = filtered.filter(e => e.level && e.level.toUpperCase() === L);
153
+ }
154
+ // tag filter
155
+ if (tag)
156
+ filtered = filtered.filter(e => e.tag && e.tag.includes(tag));
157
+ // pid filter (use detected/effective pid if available)
158
+ const pidToFilter = effectivePid;
159
+ if (pidToFilter)
160
+ filtered = filtered.filter(e => e.pid === pidToFilter);
161
+ // If appId present but no predicate returned lines, try substring match
162
+ if (appId && filtered.length === 0) {
163
+ const matched = parsed.filter(e => (e.message && e.message.includes(appId)) || (e.tag && e.tag.includes(appId)) || (e.message && processNameUsed && e.message.includes(processNameUsed)));
164
+ if (matched.length > 0)
165
+ filtered = matched;
166
+ }
167
+ return filtered;
168
+ }
@@ -0,0 +1,22 @@
1
+ import { encodeScreenshot } from '../screenshot-encoding.js';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { promises as fs } from "fs";
4
+ import { getIOSDeviceMetadata } from "../../utils/ios/utils.js";
5
+ import { parsePngSize } from '../../utils/image.js';
6
+ import { iosExecCommand } from './context.js';
7
+ export async function collectCaptureScreenshot(deviceId = "booted") {
8
+ const device = await getIOSDeviceMetadata(deviceId);
9
+ const tmpFile = `/tmp/mcp-ios-screenshot-${randomUUID()}.png`;
10
+ try {
11
+ await iosExecCommand(['simctl', 'io', deviceId, 'screenshot', tmpFile], deviceId);
12
+ const buffer = await fs.readFile(tmpFile);
13
+ const dims = parsePngSize(buffer);
14
+ const encoded = await encodeScreenshot(buffer, 'ios');
15
+ await fs.rm(tmpFile).catch(() => { });
16
+ return { device, ...encoded, resolution: { width: dims.width, height: dims.height } };
17
+ }
18
+ catch (e) {
19
+ await fs.rm(tmpFile).catch(() => { });
20
+ throw new Error(`Failed to capture screenshot: ${e instanceof Error ? e.message : String(e)}`);
21
+ }
22
+ }
@@ -0,0 +1,241 @@
1
+ export function parseIDBFrame(frame) {
2
+ if (!frame)
3
+ return [0, 0, 0, 0];
4
+ if (typeof frame === 'string') {
5
+ const nums = frame.match(/-?\d+(?:\.\d+)?/g);
6
+ if (!nums || nums.length < 4)
7
+ return [0, 0, 0, 0];
8
+ const x = Number(nums[0]);
9
+ const y = Number(nums[1]);
10
+ const w = Number(nums[2]);
11
+ const h = Number(nums[3]);
12
+ return [Math.round(x), Math.round(y), Math.round(x + w), Math.round(y + h)];
13
+ }
14
+ const x = Number(frame.x || 0);
15
+ const y = Number(frame.y || 0);
16
+ const w = Number(frame.width || frame.w || 0);
17
+ const h = Number(frame.height || frame.h || 0);
18
+ return [Math.round(x), Math.round(y), Math.round(x + w), Math.round(y + h)];
19
+ }
20
+ export function getCenter(bounds) {
21
+ const [x1, y1, x2, y2] = bounds;
22
+ return [Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)];
23
+ }
24
+ export function parseIOSNumber(value) {
25
+ if (typeof value === 'number' && Number.isFinite(value))
26
+ return value;
27
+ if (typeof value !== 'string')
28
+ return null;
29
+ const parsed = Number(value);
30
+ return Number.isFinite(parsed) ? parsed : null;
31
+ }
32
+ export function normalizeIOSType(value) {
33
+ return typeof value === 'string' ? value.trim().toLowerCase() : '';
34
+ }
35
+ export function inferIOSRole(type, traits) {
36
+ if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait)))
37
+ return 'slider';
38
+ if (/stepper/.test(type))
39
+ return 'stepper';
40
+ if (/picker|pop up button|dropdown/.test(type))
41
+ return 'dropdown';
42
+ if (/segmented control/.test(type))
43
+ return 'segmented_control';
44
+ if (/button/.test(type) || traits.some((trait) => /button/.test(trait)))
45
+ return 'button';
46
+ if (/cell/.test(type))
47
+ return 'cell';
48
+ if (/switch/.test(type))
49
+ return 'switch';
50
+ if (/text field|textfield|search field/.test(type))
51
+ return 'text_field';
52
+ if (/image/.test(type))
53
+ return 'image';
54
+ if (/window|application|group|scroll view|collection view/.test(type))
55
+ return 'container';
56
+ return null;
57
+ }
58
+ export function getIOSStableId(node) {
59
+ const candidates = [node.AXIdentifier, node.accessibilityIdentifier, node.identifier, node.AXUniqueId];
60
+ for (const candidate of candidates) {
61
+ if (typeof candidate === 'string' && candidate.trim().length > 0)
62
+ return candidate;
63
+ }
64
+ return null;
65
+ }
66
+ export function buildIOSSelectorConfidence(source) {
67
+ switch (source) {
68
+ case 'identifier':
69
+ return { score: 1, reason: 'accessibility_identifier' };
70
+ case 'label':
71
+ return { score: 0.9, reason: 'label_match' };
72
+ case 'value':
73
+ return { score: 0.75, reason: 'value_match' };
74
+ case 'type':
75
+ return { score: 0.35, reason: 'type_match' };
76
+ default:
77
+ return null;
78
+ }
79
+ }
80
+ export function buildIOSSelector(type, label, value, stableId) {
81
+ if (stableId)
82
+ return { value: stableId, confidence: buildIOSSelectorConfidence('identifier') };
83
+ if (label)
84
+ return { value: label, confidence: buildIOSSelectorConfidence('label') };
85
+ if (value)
86
+ return { value: value, confidence: buildIOSSelectorConfidence('value') };
87
+ if (type)
88
+ return { value: type, confidence: buildIOSSelectorConfidence('type') };
89
+ return null;
90
+ }
91
+ export function buildIOSSemantic(type, traits, role, value) {
92
+ const semantic = {
93
+ is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
94
+ is_container: /window|application|group|scroll view|collection view/.test(type)
95
+ };
96
+ if (role === 'slider') {
97
+ semantic.semantic_role = 'slider';
98
+ semantic.adjustable = true;
99
+ semantic.supported_actions = ['adjust'];
100
+ semantic.state_shape = 'continuous';
101
+ }
102
+ else if (role === 'stepper') {
103
+ semantic.semantic_role = 'stepper';
104
+ semantic.adjustable = true;
105
+ semantic.supported_actions = ['increment', 'decrement'];
106
+ semantic.state_shape = 'discrete';
107
+ }
108
+ else if (role === 'dropdown') {
109
+ semantic.semantic_role = 'dropdown';
110
+ semantic.supported_actions = ['tap', 'expand'];
111
+ semantic.state_shape = 'semantic';
112
+ }
113
+ else if (role === 'segmented_control') {
114
+ semantic.semantic_role = 'segmented_control';
115
+ semantic.supported_actions = ['tap'];
116
+ semantic.state_shape = 'discrete';
117
+ }
118
+ else if (traits.some((trait) => /adjustable|slider/i.test(trait)) || /adjustable|slider/.test(type)) {
119
+ semantic.semantic_role = 'custom_adjustable';
120
+ semantic.adjustable = true;
121
+ semantic.supported_actions = ['adjust'];
122
+ semantic.state_shape = 'continuous';
123
+ }
124
+ else if (semantic.is_clickable) {
125
+ semantic.supported_actions = ['tap'];
126
+ }
127
+ if (semantic.state_shape === undefined && semantic.adjustable && value !== null) {
128
+ const numericValue = parseIOSNumber(value);
129
+ if (numericValue !== null && numericValue >= 0 && numericValue <= 1) {
130
+ semantic.state_shape = 'continuous';
131
+ }
132
+ }
133
+ return semantic;
134
+ }
135
+ export function isIOSAdjustable(node, type, traits) {
136
+ return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait));
137
+ }
138
+ export function extractIOSState(node, type, label, value, traits) {
139
+ const state = {};
140
+ const normalizedTraits = traits.map((trait) => String(trait).toLowerCase());
141
+ if (normalizedTraits.some((trait) => /selected/.test(trait))) {
142
+ state.selected = label || value || true;
143
+ }
144
+ if (normalizedTraits.some((trait) => /focused/.test(trait))) {
145
+ state.focused = true;
146
+ }
147
+ if (normalizedTraits.some((trait) => /enabled/.test(trait))) {
148
+ state.enabled = true;
149
+ }
150
+ if (normalizedTraits.some((trait) => /disabled/.test(trait))) {
151
+ state.enabled = false;
152
+ }
153
+ if (value && /textfield|search|text/i.test(type)) {
154
+ state.text_value = value;
155
+ }
156
+ if (isIOSAdjustable(node, type, traits)) {
157
+ const rawValue = parseIOSNumber(value);
158
+ if (rawValue !== null) {
159
+ state.raw_value = rawValue;
160
+ state.value = rawValue >= 0 && rawValue <= 1 ? Math.round(rawValue * 100) : rawValue;
161
+ }
162
+ else if (value) {
163
+ state.raw_value = value;
164
+ state.value = value;
165
+ }
166
+ }
167
+ else if (value) {
168
+ const numericValue = parseIOSNumber(value);
169
+ if (numericValue !== null) {
170
+ state.value = numericValue;
171
+ state.raw_value = numericValue;
172
+ }
173
+ else {
174
+ state.value = value;
175
+ }
176
+ }
177
+ return Object.keys(state).length > 0 ? state : null;
178
+ }
179
+ export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
180
+ if (!node)
181
+ return -1;
182
+ let currentIndex = -1;
183
+ const { type, label, value, frame, state, stableId, selector, role, semantic, clickable, isUseful } = describeIDBNode(node);
184
+ if (isUseful) {
185
+ const bounds = parseIDBFrame(frame);
186
+ const element = {
187
+ text: label,
188
+ contentDescription: value,
189
+ type: type,
190
+ resourceId: stableId,
191
+ clickable: clickable,
192
+ enabled: true,
193
+ visible: true,
194
+ bounds: bounds,
195
+ center: getCenter(bounds),
196
+ depth: depth,
197
+ state,
198
+ stable_id: stableId,
199
+ role,
200
+ test_tag: stableId,
201
+ selector,
202
+ semantic
203
+ };
204
+ if (parentIndex !== -1) {
205
+ element.parentId = parentIndex;
206
+ }
207
+ elements.push(element);
208
+ currentIndex = elements.length - 1;
209
+ }
210
+ const nextParentIndex = currentIndex !== -1 ? currentIndex : parentIndex;
211
+ const nextDepth = currentIndex !== -1 ? depth + 1 : depth;
212
+ const childrenIndices = [];
213
+ if (node.children && Array.isArray(node.children)) {
214
+ for (const child of node.children) {
215
+ const childIndex = traverseIDBNode(child, elements, nextParentIndex, nextDepth);
216
+ if (childIndex !== -1) {
217
+ childrenIndices.push(childIndex);
218
+ }
219
+ }
220
+ }
221
+ if (currentIndex !== -1 && childrenIndices.length > 0) {
222
+ elements[currentIndex].children = childrenIndices;
223
+ }
224
+ return currentIndex;
225
+ }
226
+ function describeIDBNode(node) {
227
+ const type = node.AXElementType || node.type || "unknown";
228
+ const label = node.AXLabel || node.label || null;
229
+ const value = node.AXValue || null;
230
+ const frame = node.AXFrame || node.frame;
231
+ const traits = node.AXTraits || [];
232
+ const state = extractIOSState(node, type, label, value, traits);
233
+ const normalizedType = normalizeIOSType(type);
234
+ const stableId = getIOSStableId(node);
235
+ const selector = buildIOSSelector(type, label, value, stableId);
236
+ const role = inferIOSRole(normalizedType, traits);
237
+ const semantic = buildIOSSemantic(normalizedType, traits, role, value);
238
+ const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
239
+ const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
240
+ return { type, label, value, frame, state, stableId, selector, role, semantic, clickable, isUseful };
241
+ }
@@ -0,0 +1,137 @@
1
+ import { remainingBudget } from '../../utils/operation-budget.js';
2
+ import { execCmd } from '../../utils/exec.js';
3
+ import { createObservationMeasurement } from '../performance-metrics.js';
4
+ import { getIOSDeviceMetadata, getIdbCmd, isIDBInstalled } from "../../utils/ios/utils.js";
5
+ import { deriveSnapshotMetadata } from '../snapshot-metadata.js';
6
+ import { delay } from './context.js';
7
+ import { traverseIDBNode } from './ui-elements.js';
8
+ export async function collectGetUITree(deviceId = "booted") {
9
+ const measurement = createObservationMeasurement('ios');
10
+ let device;
11
+ try {
12
+ device = await measurement.timed('metadata_ms', () => getIOSDeviceMetadata(deviceId));
13
+ }
14
+ catch (error) {
15
+ measurement.setOutcome('unavailable');
16
+ measurement.finish();
17
+ throw error;
18
+ }
19
+ const deviceKey = `ios:${device.id}`;
20
+ const idbExists = await measurement.timed('capability_check_ms', () => isIDBInstalled());
21
+ if (!idbExists) {
22
+ const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
23
+ measurement.setOutcome('unavailable');
24
+ measurement.finish();
25
+ return missingIDBResponse(device, snapshotMetadata);
26
+ }
27
+ const targetUdid = (device.id && device.id !== 'booted') ? device.id : undefined;
28
+ const acquisition = await readIOSTree(targetUdid, measurement);
29
+ if (!acquisition.success) {
30
+ const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
31
+ measurement.setOutcome('unavailable');
32
+ measurement.finish();
33
+ return {
34
+ device, screen: '', resolution: { width: 0, height: 0 }, elements: [], ...snapshotMetadata,
35
+ error: 'Failed to retrieve valid UI dump after 3 attempts.'
36
+ };
37
+ }
38
+ const jsonContent = acquisition.tree;
39
+ try {
40
+ const elements = await measurement.timed('conversion_ms', async () => {
41
+ const parsed = [];
42
+ if (Array.isArray(jsonContent)) {
43
+ for (const node of jsonContent)
44
+ traverseIDBNode(node, parsed);
45
+ }
46
+ else {
47
+ traverseIDBNode(jsonContent, parsed);
48
+ }
49
+ return parsed;
50
+ });
51
+ let width = 0;
52
+ let height = 0;
53
+ if (elements.length > 0) {
54
+ const rootBounds = elements[0].bounds;
55
+ width = rootBounds[2] - rootBounds[0];
56
+ height = rootBounds[3] - rootBounds[1];
57
+ }
58
+ const snapshotMetadata = deriveSnapshotMetadata(deviceKey, {
59
+ screen: "",
60
+ resolution: { width, height },
61
+ elements
62
+ }, 'ui_tree');
63
+ measurement.setOutcome('observed');
64
+ const response = measurement.bindObservation({
65
+ device,
66
+ screen: "",
67
+ resolution: { width, height },
68
+ elements,
69
+ ...snapshotMetadata
70
+ });
71
+ measurement.finish();
72
+ return response;
73
+ }
74
+ catch (e) {
75
+ const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
76
+ measurement.setOutcome('unavailable');
77
+ measurement.finish();
78
+ return {
79
+ device,
80
+ screen: "",
81
+ resolution: { width: 0, height: 0 },
82
+ elements: [],
83
+ ...snapshotMetadata,
84
+ error: `Failed to parse idb output: ${e instanceof Error ? e.message : String(e)}`
85
+ };
86
+ }
87
+ }
88
+ async function readIOSTree(targetUdid, measurement) {
89
+ let jsonContent = null;
90
+ let attempts = 0;
91
+ const maxAttempts = 3;
92
+ while (attempts < maxAttempts) {
93
+ remainingBudget();
94
+ attempts++;
95
+ try {
96
+ await measurement.timed('observation_wait_ms', () => delay(Math.min(remainingBudget(300 + (attempts * 100)), 300 + (attempts * 100))));
97
+ remainingBudget();
98
+ const args = ['ui', 'describe-all', '--json'];
99
+ if (targetUdid) {
100
+ args.push('--udid', targetUdid);
101
+ }
102
+ const output = await measurement.physicalRead(async () => {
103
+ const result = await execCmd(getIdbCmd(), args, { timeout: remainingBudget() });
104
+ if (result.stderr === 'ACTION_TIMEOUT')
105
+ throw new Error('ACTION_TIMEOUT');
106
+ if (result.exitCode !== 0)
107
+ throw new Error(`idb failed (code ${result.exitCode}): ${result.stderr}`);
108
+ if (!result.stdout.trim())
109
+ throw new Error('Empty idb UI tree output');
110
+ return result.stdout;
111
+ });
112
+ if (output && output.trim().length > 0) {
113
+ jsonContent = await measurement.timed('parse_ms', async () => JSON.parse(output));
114
+ break; // Success
115
+ }
116
+ }
117
+ catch (e) {
118
+ if (e instanceof Error && e.message === 'ACTION_TIMEOUT')
119
+ throw e;
120
+ console.error(`Attempt ${attempts} failed: ${e}`);
121
+ }
122
+ if (attempts === maxAttempts) {
123
+ return { success: false, tree: null };
124
+ }
125
+ }
126
+ return { success: true, tree: jsonContent };
127
+ }
128
+ function missingIDBResponse(device, snapshotMetadata) {
129
+ return {
130
+ device,
131
+ screen: "",
132
+ resolution: { width: 0, height: 0 },
133
+ elements: [],
134
+ ...snapshotMetadata,
135
+ error: "iOS UI tree retrieval requires 'idb' (iOS Device Bridge). Please install it via Homebrew: `brew tap facebook/fb && brew install idb-companion` and `pip3 install fb-idb`."
136
+ };
137
+ }