mobile-debug-mcp 0.34.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (500) hide show
  1. package/.agents/skills/mcp-builder/SKILL.md +44 -64
  2. package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
  3. package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
  4. package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
  5. package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
  6. package/.agents/skills/test-authoring/SKILL.md +7 -6
  7. package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
  8. package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
  9. package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
  10. package/.cursor-plugin/marketplace.json +1 -1
  11. package/.github/copilot-instructions.md +14 -36
  12. package/.github/workflows/ci.yml +4 -4
  13. package/AGENTS.md +12 -3
  14. package/CONTRIBUTING.md +79 -0
  15. package/README.md +218 -129
  16. package/agent-plugin/.antigravity-plugin/README.md +48 -0
  17. package/agent-plugin/.antigravity-plugin/plugin.json +18 -0
  18. package/agent-plugin/.claude-plugin/plugin.json +2 -2
  19. package/agent-plugin/.codex-plugin/plugin.json +1 -1
  20. package/agent-plugin/.cursor-plugin/plugin.json +2 -2
  21. package/agent-plugin/.mcp.json +1 -1
  22. package/agent-plugin/README.md +50 -11
  23. package/agent-plugin/cursor/.cursor-plugin/plugin.json +23 -0
  24. package/agent-plugin/cursor/LICENSE +201 -0
  25. package/agent-plugin/cursor/README.md +9 -0
  26. package/agent-plugin/cursor/skills/mobile-debug-mcp/SKILL.md +253 -0
  27. package/agent-plugin/mcp.json +1 -1
  28. package/agent-plugin/mcp_config.json +13 -0
  29. package/agent-plugin/plugin.json +1 -1
  30. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +62 -17
  31. package/dist/contracts/execution-controls.js +24 -0
  32. package/dist/contracts/selector.js +8 -0
  33. package/dist/contracts/target.js +15 -0
  34. package/dist/contracts/ui.js +1 -0
  35. package/dist/debugger/android.js +2 -1
  36. package/dist/debugger/diagnostics.js +54 -0
  37. package/dist/debugger/guard.js +4 -4
  38. package/dist/debugger/ios-worker.js +62 -0
  39. package/dist/debugger/ios.js +57 -0
  40. package/dist/debugger/protocol.js +33 -0
  41. package/dist/debugger/sessions.js +133 -74
  42. package/dist/debugger/worker.js +59 -46
  43. package/dist/interact/adjustment/adjustment-context.js +41 -0
  44. package/dist/interact/adjustment/adjustment-gestures.js +55 -0
  45. package/dist/interact/adjustment/adjustment-plan.js +18 -0
  46. package/dist/interact/adjustment/adjustment-resolution.js +127 -0
  47. package/dist/interact/adjustment/adjustment-result.js +19 -0
  48. package/dist/interact/adjustment/adjustment-semantic.js +64 -0
  49. package/dist/interact/adjustment/adjustment-verification.js +35 -0
  50. package/dist/interact/adjustment/adjustment.js +52 -0
  51. package/dist/interact/adjustment/control-geometry.js +87 -0
  52. package/dist/interact/adjustment/control-values.js +136 -0
  53. package/dist/interact/adjustment/slider-labels.js +84 -0
  54. package/dist/interact/assertions/screen-assertion.js +37 -0
  55. package/dist/interact/assertions/state-assertion-context.js +35 -0
  56. package/dist/interact/assertions/state-assertion.js +76 -0
  57. package/dist/interact/assertions/state-comparison.js +141 -0
  58. package/dist/interact/assertions/visibility-assertion.js +34 -0
  59. package/dist/interact/conditional-action.js +12 -7
  60. package/dist/interact/index.js +77 -3092
  61. package/dist/interact/input-service.js +1 -0
  62. package/dist/interact/interaction-trace.js +21 -0
  63. package/dist/interact/model.js +1 -0
  64. package/dist/interact/observation.js +1 -0
  65. package/dist/interact/targeting/candidate-ancestor.js +58 -0
  66. package/dist/interact/targeting/candidate-result.js +23 -0
  67. package/dist/interact/targeting/candidate-scoring.js +50 -0
  68. package/dist/interact/targeting/element-helpers.js +216 -0
  69. package/dist/interact/targeting/find-element.js +65 -0
  70. package/dist/interact/targeting/nearby-control.js +56 -0
  71. package/dist/interact/targeting/resolved-elements.js +21 -0
  72. package/dist/interact/targeting/tap-target.js +17 -0
  73. package/dist/interact/targeting/target-scope.js +104 -0
  74. package/dist/interact/waits/legacy-log-wait.js +38 -0
  75. package/dist/interact/waits/legacy-wait-observations.js +103 -0
  76. package/dist/interact/waits/legacy-wait-state.js +34 -0
  77. package/dist/interact/waits/legacy-wait.js +48 -0
  78. package/dist/interact/waits/screen-wait.js +49 -0
  79. package/dist/interact/waits/ui-change-helpers.js +157 -0
  80. package/dist/interact/waits/ui-change-state.js +18 -0
  81. package/dist/interact/waits/ui-change-wait.js +77 -0
  82. package/dist/interact/waits/ui-wait-condition.js +56 -0
  83. package/dist/interact/waits/ui-wait-state.js +38 -0
  84. package/dist/interact/waits/ui-wait.js +106 -0
  85. package/dist/manage/android.js +55 -51
  86. package/dist/manage/index.js +38 -75
  87. package/dist/manage/ios/build-types.js +1 -0
  88. package/dist/manage/ios/build.js +210 -0
  89. package/dist/manage/ios/project-discovery.js +29 -0
  90. package/dist/manage/ios/scheme.js +18 -0
  91. package/dist/manage/ios.js +44 -256
  92. package/dist/observe/android/context.js +1 -0
  93. package/dist/observe/android/current-screen.js +55 -0
  94. package/dist/observe/android/fingerprint.js +13 -0
  95. package/dist/observe/android/log-stream.js +132 -0
  96. package/dist/observe/android/logs.js +107 -0
  97. package/dist/observe/android/screenshot.js +70 -0
  98. package/dist/observe/android/ui-tree.js +107 -0
  99. package/dist/observe/android.js +16 -520
  100. package/dist/observe/evidence.js +114 -75
  101. package/dist/observe/index.js +16 -263
  102. package/dist/observe/ios/context.js +16 -0
  103. package/dist/observe/ios/fingerprint.js +12 -0
  104. package/dist/observe/ios/log-stream.js +105 -0
  105. package/dist/observe/ios/logs.js +168 -0
  106. package/dist/observe/ios/screenshot.js +22 -0
  107. package/dist/observe/ios/ui-elements.js +241 -0
  108. package/dist/observe/ios/ui-tree.js +137 -0
  109. package/dist/observe/ios-trace/artifact.js +63 -0
  110. package/dist/observe/ios-trace/capabilities.js +86 -0
  111. package/dist/observe/ios-trace/native.js +93 -0
  112. package/dist/observe/ios-trace/notification.js +35 -0
  113. package/dist/observe/ios-trace/process.js +98 -0
  114. package/dist/observe/ios-trace/sessions.js +244 -0
  115. package/dist/observe/ios.js +17 -715
  116. package/dist/observe/logs/model.js +1 -0
  117. package/dist/{network → observe/network}/index.js +2 -2
  118. package/dist/observe/readiness-service.js +5 -3
  119. package/dist/observe/readiness.js +33 -14
  120. package/dist/observe/screenshot-encoding.js +41 -0
  121. package/dist/observe/snapshot-collection.js +79 -0
  122. package/dist/observe/snapshot-logs.js +30 -0
  123. package/dist/observe/snapshot-metadata.js +43 -31
  124. package/dist/observe/snapshot-semantic.js +115 -0
  125. package/dist/observe/system-trace-error.js +7 -0
  126. package/dist/observe/system-trace-router.js +81 -0
  127. package/dist/observe/system-trace.js +26 -21
  128. package/dist/runtime/action-results.js +270 -0
  129. package/dist/runtime/clock.js +4 -0
  130. package/dist/server/common.js +1 -256
  131. package/dist/server/definitions/assert.js +162 -0
  132. package/dist/server/definitions/diagnostic.js +118 -0
  133. package/dist/server/definitions/input.js +274 -0
  134. package/dist/server/definitions/manage.js +194 -0
  135. package/dist/server/definitions/observe.js +149 -0
  136. package/dist/server/definitions/target.js +138 -0
  137. package/dist/server/definitions/wait.js +105 -0
  138. package/dist/server/handlers/action-execution.js +163 -0
  139. package/dist/server/handlers/correlation.js +23 -0
  140. package/dist/server/handlers/debugger-handlers.js +67 -0
  141. package/dist/server/handlers/debugger-response.js +9 -0
  142. package/dist/server/handlers/diagnostic.js +77 -0
  143. package/dist/server/handlers/handler-types.js +1 -0
  144. package/dist/server/handlers/input.js +154 -0
  145. package/dist/server/handlers/interact.js +113 -0
  146. package/dist/server/handlers/journey-assertions.js +35 -0
  147. package/dist/server/handlers/journey.js +147 -0
  148. package/dist/server/handlers/manage.js +119 -0
  149. package/dist/server/handlers/observe.js +95 -0
  150. package/dist/server/handlers/payload.js +4 -0
  151. package/dist/server/handlers/tool-handlers.js +170 -0
  152. package/dist/server/schemas/condition-schema.js +2 -0
  153. package/dist/server/{debugger-schema.js → schemas/debugger-schema.js} +20 -12
  154. package/dist/server/{journey-schema.js → schemas/journey-schema.js} +7 -22
  155. package/dist/server/schemas/system-trace-schema.js +22 -0
  156. package/dist/server/tool-definitions.js +58 -1135
  157. package/dist/server-core.js +2 -2
  158. package/dist/server.js +4 -4
  159. package/dist/system/android-status.js +101 -0
  160. package/dist/system/android.js +37 -27
  161. package/dist/system/index.js +13 -439
  162. package/dist/system/ios-status.js +132 -0
  163. package/dist/system/ios.js +11 -8
  164. package/dist/system/model.js +1 -0
  165. package/dist/system/probes.js +105 -0
  166. package/dist/system/status-summary.js +43 -0
  167. package/dist/system/unsupported-status.js +52 -0
  168. package/dist/utils/android/commands.js +89 -0
  169. package/dist/utils/android/gradle.js +139 -0
  170. package/dist/utils/android/logs.js +103 -0
  171. package/dist/utils/android/ui-tree.js +233 -0
  172. package/dist/utils/android/utils.js +6 -564
  173. package/dist/utils/cli/idb/idb-helper.js +4 -6
  174. package/dist/utils/cli/idb/install-idb.js +13 -9
  175. package/dist/utils/cli/ios/preflight-ios.js +42 -38
  176. package/dist/utils/ios/utils.js +48 -46
  177. package/dist/utils/java.js +65 -49
  178. package/dist/utils/resolve-device.js +4 -2
  179. package/dist/utils/ui/index.js +26 -26
  180. package/docs/CHANGELOG.md +67 -1
  181. package/docs/README.md +38 -0
  182. package/docs/agent-plugin-installation-verification.md +39 -0
  183. package/docs/benchmarks/README.md +18 -0
  184. package/docs/benchmarks/interaction-latency/README.md +14 -0
  185. package/docs/benchmarks/interaction-latency/final.md +4 -1
  186. package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
  187. package/docs/benchmarks/ui-tree-performance/README.md +24 -16
  188. package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
  189. package/docs/code-structure.md +120 -0
  190. package/docs/specs/001-state-verification/spec.md +16 -14
  191. package/docs/specs/002-richer-element-identity/spec.md +12 -10
  192. package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
  193. package/docs/specs/004-action-verification-routing/spec.md +3 -3
  194. package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
  195. package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
  196. package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
  197. package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
  198. package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
  199. package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
  200. package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
  201. package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
  202. package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
  203. package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
  204. package/docs/specs/014-actionability-resolution/spec.md +20 -18
  205. package/docs/specs/015-adjustable-control-support/spec.md +17 -15
  206. package/docs/specs/016-environment-tools/spec.md +20 -20
  207. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
  208. package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
  209. package/docs/specs/020-conditional-actions/intent.md +5 -3
  210. package/docs/specs/020-conditional-actions/plan.md +8 -9
  211. package/docs/specs/020-conditional-actions/spec.md +4 -2
  212. package/docs/specs/021-app-readiness-signals/intent.md +4 -2
  213. package/docs/specs/021-app-readiness-signals/plan.md +7 -6
  214. package/docs/specs/021-app-readiness-signals/spec.md +3 -1
  215. package/docs/specs/022-correlated-evidence/intent.md +4 -4
  216. package/docs/specs/022-correlated-evidence/plan.md +10 -11
  217. package/docs/specs/022-correlated-evidence/spec.md +3 -1
  218. package/docs/specs/023-android-system-traces/intent.md +3 -2
  219. package/docs/specs/023-android-system-traces/plan.md +5 -7
  220. package/docs/specs/023-android-system-traces/spec.md +3 -1
  221. package/docs/specs/024-android-heap-dump-capture/intent.md +3 -1
  222. package/docs/specs/025-android-runtime-debugger/intent.md +4 -3
  223. package/docs/specs/025-android-runtime-debugger/plan.md +3 -1
  224. package/docs/specs/025-android-runtime-debugger/spec.md +4 -3
  225. package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
  226. package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
  227. package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
  228. package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
  229. package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
  230. package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
  231. package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
  232. package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
  233. package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
  234. package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
  235. package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
  236. package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
  237. package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
  238. package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
  239. package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
  240. package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
  241. package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
  242. package/docs/specs/030-android-coroutine-validation/report.md +240 -0
  243. package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
  244. package/docs/specs/031-ios-system-traces/intent.md +38 -0
  245. package/docs/specs/031-ios-system-traces/investigation.md +92 -0
  246. package/docs/specs/031-ios-system-traces/late-run-failure-evidence.json +32 -0
  247. package/docs/specs/031-ios-system-traces/plan.md +138 -0
  248. package/docs/specs/031-ios-system-traces/pr102-monitor-evidence.json +22 -0
  249. package/docs/specs/031-ios-system-traces/simulator-evidence.json +47 -0
  250. package/docs/specs/031-ios-system-traces/spec.md +135 -0
  251. package/docs/specs/032-ios-runtime-debugger/evidence.json +176 -0
  252. package/docs/specs/032-ios-runtime-debugger/intent.md +38 -0
  253. package/docs/specs/032-ios-runtime-debugger/plan.md +95 -0
  254. package/docs/specs/032-ios-runtime-debugger/report.md +205 -0
  255. package/docs/specs/032-ios-runtime-debugger/spec.md +153 -0
  256. package/docs/tools/TOOLS.md +31 -18
  257. package/docs/tools/{evidence.md → correlated-evidence.md} +9 -1
  258. package/docs/tools/debugger.md +120 -4
  259. package/docs/tools/interact.md +42 -20
  260. package/docs/tools/manage.md +25 -44
  261. package/docs/tools/observe.md +55 -23
  262. package/docs/tools/system-traces.md +89 -14
  263. package/docs/tools/system.md +12 -5
  264. package/package.json +12 -8
  265. package/scripts/agent-plugin.mjs +55 -5
  266. package/scripts/architecture-graph.mjs +69 -0
  267. package/scripts/check-architecture.mjs +47 -0
  268. package/scripts/code-quality-baseline.json +1 -0
  269. package/scripts/code-quality.mjs +161 -0
  270. package/src/contracts/execution-controls.ts +26 -0
  271. package/src/contracts/selector.ts +17 -0
  272. package/src/contracts/target.ts +19 -0
  273. package/src/contracts/ui.ts +94 -0
  274. package/src/debugger/android.ts +3 -2
  275. package/src/debugger/diagnostics.ts +43 -0
  276. package/src/debugger/guard.ts +10 -5
  277. package/src/debugger/ios-worker.ts +49 -0
  278. package/src/debugger/ios.ts +45 -0
  279. package/{worker → src/debugger/java}/DebuggerWorker.java +9 -3
  280. package/src/debugger/lldb/breakpoints.py +51 -0
  281. package/src/debugger/lldb/inspection.py +109 -0
  282. package/src/debugger/lldb/worker.py +203 -0
  283. package/src/debugger/protocol.ts +37 -0
  284. package/src/debugger/sessions.ts +326 -155
  285. package/src/debugger/worker.ts +57 -41
  286. package/src/interact/adjustment/adjustment-context.ts +83 -0
  287. package/src/interact/adjustment/adjustment-gestures.ts +48 -0
  288. package/src/interact/adjustment/adjustment-plan.ts +23 -0
  289. package/src/interact/adjustment/adjustment-resolution.ts +104 -0
  290. package/src/interact/adjustment/adjustment-result.ts +34 -0
  291. package/src/interact/adjustment/adjustment-semantic.ts +53 -0
  292. package/src/interact/adjustment/adjustment-verification.ts +60 -0
  293. package/src/interact/adjustment/adjustment.ts +49 -0
  294. package/src/interact/adjustment/control-geometry.ts +120 -0
  295. package/src/interact/adjustment/control-values.ts +141 -0
  296. package/src/interact/adjustment/slider-labels.ts +84 -0
  297. package/src/interact/assertions/screen-assertion.ts +39 -0
  298. package/src/interact/assertions/state-assertion-context.ts +48 -0
  299. package/src/interact/assertions/state-assertion.ts +64 -0
  300. package/src/interact/assertions/state-comparison.ts +168 -0
  301. package/src/interact/assertions/visibility-assertion.ts +41 -0
  302. package/src/interact/conditional-action.ts +18 -12
  303. package/src/interact/index.ts +89 -3494
  304. package/src/interact/input-service.ts +11 -0
  305. package/src/interact/interaction-trace.ts +36 -0
  306. package/src/interact/model.ts +109 -0
  307. package/src/interact/observation.ts +7 -0
  308. package/src/interact/targeting/candidate-ancestor.ts +49 -0
  309. package/src/interact/targeting/candidate-result.ts +24 -0
  310. package/src/interact/targeting/candidate-scoring.ts +42 -0
  311. package/src/interact/targeting/element-helpers.ts +243 -0
  312. package/src/interact/targeting/find-element.ts +47 -0
  313. package/src/interact/targeting/nearby-control.ts +46 -0
  314. package/src/interact/targeting/resolved-elements.ts +20 -0
  315. package/src/interact/targeting/tap-target.ts +22 -0
  316. package/src/interact/targeting/target-scope.ts +121 -0
  317. package/src/interact/waits/legacy-log-wait.ts +30 -0
  318. package/src/interact/waits/legacy-wait-observations.ts +55 -0
  319. package/src/interact/waits/legacy-wait-state.ts +48 -0
  320. package/src/interact/waits/legacy-wait.ts +34 -0
  321. package/src/interact/waits/screen-wait.ts +37 -0
  322. package/src/interact/waits/ui-change-helpers.ts +184 -0
  323. package/src/interact/waits/ui-change-state.ts +26 -0
  324. package/src/interact/waits/ui-change-wait.ts +59 -0
  325. package/src/interact/waits/ui-wait-condition.ts +52 -0
  326. package/src/interact/waits/ui-wait-state.ts +55 -0
  327. package/src/interact/waits/ui-wait.ts +84 -0
  328. package/src/manage/android.ts +56 -50
  329. package/src/manage/index.ts +63 -93
  330. package/src/manage/ios/build-types.ts +14 -0
  331. package/src/manage/ios/build.ts +230 -0
  332. package/src/manage/ios/project-discovery.ts +26 -0
  333. package/src/manage/ios/scheme.ts +16 -0
  334. package/src/manage/ios.ts +67 -298
  335. package/src/observe/android/context.ts +1 -0
  336. package/src/observe/android/current-screen.ts +63 -0
  337. package/src/observe/android/fingerprint.ts +13 -0
  338. package/src/observe/android/log-stream.ts +127 -0
  339. package/src/observe/android/logs.ts +110 -0
  340. package/src/observe/android/screenshot.ts +79 -0
  341. package/src/observe/android/ui-tree.ts +113 -0
  342. package/src/observe/android.ts +17 -531
  343. package/src/observe/evidence.ts +122 -70
  344. package/src/observe/index.ts +24 -286
  345. package/src/observe/ios/context.ts +22 -0
  346. package/src/observe/ios/fingerprint.ts +12 -0
  347. package/src/observe/ios/log-stream.ts +103 -0
  348. package/src/observe/ios/logs.ts +179 -0
  349. package/src/observe/ios/screenshot.ts +26 -0
  350. package/src/observe/ios/ui-elements.ts +266 -0
  351. package/src/observe/ios/ui-tree.ts +143 -0
  352. package/src/observe/ios-trace/artifact.ts +55 -0
  353. package/src/observe/ios-trace/capabilities.ts +74 -0
  354. package/src/observe/ios-trace/native.ts +72 -0
  355. package/src/observe/ios-trace/notification.ts +30 -0
  356. package/src/observe/ios-trace/process.ts +82 -0
  357. package/src/observe/ios-trace/sessions.ts +204 -0
  358. package/src/observe/ios.ts +18 -749
  359. package/src/observe/logs/model.ts +11 -0
  360. package/src/{network → observe/network}/index.ts +2 -2
  361. package/src/observe/readiness-service.ts +6 -4
  362. package/src/observe/readiness.ts +42 -21
  363. package/src/observe/screenshot-encoding.ts +32 -0
  364. package/src/observe/snapshot-collection.ts +69 -0
  365. package/src/observe/snapshot-logs.ts +26 -0
  366. package/src/observe/snapshot-metadata.ts +51 -31
  367. package/src/observe/snapshot-semantic.ts +153 -0
  368. package/src/observe/system-trace-error.ts +3 -0
  369. package/src/observe/system-trace-router.ts +61 -0
  370. package/src/observe/system-trace.ts +28 -19
  371. package/src/runtime/action-results.ts +376 -0
  372. package/src/runtime/clock.ts +9 -0
  373. package/src/server/common.ts +9 -359
  374. package/src/server/definitions/assert.ts +164 -0
  375. package/src/server/definitions/diagnostic.ts +120 -0
  376. package/src/server/definitions/input.ts +276 -0
  377. package/src/server/definitions/manage.ts +196 -0
  378. package/src/server/definitions/observe.ts +151 -0
  379. package/src/server/definitions/target.ts +140 -0
  380. package/src/server/definitions/wait.ts +107 -0
  381. package/src/server/handlers/action-execution.ts +164 -0
  382. package/src/server/handlers/correlation.ts +20 -0
  383. package/src/server/handlers/debugger-handlers.ts +58 -0
  384. package/src/server/handlers/debugger-response.ts +7 -0
  385. package/src/server/handlers/diagnostic.ts +86 -0
  386. package/src/server/handlers/handler-types.ts +7 -0
  387. package/src/server/handlers/input.ts +173 -0
  388. package/src/server/handlers/interact.ts +135 -0
  389. package/src/server/handlers/journey-assertions.ts +39 -0
  390. package/src/server/handlers/journey.ts +132 -0
  391. package/src/server/handlers/manage.ts +141 -0
  392. package/src/server/handlers/observe.ts +116 -0
  393. package/src/server/handlers/payload.ts +36 -0
  394. package/src/server/handlers/tool-handlers.ts +170 -0
  395. package/src/server/schemas/condition-schema.ts +2 -0
  396. package/src/server/schemas/debugger-schema.ts +46 -0
  397. package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +7 -22
  398. package/src/server/schemas/system-trace-schema.ts +23 -0
  399. package/src/server/tool-definitions.ts +67 -1143
  400. package/src/server-core.ts +2 -2
  401. package/src/server.ts +4 -4
  402. package/src/system/android-status.ts +124 -0
  403. package/src/system/android.ts +33 -21
  404. package/src/system/index.ts +21 -525
  405. package/src/system/ios-status.ts +147 -0
  406. package/src/system/ios.ts +14 -10
  407. package/src/system/model.ts +41 -0
  408. package/src/system/probes.ts +114 -0
  409. package/src/system/status-summary.ts +47 -0
  410. package/src/system/unsupported-status.ts +55 -0
  411. package/src/types.ts +16 -106
  412. package/src/utils/android/commands.ts +81 -0
  413. package/src/utils/android/gradle.ts +132 -0
  414. package/src/utils/android/logs.ts +103 -0
  415. package/src/utils/android/ui-tree.ts +239 -0
  416. package/src/utils/android/utils.ts +16 -556
  417. package/src/utils/cli/idb/check-idb.ts +4 -4
  418. package/src/utils/cli/idb/idb-helper.ts +6 -8
  419. package/src/utils/cli/idb/install-idb.ts +15 -9
  420. package/src/utils/cli/ios/preflight-ios.ts +39 -31
  421. package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
  422. package/src/utils/ios/utils.ts +60 -61
  423. package/src/utils/java.ts +62 -43
  424. package/src/utils/resolve-device.ts +9 -6
  425. package/src/utils/ui/index.ts +28 -28
  426. package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
  427. package/test/device/automated/debugger/coroutine-support.ts +140 -0
  428. package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
  429. package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
  430. package/test/device/automated/debugger/runtime.android.test.ts +19 -3
  431. package/test/device/automated/debugger/runtime.ios.test.ts +221 -0
  432. package/test/device/automated/interact/interaction.android.test.ts +56 -0
  433. package/test/device/automated/observe/ios-system-trace.smoke.ts +119 -0
  434. package/test/device/automated/observe/system-trace.smoke.ts +1 -1
  435. package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
  436. package/test/fixtures/contracts/tool-definitions.json +3696 -0
  437. package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
  438. package/test/fixtures/coroutine-app/README.md +48 -0
  439. package/test/fixtures/coroutine-app/build.ts +35 -0
  440. package/test/fixtures/debugger-app/MainActivity.java +21 -0
  441. package/test/fixtures/debugger-app/MainActivity.kt +32 -0
  442. package/test/fixtures/debugger-app/README.md +52 -0
  443. package/test/fixtures/debugger-app/build.ts +25 -15
  444. package/test/fixtures/ios-debugger-app/Main.swift +95 -0
  445. package/test/fixtures/ios-debugger-app/README.md +27 -0
  446. package/test/fixtures/ios-debugger-app/build.ts +21 -0
  447. package/test/fixtures/latency-journey-app/README.md +2 -0
  448. package/test/fixtures/readiness-app/README.md +2 -0
  449. package/test/helpers/ios-trace-shutdown.ts +29 -0
  450. package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
  451. package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
  452. package/test/unit/debugger/diagnostics.test.ts +80 -0
  453. package/test/unit/debugger/guards.test.ts +4 -0
  454. package/test/unit/debugger/handle-guards.test.ts +88 -0
  455. package/test/unit/debugger/ios-preparation.test.ts +57 -0
  456. package/test/unit/debugger/ios.test.ts +104 -0
  457. package/test/unit/debugger/lldb-worker.py +166 -0
  458. package/test/unit/debugger/lldb-worker.test.ts +10 -0
  459. package/test/unit/debugger/preparation.test.ts +32 -0
  460. package/test/unit/debugger/protocol.test.ts +15 -0
  461. package/test/unit/debugger/sessions.test.ts +1 -1
  462. package/test/unit/interact/adjustment_service.test.ts +42 -0
  463. package/test/unit/interact/adjustment_verification.test.ts +35 -0
  464. package/test/unit/interact/clock_dependencies.test.ts +63 -0
  465. package/test/unit/interact/element_helpers.test.ts +19 -0
  466. package/test/unit/interact/legacy_wait_service.test.ts +17 -0
  467. package/test/unit/interact/state_comparison.test.ts +20 -0
  468. package/test/unit/interact/target_scope.test.ts +35 -0
  469. package/test/unit/network/get_network_activity.test.ts +1 -1
  470. package/test/unit/observe/ios-trace-native.test.ts +90 -0
  471. package/test/unit/observe/ios-trace-sessions.test.ts +131 -0
  472. package/test/unit/observe/readiness.test.ts +1 -1
  473. package/test/unit/observe/screenshot-encoding.test.ts +30 -0
  474. package/test/unit/observe/snapshot-logs.test.ts +14 -0
  475. package/test/unit/observe/system-trace-router.test.ts +49 -0
  476. package/test/unit/quality/agent-plugin-export.test.ts +38 -0
  477. package/test/unit/quality/architecture.test.ts +60 -0
  478. package/test/unit/quality/code-quality.test.ts +58 -0
  479. package/test/unit/quality/interaction_boundaries.test.ts +24 -0
  480. package/test/unit/server/conditional_actions.test.ts +1 -1
  481. package/test/unit/server/conditional_schema.test.ts +2 -2
  482. package/test/unit/server/debugger-response.test.ts +63 -0
  483. package/test/unit/server/system-trace.test.ts +12 -3
  484. package/test/unit/server/tool_definitions_parity.test.ts +8 -0
  485. package/dist/server/debugger-handlers.js +0 -38
  486. package/dist/server/system-trace-schema.js +0 -17
  487. package/dist/server/tool-handlers.js +0 -1000
  488. package/docs/ROADMAP.md +0 -646
  489. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
  490. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
  491. package/docs/specs/baseline-spec-v0.md +0 -312
  492. package/docs/specs/mcp-tooling-spec-v1.md +0 -446
  493. package/src/server/debugger-handlers.ts +0 -29
  494. package/src/server/debugger-schema.ts +0 -37
  495. package/src/server/system-trace-schema.ts +0 -17
  496. package/src/server/tool-handlers.ts +0 -1008
  497. /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
  498. /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
  499. /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
  500. /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
@@ -0,0 +1,103 @@
1
+ import type { StreamLogEntry } from '../logs/model.js'
2
+ import { captureEvidence } from '../evidence.js'
3
+ import { spawn } from "child_process"
4
+ import { getXcrunCmd } from "../../utils/ios/utils.js"
5
+ import { createWriteStream, promises as fsPromises } from 'fs'
6
+ import path from 'path'
7
+ import { parseLogLine } from '../../utils/android/utils.js'
8
+ import { iosActiveLogStreams } from './context.js'
9
+ export async function startLogStream(bundleId: string, deviceId: string = 'booted', sessionId: string = 'default'): Promise<{ success: boolean; stream_started?: boolean; error?: string }> {
10
+ try {
11
+ const simple = bundleId.split('.').pop() || bundleId
12
+ const predicate = `process == "${simple}" or process == "${bundleId}" or subsystem contains "${bundleId}"`
13
+
14
+ if (iosActiveLogStreams.has(sessionId)) {
15
+ try { iosActiveLogStreams.get(sessionId)!.proc.kill() } catch { }
16
+ iosActiveLogStreams.delete(sessionId)
17
+ }
18
+
19
+ const args = ['simctl', 'spawn', deviceId, 'log', 'stream', '--style', 'syslog', '--predicate', predicate]
20
+ const proc = spawn(getXcrunCmd(), args)
21
+
22
+ const tmpDir = process.env.TMPDIR || '/tmp'
23
+ const file = path.join(tmpDir, `mobile-debug-ios-log-${sessionId}.ndjson`)
24
+ const stream = createWriteStream(file, { flags: 'a' })
25
+
26
+ proc.stdout.on('data', (chunk) => {
27
+ const text = chunk.toString()
28
+ const lines = text.split(/\r?\n/).filter(Boolean)
29
+ for (const l of lines) {
30
+ const entry = parseLogLine(l)
31
+ stream.write(JSON.stringify(entry) + '\n')
32
+ }
33
+ })
34
+
35
+ proc.stderr.on('data', (chunk) => {
36
+ const text = chunk.toString()
37
+ const lines = text.split(/\r?\n/).filter(Boolean)
38
+ for (const l of lines) {
39
+ const entry = { timestamp: '', level: 'E', tag: 'xcrun', message: l }
40
+ stream.write(JSON.stringify(entry) + '\n')
41
+ }
42
+ })
43
+
44
+ proc.on('close', () => {
45
+ stream.end()
46
+ if (iosActiveLogStreams.get(sessionId)?.proc === proc) iosActiveLogStreams.delete(sessionId)
47
+ })
48
+
49
+ iosActiveLogStreams.set(sessionId, { proc, file, device: { platform: 'ios', id: deviceId ?? '' } })
50
+ return { success: true, stream_started: true }
51
+ } catch {
52
+ return { success: false, error: 'log_stream_start_failed' }
53
+ }
54
+ }
55
+
56
+ export async function stopLogStream(sessionId: string = 'default'): Promise<{ success: boolean }> {
57
+ const entry = iosActiveLogStreams.get(sessionId)
58
+ if (!entry) return { success: true }
59
+ try { entry.proc.kill() } catch { }
60
+ iosActiveLogStreams.delete(sessionId)
61
+ return { success: true }
62
+ }
63
+
64
+ export async function readLogStream(sessionId: string = 'default', limit: number = 100, since?: string): Promise<{ entries: StreamLogEntry[], crash_summary?: { crash_detected: boolean, exception?: string, sample?: string } }> {
65
+ return captureEvidence('logs', iosActiveLogStreams.get(sessionId)?.device ?? null, () => collectReadLogStream(sessionId, limit, since))
66
+ }
67
+
68
+ export async function collectReadLogStream(sessionId: string = 'default', limit: number = 100, since?: string): Promise<{ entries: StreamLogEntry[], crash_summary?: { crash_detected: boolean, exception?: string, sample?: string } }> {
69
+ const entry = iosActiveLogStreams.get(sessionId)
70
+ if (!entry) return { entries: [] }
71
+ try {
72
+ const data = await fsPromises.readFile(entry.file, 'utf8').catch(() => '')
73
+ if (!data) return { entries: [], crash_summary: { crash_detected: false } }
74
+ const lines = data.split(/\r?\n/).filter(Boolean)
75
+ const parsed = lines.map(l => {
76
+ try {
77
+ return JSON.parse(l) as StreamLogEntry
78
+ } catch {
79
+ return { message: l, _iso: null, crash: false }
80
+ }
81
+ })
82
+
83
+ let filtered = parsed
84
+ if (since) {
85
+ let sinceMs: number | null = null
86
+ if (/^\d+$/.test(since)) sinceMs = Number(since)
87
+ else {
88
+ const sDate = new Date(since)
89
+ if (!isNaN(sDate.getTime())) sinceMs = sDate.getTime()
90
+ }
91
+ if (sinceMs !== null) {
92
+ filtered = parsed.filter(p => p._iso && (new Date(p._iso).getTime() >= sinceMs))
93
+ }
94
+ }
95
+
96
+ const entries = filtered.slice(-Math.max(0, limit))
97
+ const crashEntry = entries.find(e => e.crash)
98
+ const crash_summary = crashEntry ? { crash_detected: true, exception: crashEntry.exception, sample: crashEntry.message } : { crash_detected: false }
99
+ return { entries, crash_summary }
100
+ } catch {
101
+ return { entries: [], crash_summary: { crash_detected: false } }
102
+ }
103
+ }
@@ -0,0 +1,179 @@
1
+ import type { LogFilters } from '../logs/model.js'
2
+ import type { StructuredLogEntry } from '../../types.js'
3
+ import { GetLogsResponse } from "../../types.js"
4
+ import { getIOSDeviceMetadata, validateBundleId } from "../../utils/ios/utils.js"
5
+ import { iosExecCommand } from './context.js'
6
+ export async function collectGetLogs(filters: { appId?: string, deviceId?: string, pid?: number, tag?: string, level?: string, contains?: string, since_seconds?: number, limit?: number } = {}): Promise<GetLogsResponse> {
7
+ const { appId, deviceId = 'booted', pid, tag, level, contains, since_seconds, limit } = filters
8
+ const effectiveLimit = typeof limit === 'number' && limit > 0 ? limit : 50
9
+ const { args, processNameUsed } = logArguments(filters, deviceId)
10
+ try {
11
+ const detectedPid = await detectLogPid(appId, pid, deviceId)
12
+ const effectivePid = pid || detectedPid || null
13
+ const result = await iosExecCommand(args, deviceId)
14
+ const device = await getIOSDeviceMetadata(deviceId)
15
+ const rawLines = result.output ? result.output.split(/\r?\n/).filter(Boolean) : []
16
+
17
+ // Parse lines into structured entries. iOS log format: timestamp [PID:tid] <level> subsystem:category: message
18
+ const parsed = rawLines.map(line => parseIOSLog(line))
19
+
20
+ const filtered = filterLogs(parsed, filters, effectivePid, processNameUsed)
21
+ // Order oldest -> newest
22
+ filtered.sort((a, b) => {
23
+ const ta = a.timestamp ? new Date(a.timestamp).getTime() : 0
24
+ const tb = b.timestamp ? new Date(b.timestamp).getTime() : 0
25
+ return ta - tb
26
+ })
27
+
28
+ const source = effectivePid ? 'pid' : (appId ? 'process' : 'broad')
29
+ const meta = { appIdProvided: !!appId, processNameUsed: processNameUsed || null, detectedPid: detectedPid || null, filters: { tag, level, contains, since_seconds, limit: effectiveLimit }, pidExplicit: !!pid }
30
+ const limited = filtered.slice(-Math.max(0, effectiveLimit))
31
+ return { device, logs: limited, logCount: limited.length, source, meta }
32
+ } catch (err) {
33
+ console.error('iOS getLogs failed:', err)
34
+ const device = await getIOSDeviceMetadata(deviceId)
35
+ return { device, logs: [], logCount: 0, source: 'broad', meta: { error: err instanceof Error ? err.message : String(err) } }
36
+ }
37
+ }
38
+
39
+ function parseIOSLog(line: string): StructuredLogEntry {
40
+ // Example: 2023-08-12 12:34:56.789012+0000 pid <Debug> MyApp[123:456] <info> MySubsystem: MyCategory: Message here
41
+ // Simpler approach: try to extract ISO timestamp at start
42
+ let ts: string | null = null
43
+ const tsMatch = line.match(/^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d+)?)/)
44
+ if (tsMatch) {
45
+ const d = new Date(tsMatch[1])
46
+ if (!isNaN(d.getTime())) ts = d.toISOString()
47
+ }
48
+
49
+ // level mapping
50
+ let lvl = 'INFO'
51
+ const lvlMatch = line.match(/\b(Debug|Info|Default|Error|Fault|Warning)\b/i)
52
+ if (lvlMatch) {
53
+ const map: Record<string, string> = { 'debug': 'DEBUG', 'info': 'INFO', 'default': 'DEBUG', 'error': 'ERROR', 'fault': 'ERROR', 'warning': 'WARN' }
54
+ lvl = map[(lvlMatch[1] || '').toLowerCase()] || 'INFO'
55
+ }
56
+
57
+ // subsystem/category -> tag
58
+ let tagVal = ''
59
+ const tagMatch = line.match(/\s([A-Za-z0-9_./-]+):\s/)
60
+ if (tagMatch) tagVal = tagMatch[1]
61
+
62
+ // pid extraction
63
+ let pidNum: number | null = null
64
+ const pidMatch = line.match(/\[(\d+):\d+\]/)
65
+ if (pidMatch) pidNum = Number(pidMatch[1])
66
+
67
+ // message: extract robustly after the subsystem/category token when available
68
+ let message = line
69
+ if (tagMatch) {
70
+ // tagMatch[0] includes the delimiter (e.g. " MySubsystem: ") — use it to find the message start
71
+ const marker = tagMatch[0]
72
+ const idx = line.indexOf(marker)
73
+ if (idx !== -1) {
74
+ message = line.slice(idx + marker.length).trim()
75
+ } else {
76
+ // fallback: try to trim off common prefixes (timestamp, pid, level) and keep the rest
77
+ const afterPidMatch = line.match(/\]\s+/)
78
+ if (afterPidMatch) {
79
+ const afterPidIdx = line.indexOf(afterPidMatch[0]) + afterPidMatch[0].length
80
+ message = line.slice(afterPidIdx).trim()
81
+ } else {
82
+ // remove leading level tokens like <Debug> and keep remainder
83
+ message = line.replace(/^.*?<.*?>\s*/, '').trim()
84
+ }
85
+ }
86
+ } else {
87
+ // No tag found — strip obvious prefixes and keep remainder (preserve colons in message)
88
+ message = line.replace(/^.*?<.*?>\s*/, '').trim()
89
+ }
90
+
91
+ return { timestamp: ts, level: lvl, tag: tagVal, pid: pidNum, message }
92
+ }
93
+
94
+ function logArguments(filters: LogFilters, deviceId: string) {
95
+ const { appId, tag, since_seconds } = filters
96
+ const args: string[] = ['simctl', 'spawn', deviceId, 'log', 'show', '--style', 'syslog']
97
+
98
+ // Default to last N seconds if no since_seconds provided; limit lines handled after parsing
99
+
100
+ if (since_seconds) {
101
+ // log show accepts --last <time>
102
+ args.push('--last', `${since_seconds}s`)
103
+ } else {
104
+ // default to last 60s to keep quick
105
+ args.push('--last', '60s')
106
+ }
107
+
108
+ let processNameUsed: string | undefined = undefined
109
+ if (appId) {
110
+ validateBundleId(appId)
111
+ // prefer matching the simple process name (last segment of bundle id), but also match full bundle id in subsystem
112
+ const parts = appId.split('.')
113
+ const simpleName = parts[parts.length - 1]
114
+ processNameUsed = simpleName
115
+ // predicate: match process by simple name or full bundle id, or subsystem contains bundle id
116
+ args.push('--predicate', `process == "${simpleName}" or process == "${appId}" or subsystem contains "${appId}"`)
117
+ } else if (tag) {
118
+ // predicate by subsystem/category
119
+ args.push('--predicate', `subsystem contains "${tag}"`)
120
+ }
121
+
122
+ return { args, processNameUsed }
123
+ }
124
+
125
+ async function detectLogPid(appId: string | undefined, pid: number | undefined, deviceId: string) {
126
+ // Attempt pid detection if appId provided and no explicit pid supplied — prefer process name derived from bundle id
127
+ let detectedPid: number | null = null
128
+ if (appId && !pid) {
129
+ const parts = appId.split('.')
130
+ const simpleName = parts[parts.length - 1]
131
+ try {
132
+ const pgrepRes = await iosExecCommand(['simctl', 'spawn', deviceId, 'pgrep', '-f', simpleName], deviceId)
133
+ const out = pgrepRes && pgrepRes.output ? pgrepRes.output.trim() : ''
134
+ const firstLine = out.split(/\r?\n/).find(Boolean)
135
+ if (firstLine) {
136
+ const n = Number(firstLine.trim())
137
+ if (!isNaN(n) && n > 0) detectedPid = n
138
+ }
139
+ } catch {
140
+ // ignore pgrep failures — we'll fall back to process/bundle matching
141
+ }
142
+ }
143
+
144
+ return detectedPid
145
+ }
146
+
147
+ function filterLogs(parsed: StructuredLogEntry[], filters: LogFilters, effectivePid: number | null, processNameUsed: string | undefined) {
148
+ const { contains, since_seconds, level, tag, appId } = filters
149
+ // Apply contains filter
150
+ let filtered = parsed
151
+ if (contains) filtered = filtered.filter(e => e.message && e.message.includes(contains))
152
+
153
+ // Apply since_seconds already applied by log show, but double-check timestamps
154
+ if (since_seconds) {
155
+ const sinceMs = Date.now() - (since_seconds * 1000)
156
+ filtered = filtered.filter(e => e.timestamp && (new Date(e.timestamp).getTime() >= sinceMs))
157
+ }
158
+
159
+ // level filter
160
+ if (level) {
161
+ const L = level.toUpperCase()
162
+ filtered = filtered.filter(e => e.level && e.level.toUpperCase() === L)
163
+ }
164
+
165
+ // tag filter
166
+ if (tag) filtered = filtered.filter(e => e.tag && e.tag.includes(tag))
167
+
168
+ // pid filter (use detected/effective pid if available)
169
+ const pidToFilter = effectivePid
170
+ if (pidToFilter) filtered = filtered.filter(e => e.pid === pidToFilter)
171
+
172
+ // If appId present but no predicate returned lines, try substring match
173
+ if (appId && filtered.length === 0) {
174
+ const matched = parsed.filter(e => (e.message && e.message.includes(appId)) || (e.tag && e.tag.includes(appId)) || (e.message && processNameUsed && e.message.includes(processNameUsed)))
175
+ if (matched.length > 0) filtered = matched
176
+ }
177
+
178
+ return filtered
179
+ }
@@ -0,0 +1,26 @@
1
+ import { encodeScreenshot } from '../screenshot-encoding.js'
2
+ import { randomUUID } from 'node:crypto'
3
+ import { promises as fs } from "fs"
4
+ import { CaptureIOSScreenshotResponse } from "../../types.js"
5
+ import { getIOSDeviceMetadata } from "../../utils/ios/utils.js"
6
+ import { parsePngSize } from '../../utils/image.js'
7
+ import { iosExecCommand } from './context.js'
8
+ export async function collectCaptureScreenshot(deviceId: string = "booted"): Promise<CaptureIOSScreenshotResponse> {
9
+ const device = await getIOSDeviceMetadata(deviceId)
10
+ const tmpFile = `/tmp/mcp-ios-screenshot-${randomUUID()}.png`
11
+
12
+ try {
13
+ await iosExecCommand(['simctl', 'io', deviceId, 'screenshot', tmpFile], deviceId)
14
+
15
+ const buffer = await fs.readFile(tmpFile)
16
+
17
+ const dims = parsePngSize(buffer)
18
+
19
+ const encoded = await encodeScreenshot(buffer, 'ios')
20
+ await fs.rm(tmpFile).catch(() => { })
21
+ return { device, ...encoded, resolution: { width: dims.width, height: dims.height } }
22
+ } catch (e) {
23
+ await fs.rm(tmpFile).catch(() => { })
24
+ throw new Error(`Failed to capture screenshot: ${e instanceof Error ? e.message : String(e)}`)
25
+ }
26
+ }
@@ -0,0 +1,266 @@
1
+ import { UIElement, UIElementSemanticMetadata, UIElementState, UIResolutionSelector, SelectorConfidence } from "../../types.js"
2
+ export interface IDBElement {
3
+ AXFrame?: { x: number | string, y: number | string, width: number | string, height: number | string, w?: number | string, h?: number | string };
4
+ frame?: { x: number | string, y: number | string, width: number | string, height: number | string, w?: number | string, h?: number | string };
5
+ AXIdentifier?: string;
6
+ accessibilityIdentifier?: string;
7
+ identifier?: string;
8
+ AXUniqueId?: string;
9
+ AXLabel?: string;
10
+ AXValue?: string;
11
+ AXTraits?: string[];
12
+ AXElementType?: string;
13
+ type?: string;
14
+ label?: string;
15
+ children?: IDBElement[];
16
+ }
17
+
18
+ export function parseIDBFrame(frame: string | Partial<NonNullable<IDBElement['frame']>> | null | undefined): [number, number, number, number] {
19
+ if (!frame) return [0, 0, 0, 0];
20
+ if (typeof frame === 'string') {
21
+ const nums = frame.match(/-?\d+(?:\.\d+)?/g);
22
+ if (!nums || nums.length < 4) return [0, 0, 0, 0];
23
+ const x = Number(nums[0]);
24
+ const y = Number(nums[1]);
25
+ const w = Number(nums[2]);
26
+ const h = Number(nums[3]);
27
+ return [Math.round(x), Math.round(y), Math.round(x + w), Math.round(y + h)];
28
+ }
29
+
30
+ const x = Number(frame.x || 0);
31
+ const y = Number(frame.y || 0);
32
+ const w = Number(frame.width || frame.w || 0);
33
+ const h = Number(frame.height || frame.h || 0);
34
+ return [Math.round(x), Math.round(y), Math.round(x + w), Math.round(y + h)];
35
+ }
36
+
37
+ export function getCenter(bounds: [number, number, number, number]): [number, number] {
38
+ const [x1, y1, x2, y2] = bounds;
39
+ return [Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)];
40
+ }
41
+
42
+ export function parseIOSNumber(value: unknown): number | null {
43
+ if (typeof value === 'number' && Number.isFinite(value)) return value
44
+ if (typeof value !== 'string') return null
45
+ const parsed = Number(value)
46
+ return Number.isFinite(parsed) ? parsed : null
47
+ }
48
+
49
+ export function normalizeIOSType(value: unknown): string {
50
+ return typeof value === 'string' ? value.trim().toLowerCase() : ''
51
+ }
52
+
53
+ export function inferIOSRole(type: string, traits: string[]): string | null {
54
+ if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait))) return 'slider'
55
+ if (/stepper/.test(type)) return 'stepper'
56
+ if (/picker|pop up button|dropdown/.test(type)) return 'dropdown'
57
+ if (/segmented control/.test(type)) return 'segmented_control'
58
+ if (/button/.test(type) || traits.some((trait) => /button/.test(trait))) return 'button'
59
+ if (/cell/.test(type)) return 'cell'
60
+ if (/switch/.test(type)) return 'switch'
61
+ if (/text field|textfield|search field/.test(type)) return 'text_field'
62
+ if (/image/.test(type)) return 'image'
63
+ if (/window|application|group|scroll view|collection view/.test(type)) return 'container'
64
+ return null
65
+ }
66
+
67
+ export function getIOSStableId(node: IDBElement): string | null {
68
+ const candidates = [node.AXIdentifier, node.accessibilityIdentifier, node.identifier, node.AXUniqueId]
69
+ for (const candidate of candidates) {
70
+ if (typeof candidate === 'string' && candidate.trim().length > 0) return candidate
71
+ }
72
+ return null
73
+ }
74
+
75
+ export function buildIOSSelectorConfidence(source: 'identifier' | 'label' | 'value' | 'type' | 'none'): SelectorConfidence | null {
76
+ switch (source) {
77
+ case 'identifier':
78
+ return { score: 1, reason: 'accessibility_identifier' }
79
+ case 'label':
80
+ return { score: 0.9, reason: 'label_match' }
81
+ case 'value':
82
+ return { score: 0.75, reason: 'value_match' }
83
+ case 'type':
84
+ return { score: 0.35, reason: 'type_match' }
85
+ default:
86
+ return null
87
+ }
88
+ }
89
+
90
+ export function buildIOSSelector(type: string, label: string | null, value: string | null, stableId: string | null): UIResolutionSelector | null {
91
+ if (stableId) return { value: stableId, confidence: buildIOSSelectorConfidence('identifier') }
92
+ if (label) return { value: label, confidence: buildIOSSelectorConfidence('label') }
93
+ if (value) return { value: value, confidence: buildIOSSelectorConfidence('value') }
94
+ if (type) return { value: type, confidence: buildIOSSelectorConfidence('type') }
95
+ return null
96
+ }
97
+
98
+ export function buildIOSSemantic(type: string, traits: string[], role: string | null, value: string | null): UIElementSemanticMetadata {
99
+ const semantic: UIElementSemanticMetadata = {
100
+ is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
101
+ is_container: /window|application|group|scroll view|collection view/.test(type)
102
+ }
103
+
104
+ if (role === 'slider') {
105
+ semantic.semantic_role = 'slider'
106
+ semantic.adjustable = true
107
+ semantic.supported_actions = ['adjust']
108
+ semantic.state_shape = 'continuous'
109
+ } else if (role === 'stepper') {
110
+ semantic.semantic_role = 'stepper'
111
+ semantic.adjustable = true
112
+ semantic.supported_actions = ['increment', 'decrement']
113
+ semantic.state_shape = 'discrete'
114
+ } else if (role === 'dropdown') {
115
+ semantic.semantic_role = 'dropdown'
116
+ semantic.supported_actions = ['tap', 'expand']
117
+ semantic.state_shape = 'semantic'
118
+ } else if (role === 'segmented_control') {
119
+ semantic.semantic_role = 'segmented_control'
120
+ semantic.supported_actions = ['tap']
121
+ semantic.state_shape = 'discrete'
122
+ } else if (traits.some((trait) => /adjustable|slider/i.test(trait)) || /adjustable|slider/.test(type)) {
123
+ semantic.semantic_role = 'custom_adjustable'
124
+ semantic.adjustable = true
125
+ semantic.supported_actions = ['adjust']
126
+ semantic.state_shape = 'continuous'
127
+ } else if (semantic.is_clickable) {
128
+ semantic.supported_actions = ['tap']
129
+ }
130
+
131
+ if (semantic.state_shape === undefined && semantic.adjustable && value !== null) {
132
+ const numericValue = parseIOSNumber(value)
133
+ if (numericValue !== null && numericValue >= 0 && numericValue <= 1) {
134
+ semantic.state_shape = 'continuous'
135
+ }
136
+ }
137
+
138
+ return semantic
139
+ }
140
+
141
+ export function isIOSAdjustable(node: IDBElement, type: string, traits: string[]): boolean {
142
+ return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait))
143
+ }
144
+
145
+ export function extractIOSState(node: IDBElement, type: string, label: string | null, value: string | null, traits: string[]): UIElementState | null {
146
+ const state: UIElementState = {}
147
+ const normalizedTraits = traits.map((trait) => String(trait).toLowerCase())
148
+
149
+ if (normalizedTraits.some((trait) => /selected/.test(trait))) {
150
+ state.selected = label || value || true
151
+ }
152
+
153
+ if (normalizedTraits.some((trait) => /focused/.test(trait))) {
154
+ state.focused = true
155
+ }
156
+
157
+ if (normalizedTraits.some((trait) => /enabled/.test(trait))) {
158
+ state.enabled = true
159
+ }
160
+
161
+ if (normalizedTraits.some((trait) => /disabled/.test(trait))) {
162
+ state.enabled = false
163
+ }
164
+
165
+ if (value && /textfield|search|text/i.test(type)) {
166
+ state.text_value = value
167
+ }
168
+
169
+ if (isIOSAdjustable(node, type, traits)) {
170
+ const rawValue = parseIOSNumber(value)
171
+ if (rawValue !== null) {
172
+ state.raw_value = rawValue
173
+ state.value = rawValue >= 0 && rawValue <= 1 ? Math.round(rawValue * 100) : rawValue
174
+ } else if (value) {
175
+ state.raw_value = value
176
+ state.value = value
177
+ }
178
+ } else if (value) {
179
+ const numericValue = parseIOSNumber(value)
180
+ if (numericValue !== null) {
181
+ state.value = numericValue
182
+ state.raw_value = numericValue
183
+ } else {
184
+ state.value = value
185
+ }
186
+ }
187
+
188
+ return Object.keys(state).length > 0 ? state : null
189
+ }
190
+
191
+ export function traverseIDBNode(node: IDBElement, elements: UIElement[], parentIndex: number = -1, depth: number = 0): number {
192
+ if (!node) return -1;
193
+
194
+ let currentIndex = -1;
195
+
196
+ const { type, label, value, frame, state, stableId, selector, role, semantic, clickable, isUseful } = describeIDBNode(node)
197
+
198
+ if (isUseful) {
199
+ const bounds = parseIDBFrame(frame);
200
+ const element: UIElement = {
201
+ text: label,
202
+ contentDescription: value,
203
+ type: type,
204
+ resourceId: stableId,
205
+ clickable: clickable,
206
+ enabled: true,
207
+ visible: true,
208
+ bounds: bounds,
209
+ center: getCenter(bounds),
210
+ depth: depth,
211
+ state,
212
+ stable_id: stableId,
213
+ role,
214
+ test_tag: stableId,
215
+ selector,
216
+ semantic
217
+ };
218
+
219
+ if (parentIndex !== -1) {
220
+ element.parentId = parentIndex;
221
+ }
222
+
223
+ elements.push(element);
224
+ currentIndex = elements.length - 1;
225
+ }
226
+
227
+ const nextParentIndex = currentIndex !== -1 ? currentIndex : parentIndex;
228
+ const nextDepth = currentIndex !== -1 ? depth + 1 : depth;
229
+
230
+ const childrenIndices: number[] = [];
231
+
232
+ if (node.children && Array.isArray(node.children)) {
233
+ for (const child of node.children) {
234
+ const childIndex = traverseIDBNode(child, elements, nextParentIndex, nextDepth);
235
+ if (childIndex !== -1) {
236
+ childrenIndices.push(childIndex);
237
+ }
238
+ }
239
+ }
240
+
241
+ if (currentIndex !== -1 && childrenIndices.length > 0) {
242
+ elements[currentIndex].children = childrenIndices;
243
+ }
244
+
245
+ return currentIndex;
246
+ }
247
+
248
+ function describeIDBNode(node: IDBElement) {
249
+ const type = node.AXElementType || node.type || "unknown";
250
+ const label = node.AXLabel || node.label || null;
251
+ const value = node.AXValue || null;
252
+ const frame = node.AXFrame || node.frame;
253
+ const traits = node.AXTraits || [];
254
+ const state = extractIOSState(node, type, label, value, traits);
255
+ const normalizedType = normalizeIOSType(type)
256
+ const stableId = getIOSStableId(node)
257
+ const selector = buildIOSSelector(type, label, value, stableId)
258
+ const role = inferIOSRole(normalizedType, traits)
259
+ const semantic = buildIOSSemantic(normalizedType, traits, role, value)
260
+
261
+ const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
262
+
263
+ const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
264
+
265
+ return { type, label, value, frame, state, stableId, selector, role, semantic, clickable, isUseful }
266
+ }