mobile-debug-mcp 0.34.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/mcp-builder/SKILL.md +44 -64
- package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
- package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
- package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
- package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
- package/.agents/skills/test-authoring/SKILL.md +7 -6
- package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
- package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
- package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
- package/.cursor-plugin/marketplace.json +1 -1
- package/.github/copilot-instructions.md +14 -36
- package/.github/workflows/ci.yml +4 -4
- package/AGENTS.md +12 -3
- package/CONTRIBUTING.md +79 -0
- package/README.md +218 -129
- package/agent-plugin/.antigravity-plugin/README.md +48 -0
- package/agent-plugin/.antigravity-plugin/plugin.json +18 -0
- package/agent-plugin/.claude-plugin/plugin.json +2 -2
- package/agent-plugin/.codex-plugin/plugin.json +1 -1
- package/agent-plugin/.cursor-plugin/plugin.json +2 -2
- package/agent-plugin/.mcp.json +1 -1
- package/agent-plugin/README.md +50 -11
- package/agent-plugin/cursor/.cursor-plugin/plugin.json +23 -0
- package/agent-plugin/cursor/LICENSE +201 -0
- package/agent-plugin/cursor/README.md +9 -0
- package/agent-plugin/cursor/skills/mobile-debug-mcp/SKILL.md +253 -0
- package/agent-plugin/mcp.json +1 -1
- package/agent-plugin/mcp_config.json +13 -0
- package/agent-plugin/plugin.json +1 -1
- package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +62 -17
- package/dist/contracts/execution-controls.js +24 -0
- package/dist/contracts/selector.js +8 -0
- package/dist/contracts/target.js +15 -0
- package/dist/contracts/ui.js +1 -0
- package/dist/debugger/android.js +2 -1
- package/dist/debugger/diagnostics.js +54 -0
- package/dist/debugger/guard.js +4 -4
- package/dist/debugger/ios-worker.js +62 -0
- package/dist/debugger/ios.js +57 -0
- package/dist/debugger/protocol.js +33 -0
- package/dist/debugger/sessions.js +133 -74
- package/dist/debugger/worker.js +59 -46
- package/dist/interact/adjustment/adjustment-context.js +41 -0
- package/dist/interact/adjustment/adjustment-gestures.js +55 -0
- package/dist/interact/adjustment/adjustment-plan.js +18 -0
- package/dist/interact/adjustment/adjustment-resolution.js +127 -0
- package/dist/interact/adjustment/adjustment-result.js +19 -0
- package/dist/interact/adjustment/adjustment-semantic.js +64 -0
- package/dist/interact/adjustment/adjustment-verification.js +35 -0
- package/dist/interact/adjustment/adjustment.js +52 -0
- package/dist/interact/adjustment/control-geometry.js +87 -0
- package/dist/interact/adjustment/control-values.js +136 -0
- package/dist/interact/adjustment/slider-labels.js +84 -0
- package/dist/interact/assertions/screen-assertion.js +37 -0
- package/dist/interact/assertions/state-assertion-context.js +35 -0
- package/dist/interact/assertions/state-assertion.js +76 -0
- package/dist/interact/assertions/state-comparison.js +141 -0
- package/dist/interact/assertions/visibility-assertion.js +34 -0
- package/dist/interact/conditional-action.js +12 -7
- package/dist/interact/index.js +77 -3092
- package/dist/interact/input-service.js +1 -0
- package/dist/interact/interaction-trace.js +21 -0
- package/dist/interact/model.js +1 -0
- package/dist/interact/observation.js +1 -0
- package/dist/interact/targeting/candidate-ancestor.js +58 -0
- package/dist/interact/targeting/candidate-result.js +23 -0
- package/dist/interact/targeting/candidate-scoring.js +50 -0
- package/dist/interact/targeting/element-helpers.js +216 -0
- package/dist/interact/targeting/find-element.js +65 -0
- package/dist/interact/targeting/nearby-control.js +56 -0
- package/dist/interact/targeting/resolved-elements.js +21 -0
- package/dist/interact/targeting/tap-target.js +17 -0
- package/dist/interact/targeting/target-scope.js +104 -0
- package/dist/interact/waits/legacy-log-wait.js +38 -0
- package/dist/interact/waits/legacy-wait-observations.js +103 -0
- package/dist/interact/waits/legacy-wait-state.js +34 -0
- package/dist/interact/waits/legacy-wait.js +48 -0
- package/dist/interact/waits/screen-wait.js +49 -0
- package/dist/interact/waits/ui-change-helpers.js +157 -0
- package/dist/interact/waits/ui-change-state.js +18 -0
- package/dist/interact/waits/ui-change-wait.js +77 -0
- package/dist/interact/waits/ui-wait-condition.js +56 -0
- package/dist/interact/waits/ui-wait-state.js +38 -0
- package/dist/interact/waits/ui-wait.js +106 -0
- package/dist/manage/android.js +55 -51
- package/dist/manage/index.js +38 -75
- package/dist/manage/ios/build-types.js +1 -0
- package/dist/manage/ios/build.js +210 -0
- package/dist/manage/ios/project-discovery.js +29 -0
- package/dist/manage/ios/scheme.js +18 -0
- package/dist/manage/ios.js +44 -256
- package/dist/observe/android/context.js +1 -0
- package/dist/observe/android/current-screen.js +55 -0
- package/dist/observe/android/fingerprint.js +13 -0
- package/dist/observe/android/log-stream.js +132 -0
- package/dist/observe/android/logs.js +107 -0
- package/dist/observe/android/screenshot.js +70 -0
- package/dist/observe/android/ui-tree.js +107 -0
- package/dist/observe/android.js +16 -520
- package/dist/observe/evidence.js +114 -75
- package/dist/observe/index.js +16 -263
- package/dist/observe/ios/context.js +16 -0
- package/dist/observe/ios/fingerprint.js +12 -0
- package/dist/observe/ios/log-stream.js +105 -0
- package/dist/observe/ios/logs.js +168 -0
- package/dist/observe/ios/screenshot.js +22 -0
- package/dist/observe/ios/ui-elements.js +241 -0
- package/dist/observe/ios/ui-tree.js +137 -0
- package/dist/observe/ios-trace/artifact.js +63 -0
- package/dist/observe/ios-trace/capabilities.js +86 -0
- package/dist/observe/ios-trace/native.js +93 -0
- package/dist/observe/ios-trace/notification.js +35 -0
- package/dist/observe/ios-trace/process.js +98 -0
- package/dist/observe/ios-trace/sessions.js +244 -0
- package/dist/observe/ios.js +17 -715
- package/dist/observe/logs/model.js +1 -0
- package/dist/{network → observe/network}/index.js +2 -2
- package/dist/observe/readiness-service.js +5 -3
- package/dist/observe/readiness.js +33 -14
- package/dist/observe/screenshot-encoding.js +41 -0
- package/dist/observe/snapshot-collection.js +79 -0
- package/dist/observe/snapshot-logs.js +30 -0
- package/dist/observe/snapshot-metadata.js +43 -31
- package/dist/observe/snapshot-semantic.js +115 -0
- package/dist/observe/system-trace-error.js +7 -0
- package/dist/observe/system-trace-router.js +81 -0
- package/dist/observe/system-trace.js +26 -21
- package/dist/runtime/action-results.js +270 -0
- package/dist/runtime/clock.js +4 -0
- package/dist/server/common.js +1 -256
- package/dist/server/definitions/assert.js +162 -0
- package/dist/server/definitions/diagnostic.js +118 -0
- package/dist/server/definitions/input.js +274 -0
- package/dist/server/definitions/manage.js +194 -0
- package/dist/server/definitions/observe.js +149 -0
- package/dist/server/definitions/target.js +138 -0
- package/dist/server/definitions/wait.js +105 -0
- package/dist/server/handlers/action-execution.js +163 -0
- package/dist/server/handlers/correlation.js +23 -0
- package/dist/server/handlers/debugger-handlers.js +67 -0
- package/dist/server/handlers/debugger-response.js +9 -0
- package/dist/server/handlers/diagnostic.js +77 -0
- package/dist/server/handlers/handler-types.js +1 -0
- package/dist/server/handlers/input.js +154 -0
- package/dist/server/handlers/interact.js +113 -0
- package/dist/server/handlers/journey-assertions.js +35 -0
- package/dist/server/handlers/journey.js +147 -0
- package/dist/server/handlers/manage.js +119 -0
- package/dist/server/handlers/observe.js +95 -0
- package/dist/server/handlers/payload.js +4 -0
- package/dist/server/handlers/tool-handlers.js +170 -0
- package/dist/server/schemas/condition-schema.js +2 -0
- package/dist/server/{debugger-schema.js → schemas/debugger-schema.js} +20 -12
- package/dist/server/{journey-schema.js → schemas/journey-schema.js} +7 -22
- package/dist/server/schemas/system-trace-schema.js +22 -0
- package/dist/server/tool-definitions.js +58 -1135
- package/dist/server-core.js +2 -2
- package/dist/server.js +4 -4
- package/dist/system/android-status.js +101 -0
- package/dist/system/android.js +37 -27
- package/dist/system/index.js +13 -439
- package/dist/system/ios-status.js +132 -0
- package/dist/system/ios.js +11 -8
- package/dist/system/model.js +1 -0
- package/dist/system/probes.js +105 -0
- package/dist/system/status-summary.js +43 -0
- package/dist/system/unsupported-status.js +52 -0
- package/dist/utils/android/commands.js +89 -0
- package/dist/utils/android/gradle.js +139 -0
- package/dist/utils/android/logs.js +103 -0
- package/dist/utils/android/ui-tree.js +233 -0
- package/dist/utils/android/utils.js +6 -564
- package/dist/utils/cli/idb/idb-helper.js +4 -6
- package/dist/utils/cli/idb/install-idb.js +13 -9
- package/dist/utils/cli/ios/preflight-ios.js +42 -38
- package/dist/utils/ios/utils.js +48 -46
- package/dist/utils/java.js +65 -49
- package/dist/utils/resolve-device.js +4 -2
- package/dist/utils/ui/index.js +26 -26
- package/docs/CHANGELOG.md +67 -1
- package/docs/README.md +38 -0
- package/docs/agent-plugin-installation-verification.md +39 -0
- package/docs/benchmarks/README.md +18 -0
- package/docs/benchmarks/interaction-latency/README.md +14 -0
- package/docs/benchmarks/interaction-latency/final.md +4 -1
- package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
- package/docs/benchmarks/ui-tree-performance/README.md +24 -16
- package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
- package/docs/code-structure.md +120 -0
- package/docs/specs/001-state-verification/spec.md +16 -14
- package/docs/specs/002-richer-element-identity/spec.md +12 -10
- package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
- package/docs/specs/004-action-verification-routing/spec.md +3 -3
- package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
- package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
- package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
- package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
- package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
- package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
- package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
- package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
- package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
- package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
- package/docs/specs/014-actionability-resolution/spec.md +20 -18
- package/docs/specs/015-adjustable-control-support/spec.md +17 -15
- package/docs/specs/016-environment-tools/spec.md +20 -20
- package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
- package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
- package/docs/specs/020-conditional-actions/intent.md +5 -3
- package/docs/specs/020-conditional-actions/plan.md +8 -9
- package/docs/specs/020-conditional-actions/spec.md +4 -2
- package/docs/specs/021-app-readiness-signals/intent.md +4 -2
- package/docs/specs/021-app-readiness-signals/plan.md +7 -6
- package/docs/specs/021-app-readiness-signals/spec.md +3 -1
- package/docs/specs/022-correlated-evidence/intent.md +4 -4
- package/docs/specs/022-correlated-evidence/plan.md +10 -11
- package/docs/specs/022-correlated-evidence/spec.md +3 -1
- package/docs/specs/023-android-system-traces/intent.md +3 -2
- package/docs/specs/023-android-system-traces/plan.md +5 -7
- package/docs/specs/023-android-system-traces/spec.md +3 -1
- package/docs/specs/024-android-heap-dump-capture/intent.md +3 -1
- package/docs/specs/025-android-runtime-debugger/intent.md +4 -3
- package/docs/specs/025-android-runtime-debugger/plan.md +3 -1
- package/docs/specs/025-android-runtime-debugger/spec.md +4 -3
- package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
- package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
- package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
- package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
- package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
- package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
- package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
- package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
- package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
- package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
- package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
- package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
- package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
- package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
- package/docs/specs/030-android-coroutine-validation/report.md +240 -0
- package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
- package/docs/specs/031-ios-system-traces/intent.md +38 -0
- package/docs/specs/031-ios-system-traces/investigation.md +92 -0
- package/docs/specs/031-ios-system-traces/late-run-failure-evidence.json +32 -0
- package/docs/specs/031-ios-system-traces/plan.md +138 -0
- package/docs/specs/031-ios-system-traces/pr102-monitor-evidence.json +22 -0
- package/docs/specs/031-ios-system-traces/simulator-evidence.json +47 -0
- package/docs/specs/031-ios-system-traces/spec.md +135 -0
- package/docs/specs/032-ios-runtime-debugger/evidence.json +176 -0
- package/docs/specs/032-ios-runtime-debugger/intent.md +38 -0
- package/docs/specs/032-ios-runtime-debugger/plan.md +95 -0
- package/docs/specs/032-ios-runtime-debugger/report.md +205 -0
- package/docs/specs/032-ios-runtime-debugger/spec.md +153 -0
- package/docs/tools/TOOLS.md +31 -18
- package/docs/tools/{evidence.md → correlated-evidence.md} +9 -1
- package/docs/tools/debugger.md +120 -4
- package/docs/tools/interact.md +42 -20
- package/docs/tools/manage.md +25 -44
- package/docs/tools/observe.md +55 -23
- package/docs/tools/system-traces.md +89 -14
- package/docs/tools/system.md +12 -5
- package/package.json +12 -8
- package/scripts/agent-plugin.mjs +55 -5
- package/scripts/architecture-graph.mjs +69 -0
- package/scripts/check-architecture.mjs +47 -0
- package/scripts/code-quality-baseline.json +1 -0
- package/scripts/code-quality.mjs +161 -0
- package/src/contracts/execution-controls.ts +26 -0
- package/src/contracts/selector.ts +17 -0
- package/src/contracts/target.ts +19 -0
- package/src/contracts/ui.ts +94 -0
- package/src/debugger/android.ts +3 -2
- package/src/debugger/diagnostics.ts +43 -0
- package/src/debugger/guard.ts +10 -5
- package/src/debugger/ios-worker.ts +49 -0
- package/src/debugger/ios.ts +45 -0
- package/{worker → src/debugger/java}/DebuggerWorker.java +9 -3
- package/src/debugger/lldb/breakpoints.py +51 -0
- package/src/debugger/lldb/inspection.py +109 -0
- package/src/debugger/lldb/worker.py +203 -0
- package/src/debugger/protocol.ts +37 -0
- package/src/debugger/sessions.ts +326 -155
- package/src/debugger/worker.ts +57 -41
- package/src/interact/adjustment/adjustment-context.ts +83 -0
- package/src/interact/adjustment/adjustment-gestures.ts +48 -0
- package/src/interact/adjustment/adjustment-plan.ts +23 -0
- package/src/interact/adjustment/adjustment-resolution.ts +104 -0
- package/src/interact/adjustment/adjustment-result.ts +34 -0
- package/src/interact/adjustment/adjustment-semantic.ts +53 -0
- package/src/interact/adjustment/adjustment-verification.ts +60 -0
- package/src/interact/adjustment/adjustment.ts +49 -0
- package/src/interact/adjustment/control-geometry.ts +120 -0
- package/src/interact/adjustment/control-values.ts +141 -0
- package/src/interact/adjustment/slider-labels.ts +84 -0
- package/src/interact/assertions/screen-assertion.ts +39 -0
- package/src/interact/assertions/state-assertion-context.ts +48 -0
- package/src/interact/assertions/state-assertion.ts +64 -0
- package/src/interact/assertions/state-comparison.ts +168 -0
- package/src/interact/assertions/visibility-assertion.ts +41 -0
- package/src/interact/conditional-action.ts +18 -12
- package/src/interact/index.ts +89 -3494
- package/src/interact/input-service.ts +11 -0
- package/src/interact/interaction-trace.ts +36 -0
- package/src/interact/model.ts +109 -0
- package/src/interact/observation.ts +7 -0
- package/src/interact/targeting/candidate-ancestor.ts +49 -0
- package/src/interact/targeting/candidate-result.ts +24 -0
- package/src/interact/targeting/candidate-scoring.ts +42 -0
- package/src/interact/targeting/element-helpers.ts +243 -0
- package/src/interact/targeting/find-element.ts +47 -0
- package/src/interact/targeting/nearby-control.ts +46 -0
- package/src/interact/targeting/resolved-elements.ts +20 -0
- package/src/interact/targeting/tap-target.ts +22 -0
- package/src/interact/targeting/target-scope.ts +121 -0
- package/src/interact/waits/legacy-log-wait.ts +30 -0
- package/src/interact/waits/legacy-wait-observations.ts +55 -0
- package/src/interact/waits/legacy-wait-state.ts +48 -0
- package/src/interact/waits/legacy-wait.ts +34 -0
- package/src/interact/waits/screen-wait.ts +37 -0
- package/src/interact/waits/ui-change-helpers.ts +184 -0
- package/src/interact/waits/ui-change-state.ts +26 -0
- package/src/interact/waits/ui-change-wait.ts +59 -0
- package/src/interact/waits/ui-wait-condition.ts +52 -0
- package/src/interact/waits/ui-wait-state.ts +55 -0
- package/src/interact/waits/ui-wait.ts +84 -0
- package/src/manage/android.ts +56 -50
- package/src/manage/index.ts +63 -93
- package/src/manage/ios/build-types.ts +14 -0
- package/src/manage/ios/build.ts +230 -0
- package/src/manage/ios/project-discovery.ts +26 -0
- package/src/manage/ios/scheme.ts +16 -0
- package/src/manage/ios.ts +67 -298
- package/src/observe/android/context.ts +1 -0
- package/src/observe/android/current-screen.ts +63 -0
- package/src/observe/android/fingerprint.ts +13 -0
- package/src/observe/android/log-stream.ts +127 -0
- package/src/observe/android/logs.ts +110 -0
- package/src/observe/android/screenshot.ts +79 -0
- package/src/observe/android/ui-tree.ts +113 -0
- package/src/observe/android.ts +17 -531
- package/src/observe/evidence.ts +122 -70
- package/src/observe/index.ts +24 -286
- package/src/observe/ios/context.ts +22 -0
- package/src/observe/ios/fingerprint.ts +12 -0
- package/src/observe/ios/log-stream.ts +103 -0
- package/src/observe/ios/logs.ts +179 -0
- package/src/observe/ios/screenshot.ts +26 -0
- package/src/observe/ios/ui-elements.ts +266 -0
- package/src/observe/ios/ui-tree.ts +143 -0
- package/src/observe/ios-trace/artifact.ts +55 -0
- package/src/observe/ios-trace/capabilities.ts +74 -0
- package/src/observe/ios-trace/native.ts +72 -0
- package/src/observe/ios-trace/notification.ts +30 -0
- package/src/observe/ios-trace/process.ts +82 -0
- package/src/observe/ios-trace/sessions.ts +204 -0
- package/src/observe/ios.ts +18 -749
- package/src/observe/logs/model.ts +11 -0
- package/src/{network → observe/network}/index.ts +2 -2
- package/src/observe/readiness-service.ts +6 -4
- package/src/observe/readiness.ts +42 -21
- package/src/observe/screenshot-encoding.ts +32 -0
- package/src/observe/snapshot-collection.ts +69 -0
- package/src/observe/snapshot-logs.ts +26 -0
- package/src/observe/snapshot-metadata.ts +51 -31
- package/src/observe/snapshot-semantic.ts +153 -0
- package/src/observe/system-trace-error.ts +3 -0
- package/src/observe/system-trace-router.ts +61 -0
- package/src/observe/system-trace.ts +28 -19
- package/src/runtime/action-results.ts +376 -0
- package/src/runtime/clock.ts +9 -0
- package/src/server/common.ts +9 -359
- package/src/server/definitions/assert.ts +164 -0
- package/src/server/definitions/diagnostic.ts +120 -0
- package/src/server/definitions/input.ts +276 -0
- package/src/server/definitions/manage.ts +196 -0
- package/src/server/definitions/observe.ts +151 -0
- package/src/server/definitions/target.ts +140 -0
- package/src/server/definitions/wait.ts +107 -0
- package/src/server/handlers/action-execution.ts +164 -0
- package/src/server/handlers/correlation.ts +20 -0
- package/src/server/handlers/debugger-handlers.ts +58 -0
- package/src/server/handlers/debugger-response.ts +7 -0
- package/src/server/handlers/diagnostic.ts +86 -0
- package/src/server/handlers/handler-types.ts +7 -0
- package/src/server/handlers/input.ts +173 -0
- package/src/server/handlers/interact.ts +135 -0
- package/src/server/handlers/journey-assertions.ts +39 -0
- package/src/server/handlers/journey.ts +132 -0
- package/src/server/handlers/manage.ts +141 -0
- package/src/server/handlers/observe.ts +116 -0
- package/src/server/handlers/payload.ts +36 -0
- package/src/server/handlers/tool-handlers.ts +170 -0
- package/src/server/schemas/condition-schema.ts +2 -0
- package/src/server/schemas/debugger-schema.ts +46 -0
- package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +7 -22
- package/src/server/schemas/system-trace-schema.ts +23 -0
- package/src/server/tool-definitions.ts +67 -1143
- package/src/server-core.ts +2 -2
- package/src/server.ts +4 -4
- package/src/system/android-status.ts +124 -0
- package/src/system/android.ts +33 -21
- package/src/system/index.ts +21 -525
- package/src/system/ios-status.ts +147 -0
- package/src/system/ios.ts +14 -10
- package/src/system/model.ts +41 -0
- package/src/system/probes.ts +114 -0
- package/src/system/status-summary.ts +47 -0
- package/src/system/unsupported-status.ts +55 -0
- package/src/types.ts +16 -106
- package/src/utils/android/commands.ts +81 -0
- package/src/utils/android/gradle.ts +132 -0
- package/src/utils/android/logs.ts +103 -0
- package/src/utils/android/ui-tree.ts +239 -0
- package/src/utils/android/utils.ts +16 -556
- package/src/utils/cli/idb/check-idb.ts +4 -4
- package/src/utils/cli/idb/idb-helper.ts +6 -8
- package/src/utils/cli/idb/install-idb.ts +15 -9
- package/src/utils/cli/ios/preflight-ios.ts +39 -31
- package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
- package/src/utils/ios/utils.ts +60 -61
- package/src/utils/java.ts +62 -43
- package/src/utils/resolve-device.ts +9 -6
- package/src/utils/ui/index.ts +28 -28
- package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
- package/test/device/automated/debugger/coroutine-support.ts +140 -0
- package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
- package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
- package/test/device/automated/debugger/runtime.android.test.ts +19 -3
- package/test/device/automated/debugger/runtime.ios.test.ts +221 -0
- package/test/device/automated/interact/interaction.android.test.ts +56 -0
- package/test/device/automated/observe/ios-system-trace.smoke.ts +119 -0
- package/test/device/automated/observe/system-trace.smoke.ts +1 -1
- package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
- package/test/fixtures/contracts/tool-definitions.json +3696 -0
- package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
- package/test/fixtures/coroutine-app/README.md +48 -0
- package/test/fixtures/coroutine-app/build.ts +35 -0
- package/test/fixtures/debugger-app/MainActivity.java +21 -0
- package/test/fixtures/debugger-app/MainActivity.kt +32 -0
- package/test/fixtures/debugger-app/README.md +52 -0
- package/test/fixtures/debugger-app/build.ts +25 -15
- package/test/fixtures/ios-debugger-app/Main.swift +95 -0
- package/test/fixtures/ios-debugger-app/README.md +27 -0
- package/test/fixtures/ios-debugger-app/build.ts +21 -0
- package/test/fixtures/latency-journey-app/README.md +2 -0
- package/test/fixtures/readiness-app/README.md +2 -0
- package/test/helpers/ios-trace-shutdown.ts +29 -0
- package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
- package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
- package/test/unit/debugger/diagnostics.test.ts +80 -0
- package/test/unit/debugger/guards.test.ts +4 -0
- package/test/unit/debugger/handle-guards.test.ts +88 -0
- package/test/unit/debugger/ios-preparation.test.ts +57 -0
- package/test/unit/debugger/ios.test.ts +104 -0
- package/test/unit/debugger/lldb-worker.py +166 -0
- package/test/unit/debugger/lldb-worker.test.ts +10 -0
- package/test/unit/debugger/preparation.test.ts +32 -0
- package/test/unit/debugger/protocol.test.ts +15 -0
- package/test/unit/debugger/sessions.test.ts +1 -1
- package/test/unit/interact/adjustment_service.test.ts +42 -0
- package/test/unit/interact/adjustment_verification.test.ts +35 -0
- package/test/unit/interact/clock_dependencies.test.ts +63 -0
- package/test/unit/interact/element_helpers.test.ts +19 -0
- package/test/unit/interact/legacy_wait_service.test.ts +17 -0
- package/test/unit/interact/state_comparison.test.ts +20 -0
- package/test/unit/interact/target_scope.test.ts +35 -0
- package/test/unit/network/get_network_activity.test.ts +1 -1
- package/test/unit/observe/ios-trace-native.test.ts +90 -0
- package/test/unit/observe/ios-trace-sessions.test.ts +131 -0
- package/test/unit/observe/readiness.test.ts +1 -1
- package/test/unit/observe/screenshot-encoding.test.ts +30 -0
- package/test/unit/observe/snapshot-logs.test.ts +14 -0
- package/test/unit/observe/system-trace-router.test.ts +49 -0
- package/test/unit/quality/agent-plugin-export.test.ts +38 -0
- package/test/unit/quality/architecture.test.ts +60 -0
- package/test/unit/quality/code-quality.test.ts +58 -0
- package/test/unit/quality/interaction_boundaries.test.ts +24 -0
- package/test/unit/server/conditional_actions.test.ts +1 -1
- package/test/unit/server/conditional_schema.test.ts +2 -2
- package/test/unit/server/debugger-response.test.ts +63 -0
- package/test/unit/server/system-trace.test.ts +12 -3
- package/test/unit/server/tool_definitions_parity.test.ts +8 -0
- package/dist/server/debugger-handlers.js +0 -38
- package/dist/server/system-trace-schema.js +0 -17
- package/dist/server/tool-handlers.js +0 -1000
- package/docs/ROADMAP.md +0 -646
- package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
- package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
- package/docs/specs/baseline-spec-v0.md +0 -312
- package/docs/specs/mcp-tooling-spec-v1.md +0 -446
- package/src/server/debugger-handlers.ts +0 -29
- package/src/server/debugger-schema.ts +0 -37
- package/src/server/system-trace-schema.ts +0 -17
- package/src/server/tool-handlers.ts +0 -1008
- /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
- /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
- /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
- /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { systemClock } from '../runtime/clock.js';
|
|
2
|
+
import { createTraceStep, nextActionId } from '../runtime/action-results.js';
|
|
3
|
+
export function buildObservationTrace({ actionType, stage, success, attempts, metadata }, clock = systemClock) {
|
|
4
|
+
const now = clock.now();
|
|
5
|
+
const actionId = nextActionId(actionType, now);
|
|
6
|
+
const steps = [
|
|
7
|
+
createTraceStep({
|
|
8
|
+
stage,
|
|
9
|
+
timestamp: now,
|
|
10
|
+
result: success ? 'success' : 'failure',
|
|
11
|
+
attemptIndex: 0,
|
|
12
|
+
metadata
|
|
13
|
+
})
|
|
14
|
+
];
|
|
15
|
+
return {
|
|
16
|
+
action_id: actionId,
|
|
17
|
+
steps,
|
|
18
|
+
final_outcome: success ? 'success' : 'failure',
|
|
19
|
+
attempts: Math.max(1, Math.floor(attempts || 1))
|
|
20
|
+
};
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { _isSemanticActionable } from './element-helpers.js';
|
|
2
|
+
import { nearbyControl } from './nearby-control.js';
|
|
3
|
+
function parentOf(elements, parent) {
|
|
4
|
+
const index = typeof parent === 'number' ? parent : /^\d+$/.test(parent) ? Number(parent) : null;
|
|
5
|
+
if (index !== null && elements[index])
|
|
6
|
+
return { el: elements[index], idx: index };
|
|
7
|
+
if (typeof parent !== 'string')
|
|
8
|
+
return null;
|
|
9
|
+
const found = elements.findIndex(el => el.resourceId === parent || el.id === parent);
|
|
10
|
+
return found >= 0 && elements[found] ? { el: elements[found], idx: found } : null;
|
|
11
|
+
}
|
|
12
|
+
function ancestor(elements, chosen) {
|
|
13
|
+
if (!Array.isArray(chosen.el.bounds) || chosen.el.parentId == null)
|
|
14
|
+
return null;
|
|
15
|
+
let current = chosen, safety = 0;
|
|
16
|
+
while (safety < 20 && !(current.el.clickable || current.el.focusable) && current.el.parentId != null) {
|
|
17
|
+
const parent = parentOf(elements, current.el.parentId);
|
|
18
|
+
if (!parent)
|
|
19
|
+
break;
|
|
20
|
+
current = parent;
|
|
21
|
+
if (current.el.clickable || current.el.enabled || current.el.focusable)
|
|
22
|
+
return current;
|
|
23
|
+
safety++;
|
|
24
|
+
}
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
function containing(elements, chosen) {
|
|
28
|
+
const child = chosen.el.bounds;
|
|
29
|
+
if (!Array.isArray(child))
|
|
30
|
+
return null;
|
|
31
|
+
let best = null, area = Infinity;
|
|
32
|
+
for (let idx = 0; idx < elements.length; idx++) {
|
|
33
|
+
const el = elements[idx];
|
|
34
|
+
if (!el || !(el.clickable || el.focusable) || !Array.isArray(el.bounds) || el.bounds.length < 4)
|
|
35
|
+
continue;
|
|
36
|
+
const [l, t, r, b] = el.bounds;
|
|
37
|
+
if (l <= child[0] && t <= child[1] && r >= child[2] && b >= child[3]) {
|
|
38
|
+
const size = (r - l) * (b - t);
|
|
39
|
+
if (size < area) {
|
|
40
|
+
area = size;
|
|
41
|
+
best = { el, idx };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return best;
|
|
46
|
+
}
|
|
47
|
+
export function preferActionable(best, elements, screen) {
|
|
48
|
+
const parent = ancestor(elements, best) ?? containing(elements, best);
|
|
49
|
+
if (parent)
|
|
50
|
+
best = { ...parent, score: Math.min(1, best.score + .02), reason: 'clickable_parent_preferred', interactable: true };
|
|
51
|
+
if (!(best.el.clickable || best.el.focusable || _isSemanticActionable(best.el))) {
|
|
52
|
+
const nearby = nearbyControl(elements, best, screen);
|
|
53
|
+
if (nearby)
|
|
54
|
+
best = { el: nearby.el, idx: nearby.idx, score: Math.min(1, best.score + .02),
|
|
55
|
+
reason: nearby.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control', interactable: true };
|
|
56
|
+
}
|
|
57
|
+
return best;
|
|
58
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { _summarizeResolutionCandidate } from './element-helpers.js';
|
|
2
|
+
export function candidateResult(best, candidates) {
|
|
3
|
+
const b = best.el.bounds;
|
|
4
|
+
const bounds = Array.isArray(b) ? { left: b[0], top: b[1], right: b[2], bottom: b[3] } : null;
|
|
5
|
+
const unique = candidates.filter((candidate, index, array) => index === array.findIndex(other => other.idx === candidate.idx && other.el === candidate.el));
|
|
6
|
+
const alternates = unique.filter(candidate => candidate.idx !== best.idx || candidate.el !== best.el).slice(0, 3).map(_summarizeResolutionCandidate);
|
|
7
|
+
const element = {
|
|
8
|
+
text: best.el.text ?? null, resourceId: best.el.resourceId ?? null,
|
|
9
|
+
contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null, class: best.el.type ?? best.el.class ?? null,
|
|
10
|
+
bounds, clickable: !!best.el.clickable, enabled: !!best.el.enabled,
|
|
11
|
+
stable_id: best.el.stable_id ?? null, role: best.el.role ?? null, test_tag: best.el.test_tag ?? null,
|
|
12
|
+
selector: best.el.selector ?? null, semantic: best.el.semantic ?? null,
|
|
13
|
+
tapCoordinates: bounds ? { x: Math.floor((bounds.left + bounds.right) / 2), y: Math.floor((bounds.top + bounds.bottom) / 2) } : null,
|
|
14
|
+
telemetry: { matchedIndex: best.idx ?? null, matchedInteractable: !!best.interactable, sliderLike: best.reason === 'slider_track_preferred' }
|
|
15
|
+
};
|
|
16
|
+
if (best.reason === 'slider_track_preferred')
|
|
17
|
+
element.interactionHint = { kind: 'slider',
|
|
18
|
+
axis: bounds && (bounds.bottom - bounds.top) > (bounds.right - bounds.left) ? 'vertical' : 'horizontal', trackBounds: bounds };
|
|
19
|
+
const score = Math.min(1, Number(best.score.toFixed(3)));
|
|
20
|
+
return { found: true, element, score, confidence: score, resolution: {
|
|
21
|
+
confidence: score, reason: best.reason, fallback_available: alternates.length > 0, matched_count: unique.length, alternates
|
|
22
|
+
} };
|
|
23
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { _normalize, _isSemanticActionable } from './element-helpers.js';
|
|
2
|
+
function textScore(el, query, exact) {
|
|
3
|
+
const text = _normalize(el.text ?? el.label ?? el.value ?? '');
|
|
4
|
+
const content = _normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
|
|
5
|
+
const resource = _normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
|
|
6
|
+
const type = _normalize(el.type ?? el.class ?? '');
|
|
7
|
+
const choices = [
|
|
8
|
+
[text, text === query, 1, 'exact_text_match'], [content, content === query, .95, 'exact_content_desc_match'],
|
|
9
|
+
[resource, resource === query, .92, 'exact_resource_id_match']
|
|
10
|
+
];
|
|
11
|
+
if (!exact)
|
|
12
|
+
choices.push([text, text.includes(query), .6, 'partial_text_match'], [content, content.includes(query), .55, 'partial_content_desc_match'], [resource, resource.includes(query), .7, 'partial_resource_id_match']);
|
|
13
|
+
choices.push([type, exact ? type === query : type.includes(query), .3, exact ? 'exact_class_match' : 'partial_class_match']);
|
|
14
|
+
for (const [value, matched, score, reason] of choices)
|
|
15
|
+
if (value && matched)
|
|
16
|
+
return { score, reason };
|
|
17
|
+
return { score: 0, reason: 'best_scoring_candidate' };
|
|
18
|
+
}
|
|
19
|
+
function semanticScore(el, query, exact, initial) {
|
|
20
|
+
const role = _normalize(el.semantic?.semantic_role ?? '');
|
|
21
|
+
const actions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map(_normalize).filter(Boolean) : [];
|
|
22
|
+
const result = { ...initial };
|
|
23
|
+
if (!result.score && role && (exact ? role === query : role.includes(query))) {
|
|
24
|
+
result.score = .5;
|
|
25
|
+
result.reason = 'semantic_role_match';
|
|
26
|
+
}
|
|
27
|
+
if (actions.some(action => exact ? action === query : action.includes(query))) {
|
|
28
|
+
result.score = Math.max(result.score, result.score > 0 ? .65 : .6);
|
|
29
|
+
result.reason = 'semantic_action_match';
|
|
30
|
+
}
|
|
31
|
+
if (!exact && result.score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(query)) {
|
|
32
|
+
result.score = .45;
|
|
33
|
+
result.reason = 'semantic_control_match';
|
|
34
|
+
}
|
|
35
|
+
return result;
|
|
36
|
+
}
|
|
37
|
+
export function scoreCandidate(el, idx, query, exact) {
|
|
38
|
+
if (!el || !el.visible)
|
|
39
|
+
return null;
|
|
40
|
+
const bounds = el.bounds || [0, 0, 0, 0];
|
|
41
|
+
if (!Array.isArray(bounds) || bounds.length < 4)
|
|
42
|
+
return null;
|
|
43
|
+
if (bounds[2] <= bounds[0] || bounds[3] <= bounds[1])
|
|
44
|
+
return null;
|
|
45
|
+
const interactable = !!(el.clickable || el.enabled || el.focusable || _isSemanticActionable(el));
|
|
46
|
+
const result = semanticScore(el, query, exact, textScore(el, query, exact));
|
|
47
|
+
if (result.score > 0 && interactable)
|
|
48
|
+
result.score += .05;
|
|
49
|
+
return result.score > 0 ? { el, idx, ...result, interactable } : null;
|
|
50
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
export function _normalize(s) {
|
|
3
|
+
if (s === null || s === undefined)
|
|
4
|
+
return '';
|
|
5
|
+
try {
|
|
6
|
+
return String(s).toLowerCase().trim();
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return '';
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export function _normalizeBounds(bounds) {
|
|
13
|
+
if (!Array.isArray(bounds) || bounds.length < 4)
|
|
14
|
+
return null;
|
|
15
|
+
const normalized = bounds.slice(0, 4).map((value) => Number(value));
|
|
16
|
+
if (normalized.some((value) => Number.isNaN(value)))
|
|
17
|
+
return null;
|
|
18
|
+
return normalized;
|
|
19
|
+
}
|
|
20
|
+
export function _hash(value) {
|
|
21
|
+
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
22
|
+
}
|
|
23
|
+
function matchesField(value, query, contains) {
|
|
24
|
+
if (query === undefined || query === null)
|
|
25
|
+
return true;
|
|
26
|
+
const normalized = _normalize(query);
|
|
27
|
+
return contains ? value.includes(normalized) : value === normalized;
|
|
28
|
+
}
|
|
29
|
+
export function _matchesSelector(el, selector) {
|
|
30
|
+
if (!selector)
|
|
31
|
+
return false;
|
|
32
|
+
const contains = !!selector.contains;
|
|
33
|
+
return matchesField(_normalize(el.text ?? el.label ?? el.value ?? ''), selector.text, contains) &&
|
|
34
|
+
matchesField(_normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''), selector.resource_id, contains) &&
|
|
35
|
+
matchesField(_normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''), selector.accessibility_id, contains);
|
|
36
|
+
}
|
|
37
|
+
export function _findFirstMatchingElement(elements, selector) {
|
|
38
|
+
if (!selector)
|
|
39
|
+
return null;
|
|
40
|
+
for (let i = 0; i < elements.length; i++) {
|
|
41
|
+
const el = elements[i];
|
|
42
|
+
if (!el)
|
|
43
|
+
continue;
|
|
44
|
+
if (_matchesSelector(el, selector))
|
|
45
|
+
return { el, idx: i };
|
|
46
|
+
}
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
export function _resolveParentIndex(elements, parentId) {
|
|
50
|
+
if (parentId === undefined || parentId === null)
|
|
51
|
+
return null;
|
|
52
|
+
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
53
|
+
return parentId;
|
|
54
|
+
}
|
|
55
|
+
if (typeof parentId === 'string') {
|
|
56
|
+
const normalized = _normalize(parentId);
|
|
57
|
+
if (!normalized)
|
|
58
|
+
return null;
|
|
59
|
+
if (/^\d+$/.test(normalized)) {
|
|
60
|
+
const index = Number(normalized);
|
|
61
|
+
if (index >= 0 && index < elements.length)
|
|
62
|
+
return index;
|
|
63
|
+
}
|
|
64
|
+
const foundIndex = elements.findIndex((el) => {
|
|
65
|
+
if (!el)
|
|
66
|
+
return false;
|
|
67
|
+
return _normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
68
|
+
_normalize(el.stable_id ?? '') === normalized;
|
|
69
|
+
});
|
|
70
|
+
return foundIndex >= 0 ? foundIndex : null;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
export function _isVisibleElement(el) {
|
|
75
|
+
const bounds = _normalizeBounds(el.bounds);
|
|
76
|
+
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
|
|
77
|
+
}
|
|
78
|
+
export function _isTapActionable(el, storedStableId, platform) {
|
|
79
|
+
if (!_isVisibleElement(el)) {
|
|
80
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
81
|
+
}
|
|
82
|
+
if (el.enabled === false) {
|
|
83
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
84
|
+
}
|
|
85
|
+
const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
|
|
86
|
+
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => _normalize(action) === 'tap')));
|
|
87
|
+
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
88
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
|
|
89
|
+
}
|
|
90
|
+
if (storedStableId) {
|
|
91
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
92
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return { actionable: true };
|
|
96
|
+
}
|
|
97
|
+
export function _isAdjustableActionable(el, storedStableId) {
|
|
98
|
+
if (!_isVisibleElement(el)) {
|
|
99
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
100
|
+
}
|
|
101
|
+
if (el.enabled === false) {
|
|
102
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
103
|
+
}
|
|
104
|
+
if (storedStableId) {
|
|
105
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
106
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return { actionable: true };
|
|
110
|
+
}
|
|
111
|
+
export function _computeElementId(platform, deviceId, el, index) {
|
|
112
|
+
const identity = {
|
|
113
|
+
platform,
|
|
114
|
+
deviceId: deviceId || '',
|
|
115
|
+
text: _normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
116
|
+
resourceId: _normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
117
|
+
accessibilityId: _normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
|
|
118
|
+
class: _normalize(el.type ?? el.class ?? ''),
|
|
119
|
+
bounds: _normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
120
|
+
index
|
|
121
|
+
};
|
|
122
|
+
return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`;
|
|
123
|
+
}
|
|
124
|
+
export function _resolvedTargetFromElement(elementId, element, index) {
|
|
125
|
+
return {
|
|
126
|
+
elementId,
|
|
127
|
+
text: element.text ?? null,
|
|
128
|
+
resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
|
|
129
|
+
accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
|
|
130
|
+
class: element.type ?? element.class ?? null,
|
|
131
|
+
bounds: _normalizeBounds(element.bounds),
|
|
132
|
+
index,
|
|
133
|
+
state: element.state ?? null,
|
|
134
|
+
stable_id: element.stable_id ?? null,
|
|
135
|
+
role: element.role ?? null,
|
|
136
|
+
test_tag: element.test_tag ?? null,
|
|
137
|
+
selector: element.selector ?? null,
|
|
138
|
+
semantic: element.semantic ?? null
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
export function _summarizeResolutionCandidate(candidate) {
|
|
142
|
+
const bounds = _normalizeBounds(candidate.el.bounds);
|
|
143
|
+
return {
|
|
144
|
+
text: candidate.el.text ?? null,
|
|
145
|
+
resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
|
|
146
|
+
accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
|
|
147
|
+
class: candidate.el.type ?? candidate.el.class ?? null,
|
|
148
|
+
bounds: bounds
|
|
149
|
+
? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
|
|
150
|
+
: null,
|
|
151
|
+
clickable: !!candidate.el.clickable,
|
|
152
|
+
enabled: !!candidate.el.enabled,
|
|
153
|
+
score: candidate.score,
|
|
154
|
+
reason: candidate.reason
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
export function _isAdjustableControl(el) {
|
|
158
|
+
if (!el)
|
|
159
|
+
return false;
|
|
160
|
+
const type = _normalize(el.type ?? el.class ?? '');
|
|
161
|
+
const role = _normalize(el.role ?? '');
|
|
162
|
+
return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
|
|
163
|
+
}
|
|
164
|
+
export function _isSemanticActionable(el) {
|
|
165
|
+
if (!el?.semantic)
|
|
166
|
+
return false;
|
|
167
|
+
if (el.semantic.adjustable)
|
|
168
|
+
return true;
|
|
169
|
+
return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0;
|
|
170
|
+
}
|
|
171
|
+
export function _resolveActionableAncestor(elements, chosen) {
|
|
172
|
+
if (!chosen)
|
|
173
|
+
return null;
|
|
174
|
+
if (hasActionableTrait(chosen.el))
|
|
175
|
+
return chosen;
|
|
176
|
+
let current = chosen;
|
|
177
|
+
let safety = 0;
|
|
178
|
+
while (safety < 20 && !(hasActionableTrait(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
179
|
+
const parentIndex = _resolveParentIndex(elements, current.el.parentId);
|
|
180
|
+
if (parentIndex !== null && elements[parentIndex]) {
|
|
181
|
+
current = { el: elements[parentIndex], idx: parentIndex };
|
|
182
|
+
if (hasActionableTrait(current.el))
|
|
183
|
+
return current;
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
safety++;
|
|
189
|
+
}
|
|
190
|
+
const childBounds = _normalizeBounds(chosen.el.bounds);
|
|
191
|
+
if (!childBounds)
|
|
192
|
+
return null;
|
|
193
|
+
const [cl, ct, cr, cb] = childBounds;
|
|
194
|
+
let best = null;
|
|
195
|
+
let bestArea = Infinity;
|
|
196
|
+
for (let i = 0; i < elements.length; i++) {
|
|
197
|
+
const el = elements[i];
|
|
198
|
+
if (!el || !(hasActionableTrait(el)))
|
|
199
|
+
continue;
|
|
200
|
+
const bounds = _normalizeBounds(el.bounds);
|
|
201
|
+
if (!bounds)
|
|
202
|
+
continue;
|
|
203
|
+
const [pl, pt, pr, pb] = bounds;
|
|
204
|
+
if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
|
|
205
|
+
const area = (pr - pl) * (pb - pt);
|
|
206
|
+
if (area < bestArea) {
|
|
207
|
+
bestArea = area;
|
|
208
|
+
best = { el, idx: i };
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return best;
|
|
213
|
+
}
|
|
214
|
+
function hasActionableTrait(el) {
|
|
215
|
+
return !!(el.clickable || el.focusable || _isSemanticActionable(el));
|
|
216
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { systemClock } from '../../runtime/clock.js';
|
|
2
|
+
import { _normalize } from './element-helpers.js';
|
|
3
|
+
import { checkDebuggerGuard } from '../../debugger/guard.js';
|
|
4
|
+
import { scoreCandidate } from './candidate-scoring.js';
|
|
5
|
+
import { preferActionable } from './candidate-ancestor.js';
|
|
6
|
+
import { candidateResult } from './candidate-result.js';
|
|
7
|
+
function scoreTree(search, tree, query, exact) {
|
|
8
|
+
if (!tree || !Array.isArray(tree.elements))
|
|
9
|
+
return;
|
|
10
|
+
const candidates = [];
|
|
11
|
+
let improved = false;
|
|
12
|
+
for (let i = 0; i < tree.elements.length; i++) {
|
|
13
|
+
try {
|
|
14
|
+
const candidate = scoreCandidate(tree.elements[i], i, query, exact);
|
|
15
|
+
if (!candidate)
|
|
16
|
+
continue;
|
|
17
|
+
candidates.push(candidate);
|
|
18
|
+
if (!search.best || candidate.score > search.best.score) {
|
|
19
|
+
search.best = candidate;
|
|
20
|
+
search.tree = tree;
|
|
21
|
+
improved = true;
|
|
22
|
+
if (candidate.score >= .95) {
|
|
23
|
+
search.stop = true;
|
|
24
|
+
break;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
checkDebuggerGuard();
|
|
30
|
+
console.error('Error scoring element:', error);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (improved)
|
|
34
|
+
search.candidates = candidates.slice();
|
|
35
|
+
}
|
|
36
|
+
export async function findElement({ query, exact = false, timeoutMs = 3000, platform, deviceId }, observe, clock = systemClock) {
|
|
37
|
+
const start = clock.now(), deadline = start + timeoutMs, normalized = _normalize(query);
|
|
38
|
+
if (!normalized)
|
|
39
|
+
return { found: false, error: 'Empty query' };
|
|
40
|
+
const search = { best: null, tree: null, candidates: [], stop: false };
|
|
41
|
+
while (clock.now() <= deadline) {
|
|
42
|
+
try {
|
|
43
|
+
scoreTree(search, await observe({ platform, deviceId }), normalized, exact);
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
checkDebuggerGuard();
|
|
47
|
+
console.error('Error fetching UI tree:', error);
|
|
48
|
+
}
|
|
49
|
+
if (search.stop || clock.now() > deadline)
|
|
50
|
+
break;
|
|
51
|
+
await clock.sleep(100);
|
|
52
|
+
}
|
|
53
|
+
if (!search.best)
|
|
54
|
+
return { found: false, error: 'Element not found' };
|
|
55
|
+
try {
|
|
56
|
+
const elements = search.tree && Array.isArray(search.tree.elements) ? search.tree.elements : [];
|
|
57
|
+
const screen = search.tree?.resolution && typeof search.tree.resolution === 'object' ? search.tree.resolution : null;
|
|
58
|
+
search.best = preferActionable(search.best, elements, screen);
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
checkDebuggerGuard();
|
|
62
|
+
console.error('Error resolving ancestor:', error);
|
|
63
|
+
}
|
|
64
|
+
return candidateResult(search.best, search.candidates);
|
|
65
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { _normalize, _normalizeBounds } from './element-helpers.js';
|
|
2
|
+
function trackLimits(label, screen) {
|
|
3
|
+
const width = Number(screen?.width) > 0 ? Number(screen?.width) : 0;
|
|
4
|
+
const height = Number(screen?.height) > 0 ? Number(screen?.height) : 0;
|
|
5
|
+
return { length: Math.max(220, width > 0 ? Math.floor(width * .18) : 0, height > 0 ? Math.floor(height * .18) : 0),
|
|
6
|
+
thickness: Math.max(180, width > 0 ? Math.floor(width * .08) : 0, height > 0 ? Math.floor(height * .08) : 0),
|
|
7
|
+
labelWidth: label[2] - label[0], labelHeight: label[3] - label[1] };
|
|
8
|
+
}
|
|
9
|
+
function nearbyScore(el, label, limits) {
|
|
10
|
+
if (!el || !(el.clickable || el.focusable) || el.visible === false)
|
|
11
|
+
return null;
|
|
12
|
+
const b = _normalizeBounds(el.bounds);
|
|
13
|
+
if (!b)
|
|
14
|
+
return null;
|
|
15
|
+
const width = b[2] - b[0], height = b[3] - b[1], gap = b[1] - label[3];
|
|
16
|
+
if (gap < -32 || gap > 640 || Math.min(label[2], b[2]) - Math.max(label[0], b[0]) < -32)
|
|
17
|
+
return null;
|
|
18
|
+
const horizontal = width >= Math.max(limits.length, Math.floor(limits.labelWidth * 1.5)) && height <= limits.thickness;
|
|
19
|
+
const vertical = height >= Math.max(limits.length, Math.floor(limits.labelHeight * 1.5)) && width <= limits.thickness;
|
|
20
|
+
const sliderLike = /slider|seek|range/i.test(_normalize(el.type ?? el.class ?? '')) || horizontal || vertical;
|
|
21
|
+
return { score: appearanceScore(gap, el, width, limits.labelWidth, sliderLike), sliderLike };
|
|
22
|
+
}
|
|
23
|
+
function appearanceScore(gap, el, width, labelWidth, sliderLike) {
|
|
24
|
+
const text = _normalize(el.text ?? el.label ?? el.value ?? '');
|
|
25
|
+
const content = _normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
|
|
26
|
+
const type = _normalize(el.type ?? el.class ?? '');
|
|
27
|
+
let score = gap;
|
|
28
|
+
if (!text && !content)
|
|
29
|
+
score -= 18;
|
|
30
|
+
if (sliderLike)
|
|
31
|
+
score -= 30;
|
|
32
|
+
if (/view|layout|group|frame/i.test(type))
|
|
33
|
+
score -= 10;
|
|
34
|
+
if (width > labelWidth * 1.5)
|
|
35
|
+
score -= 8;
|
|
36
|
+
if (text || content)
|
|
37
|
+
score += 20;
|
|
38
|
+
return score;
|
|
39
|
+
}
|
|
40
|
+
export function nearbyControl(elements, chosen, screen) {
|
|
41
|
+
if (!chosen)
|
|
42
|
+
return null;
|
|
43
|
+
const label = _normalizeBounds(chosen.el.bounds);
|
|
44
|
+
if (!label)
|
|
45
|
+
return null;
|
|
46
|
+
const limits = trackLimits(label, screen);
|
|
47
|
+
let best = null, score = Infinity;
|
|
48
|
+
for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + 8); i++) {
|
|
49
|
+
const candidate = nearbyScore(elements[i], label, limits);
|
|
50
|
+
if (candidate && candidate.score < score) {
|
|
51
|
+
score = candidate.score;
|
|
52
|
+
best = { el: elements[i], idx: i, sliderLike: candidate.sliderLike };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return best;
|
|
56
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** Bounded, insertion-ordered handles; repeated observations refresh retention order. */
|
|
2
|
+
export class ResolvedElements {
|
|
3
|
+
maximum;
|
|
4
|
+
elements = new Map();
|
|
5
|
+
constructor(maximum = 256) {
|
|
6
|
+
this.maximum = maximum;
|
|
7
|
+
}
|
|
8
|
+
get(id) { return this.elements.get(id); }
|
|
9
|
+
clear() { this.elements.clear(); }
|
|
10
|
+
remember(id, context) {
|
|
11
|
+
if (this.elements.has(id))
|
|
12
|
+
this.elements.delete(id);
|
|
13
|
+
this.elements.set(id, context);
|
|
14
|
+
while (this.elements.size > this.maximum) {
|
|
15
|
+
const oldest = this.elements.keys().next().value;
|
|
16
|
+
if (!oldest)
|
|
17
|
+
break;
|
|
18
|
+
this.elements.delete(oldest);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { _isTapActionable, _normalizeBounds } from './element-helpers.js';
|
|
2
|
+
export function resolveTap(tree, stored, deps) {
|
|
3
|
+
const platform = tree?.device?.platform === 'ios' ? 'ios' : stored.platform;
|
|
4
|
+
const deviceId = tree?.device?.id || stored.deviceId;
|
|
5
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
6
|
+
const current = deps.current(elements, platform, deviceId, stored);
|
|
7
|
+
if (!current)
|
|
8
|
+
return { failure: 'STALE_REFERENCE', target: null };
|
|
9
|
+
const target = deps.target(current.el, current.index);
|
|
10
|
+
const actionable = _isTapActionable(current.el, stored.stable_id, stored.platform);
|
|
11
|
+
if (!actionable.actionable)
|
|
12
|
+
return { failure: actionable.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', target };
|
|
13
|
+
const bounds = _normalizeBounds(current.el.bounds) ?? stored.bounds;
|
|
14
|
+
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1])
|
|
15
|
+
return { failure: 'ELEMENT_NOT_INTERACTABLE', target };
|
|
16
|
+
return { target, point: { x: Math.floor((bounds[0] + bounds[2]) / 2), y: Math.floor((bounds[1] + bounds[3]) / 2) } };
|
|
17
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { _normalize, _computeElementId } from './element-helpers.js';
|
|
2
|
+
import { _collectSubtreeIndices } from '../waits/ui-change-helpers.js';
|
|
3
|
+
export function findScopedElement(tree, targetElementId, store) {
|
|
4
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
5
|
+
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
|
|
6
|
+
const deviceId = tree?.device?.id ?? undefined;
|
|
7
|
+
const normalizedTarget = _normalize(targetElementId);
|
|
8
|
+
for (let i = 0; i < elements.length; i++) {
|
|
9
|
+
const el = elements[i];
|
|
10
|
+
if (!el)
|
|
11
|
+
continue;
|
|
12
|
+
const computedElementId = _computeElementId(platform, deviceId, el, i);
|
|
13
|
+
if (computedElementId === targetElementId) {
|
|
14
|
+
return {
|
|
15
|
+
index: i,
|
|
16
|
+
stableId: el.stable_id ?? null,
|
|
17
|
+
reason: 'element_id_match'
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const directStable = findStableElement(elements, normalizedTarget, 'stable_id_match');
|
|
22
|
+
if (directStable)
|
|
23
|
+
return directStable;
|
|
24
|
+
const storedContext = store.get(targetElementId);
|
|
25
|
+
return storedContext?.stable_id
|
|
26
|
+
? findStableElement(elements, _normalize(storedContext.stable_id), 'stored_stable_id_match')
|
|
27
|
+
: null;
|
|
28
|
+
}
|
|
29
|
+
export function resolveUiChangeScope(tree, scope, target, find) {
|
|
30
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
31
|
+
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
|
|
32
|
+
if (normalizedScope === 'screen') {
|
|
33
|
+
return {
|
|
34
|
+
elements,
|
|
35
|
+
resolution: {
|
|
36
|
+
scope: 'screen',
|
|
37
|
+
target: null,
|
|
38
|
+
resolved: true,
|
|
39
|
+
resolvedIndex: null,
|
|
40
|
+
resolvedStableId: null,
|
|
41
|
+
reason: 'screen scope'
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
|
|
46
|
+
if (!requestedTarget) {
|
|
47
|
+
return {
|
|
48
|
+
elements: [],
|
|
49
|
+
resolution: {
|
|
50
|
+
scope: 'subtree',
|
|
51
|
+
target: null,
|
|
52
|
+
resolved: false,
|
|
53
|
+
resolvedIndex: null,
|
|
54
|
+
resolvedStableId: null,
|
|
55
|
+
reason: 'subtree scope requires a target element id'
|
|
56
|
+
},
|
|
57
|
+
error: {
|
|
58
|
+
code: 'INVALID_SCOPE',
|
|
59
|
+
message: 'scope=subtree requires a target element_id'
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
const resolved = find(tree, requestedTarget);
|
|
64
|
+
if (!resolved) {
|
|
65
|
+
return {
|
|
66
|
+
elements: [],
|
|
67
|
+
resolution: {
|
|
68
|
+
scope: 'subtree',
|
|
69
|
+
target: requestedTarget,
|
|
70
|
+
resolved: false,
|
|
71
|
+
resolvedIndex: null,
|
|
72
|
+
resolvedStableId: null,
|
|
73
|
+
reason: 'target element could not be resolved'
|
|
74
|
+
},
|
|
75
|
+
error: {
|
|
76
|
+
code: 'ELEMENT_NOT_FOUND',
|
|
77
|
+
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
const subtreeIndices = _collectSubtreeIndices(elements, resolved.index);
|
|
82
|
+
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
|
|
83
|
+
return {
|
|
84
|
+
elements: scopedElements,
|
|
85
|
+
resolution: {
|
|
86
|
+
scope: 'subtree',
|
|
87
|
+
target: requestedTarget,
|
|
88
|
+
resolved: true,
|
|
89
|
+
resolvedIndex: resolved.index,
|
|
90
|
+
resolvedStableId: resolved.stableId,
|
|
91
|
+
reason: resolved.reason
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
function findStableElement(elements, normalizedTarget, reason) {
|
|
96
|
+
for (let i = 0; i < elements.length; i++) {
|
|
97
|
+
const el = elements[i];
|
|
98
|
+
if (!el?.stable_id)
|
|
99
|
+
continue;
|
|
100
|
+
if (_normalize(el.stable_id) === normalizedTarget)
|
|
101
|
+
return { index: i, stableId: el.stable_id, reason };
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|