mobile-debug-mcp 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/mcp-builder/SKILL.md +44 -64
- package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
- package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
- package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
- package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
- package/.agents/skills/test-authoring/SKILL.md +7 -6
- package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
- package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
- package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
- package/.github/copilot-instructions.md +14 -36
- package/.github/workflows/ci.yml +23 -2
- package/AGENTS.md +12 -3
- package/CONTRIBUTING.md +79 -0
- package/README.md +158 -124
- package/agent-plugin/.claude-plugin/plugin.json +2 -2
- package/agent-plugin/.codex-plugin/plugin.json +1 -1
- package/agent-plugin/.cursor-plugin/plugin.json +2 -2
- package/agent-plugin/.mcp.json +1 -1
- package/agent-plugin/README.md +20 -6
- package/agent-plugin/mcp.json +1 -1
- package/agent-plugin/plugin.json +1 -1
- package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +113 -4
- package/dist/contracts/execution-controls.js +24 -0
- package/dist/contracts/selector.js +8 -0
- package/dist/contracts/target.js +15 -0
- package/dist/contracts/ui.js +1 -0
- package/dist/debugger/android.js +79 -0
- package/dist/debugger/diagnostics.js +54 -0
- package/dist/debugger/guard.js +30 -0
- package/dist/debugger/protocol.js +33 -0
- package/dist/debugger/sessions.js +304 -0
- package/dist/debugger/worker.js +176 -0
- package/dist/interact/adjustment/adjustment-context.js +41 -0
- package/dist/interact/adjustment/adjustment-gestures.js +55 -0
- package/dist/interact/adjustment/adjustment-plan.js +18 -0
- package/dist/interact/adjustment/adjustment-resolution.js +127 -0
- package/dist/interact/adjustment/adjustment-result.js +19 -0
- package/dist/interact/adjustment/adjustment-semantic.js +64 -0
- package/dist/interact/adjustment/adjustment-verification.js +35 -0
- package/dist/interact/adjustment/adjustment.js +52 -0
- package/dist/interact/adjustment/control-geometry.js +87 -0
- package/dist/interact/adjustment/control-values.js +136 -0
- package/dist/interact/adjustment/slider-labels.js +84 -0
- package/dist/interact/assertions/screen-assertion.js +37 -0
- package/dist/interact/assertions/state-assertion-context.js +35 -0
- package/dist/interact/assertions/state-assertion.js +76 -0
- package/dist/interact/assertions/state-comparison.js +141 -0
- package/dist/interact/assertions/visibility-assertion.js +34 -0
- package/dist/interact/conditional-action.js +12 -7
- package/dist/interact/index.js +65 -3074
- package/dist/interact/input-service.js +1 -0
- package/dist/interact/interaction-trace.js +21 -0
- package/dist/interact/model.js +1 -0
- package/dist/interact/observation.js +1 -0
- package/dist/interact/targeting/candidate-ancestor.js +58 -0
- package/dist/interact/targeting/candidate-result.js +23 -0
- package/dist/interact/targeting/candidate-scoring.js +50 -0
- package/dist/interact/targeting/element-helpers.js +216 -0
- package/dist/interact/targeting/find-element.js +65 -0
- package/dist/interact/targeting/nearby-control.js +56 -0
- package/dist/interact/targeting/resolved-elements.js +21 -0
- package/dist/interact/targeting/tap-target.js +17 -0
- package/dist/interact/targeting/target-scope.js +104 -0
- package/dist/interact/waits/legacy-log-wait.js +38 -0
- package/dist/interact/waits/legacy-wait-observations.js +103 -0
- package/dist/interact/waits/legacy-wait-state.js +34 -0
- package/dist/interact/waits/legacy-wait.js +48 -0
- package/dist/interact/waits/screen-wait.js +49 -0
- package/dist/interact/waits/ui-change-helpers.js +157 -0
- package/dist/interact/waits/ui-change-state.js +18 -0
- package/dist/interact/waits/ui-change-wait.js +77 -0
- package/dist/interact/waits/ui-wait-condition.js +56 -0
- package/dist/interact/waits/ui-wait-state.js +38 -0
- package/dist/interact/waits/ui-wait.js +106 -0
- package/dist/manage/android.js +55 -51
- package/dist/manage/index.js +38 -75
- package/dist/manage/ios/build-types.js +1 -0
- package/dist/manage/ios/build.js +210 -0
- package/dist/manage/ios/project-discovery.js +29 -0
- package/dist/manage/ios/scheme.js +18 -0
- package/dist/manage/ios.js +42 -256
- package/dist/observe/android/context.js +1 -0
- package/dist/observe/android/current-screen.js +55 -0
- package/dist/observe/android/fingerprint.js +13 -0
- package/dist/observe/android/log-stream.js +132 -0
- package/dist/observe/android/logs.js +107 -0
- package/dist/observe/android/screenshot.js +70 -0
- package/dist/observe/android/ui-tree.js +107 -0
- package/dist/observe/android.js +16 -520
- package/dist/observe/evidence.js +122 -76
- package/dist/observe/index.js +16 -263
- package/dist/observe/ios/context.js +16 -0
- package/dist/observe/ios/fingerprint.js +12 -0
- package/dist/observe/ios/log-stream.js +105 -0
- package/dist/observe/ios/logs.js +168 -0
- package/dist/observe/ios/screenshot.js +22 -0
- package/dist/observe/ios/ui-elements.js +241 -0
- package/dist/observe/ios/ui-tree.js +137 -0
- package/dist/observe/ios.js +17 -715
- package/dist/observe/logs/model.js +1 -0
- package/dist/{network → observe/network}/index.js +2 -2
- package/dist/observe/readiness-service.js +13 -2
- package/dist/observe/readiness.js +44 -18
- package/dist/observe/screenshot-encoding.js +41 -0
- package/dist/observe/snapshot-collection.js +79 -0
- package/dist/observe/snapshot-logs.js +30 -0
- package/dist/observe/snapshot-metadata.js +43 -31
- package/dist/observe/snapshot-semantic.js +115 -0
- package/dist/observe/system-trace-android.js +135 -0
- package/dist/observe/system-trace.js +294 -0
- package/dist/runtime/action-results.js +270 -0
- package/dist/runtime/clock.js +4 -0
- package/dist/server/common.js +1 -256
- package/dist/server/definitions/assert.js +162 -0
- package/dist/server/definitions/diagnostic.js +118 -0
- package/dist/server/definitions/input.js +274 -0
- package/dist/server/definitions/manage.js +194 -0
- package/dist/server/definitions/observe.js +149 -0
- package/dist/server/definitions/target.js +138 -0
- package/dist/server/definitions/wait.js +105 -0
- package/dist/server/handlers/action-execution.js +163 -0
- package/dist/server/handlers/correlation.js +23 -0
- package/dist/server/handlers/debugger-handlers.js +48 -0
- package/dist/server/handlers/debugger-response.js +9 -0
- package/dist/server/handlers/diagnostic.js +69 -0
- package/dist/server/handlers/handler-types.js +1 -0
- package/dist/server/handlers/input.js +154 -0
- package/dist/server/handlers/interact.js +113 -0
- package/dist/server/handlers/journey-assertions.js +35 -0
- package/dist/server/handlers/journey.js +147 -0
- package/dist/server/handlers/manage.js +119 -0
- package/dist/server/handlers/observe.js +95 -0
- package/dist/server/handlers/payload.js +4 -0
- package/dist/server/handlers/tool-handlers.js +160 -0
- package/dist/server/schemas/condition-schema.js +2 -0
- package/dist/server/schemas/debugger-schema.js +47 -0
- package/dist/server/{journey-schema.js → schemas/journey-schema.js} +8 -22
- package/dist/server/schemas/system-trace-schema.js +17 -0
- package/dist/server/tool-definitions.js +62 -1134
- package/dist/server-core.js +4 -4
- package/dist/server.js +34 -0
- package/dist/system/android-status.js +101 -0
- package/dist/system/android.js +37 -27
- package/dist/system/index.js +13 -438
- package/dist/system/ios-status.js +132 -0
- package/dist/system/ios.js +11 -8
- package/dist/system/model.js +1 -0
- package/dist/system/probes.js +105 -0
- package/dist/system/status-summary.js +43 -0
- package/dist/system/unsupported-status.js +52 -0
- package/dist/utils/android/commands.js +89 -0
- package/dist/utils/android/gradle.js +139 -0
- package/dist/utils/android/logs.js +103 -0
- package/dist/utils/android/ui-tree.js +233 -0
- package/dist/utils/android/utils.js +6 -564
- package/dist/utils/cli/idb/idb-helper.js +4 -6
- package/dist/utils/cli/idb/install-idb.js +13 -9
- package/dist/utils/cli/ios/preflight-ios.js +42 -38
- package/dist/utils/exec.js +2 -0
- package/dist/utils/ios/utils.js +45 -46
- package/dist/utils/java.js +75 -57
- package/dist/utils/operation-budget.js +2 -0
- package/dist/utils/resolve-device.js +6 -2
- package/dist/utils/ui/index.js +26 -26
- package/docs/CHANGELOG.md +48 -0
- package/docs/README.md +38 -0
- package/docs/benchmarks/README.md +18 -0
- package/docs/benchmarks/interaction-latency/README.md +14 -0
- package/docs/benchmarks/interaction-latency/final.md +4 -1
- package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
- package/docs/benchmarks/ui-tree-performance/README.md +24 -16
- package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
- package/docs/code-structure.md +118 -0
- package/docs/specs/001-state-verification/spec.md +16 -14
- package/docs/specs/002-richer-element-identity/spec.md +12 -10
- package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
- package/docs/specs/004-action-verification-routing/spec.md +3 -3
- package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
- package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
- package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
- package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
- package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
- package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
- package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
- package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
- package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
- package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
- package/docs/specs/014-actionability-resolution/spec.md +20 -18
- package/docs/specs/015-adjustable-control-support/spec.md +17 -15
- package/docs/specs/016-environment-tools/spec.md +20 -20
- package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
- package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
- package/docs/specs/020-conditional-actions/intent.md +5 -3
- package/docs/specs/020-conditional-actions/plan.md +8 -9
- package/docs/specs/020-conditional-actions/spec.md +4 -2
- package/docs/specs/021-app-readiness-signals/intent.md +4 -2
- package/docs/specs/021-app-readiness-signals/plan.md +7 -6
- package/docs/specs/021-app-readiness-signals/spec.md +3 -1
- package/docs/specs/022-correlated-evidence/intent.md +4 -4
- package/docs/specs/022-correlated-evidence/plan.md +10 -11
- package/docs/specs/022-correlated-evidence/spec.md +3 -1
- package/docs/specs/023-android-system-traces/device-evidence.json +38 -0
- package/docs/specs/023-android-system-traces/emulator-recheck.json +38 -0
- package/docs/specs/023-android-system-traces/intent.md +41 -0
- package/docs/specs/023-android-system-traces/plan.md +115 -0
- package/docs/specs/023-android-system-traces/spec.md +122 -0
- package/docs/specs/024-android-heap-dump-capture/intent.md +41 -0
- package/docs/specs/025-android-runtime-debugger/intent.md +41 -0
- package/docs/specs/025-android-runtime-debugger/plan.md +106 -0
- package/docs/specs/025-android-runtime-debugger/spec.md +183 -0
- package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
- package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
- package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
- package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
- package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
- package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
- package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
- package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
- package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
- package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
- package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
- package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
- package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
- package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
- package/docs/specs/030-android-coroutine-validation/report.md +240 -0
- package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
- package/docs/tools/TOOLS.md +31 -16
- package/docs/tools/{evidence.md → correlated-evidence.md} +14 -3
- package/docs/tools/debugger.md +180 -0
- package/docs/tools/interact.md +42 -20
- package/docs/tools/manage.md +25 -44
- package/docs/tools/observe.md +64 -25
- package/docs/tools/system-traces.md +134 -0
- package/docs/tools/system.md +12 -5
- package/package.json +11 -8
- package/scripts/architecture-graph.mjs +69 -0
- package/scripts/check-architecture.mjs +47 -0
- package/scripts/code-quality-baseline.json +1 -0
- package/scripts/code-quality.mjs +161 -0
- package/src/contracts/execution-controls.ts +26 -0
- package/src/contracts/selector.ts +17 -0
- package/src/contracts/target.ts +19 -0
- package/src/contracts/ui.ts +94 -0
- package/src/debugger/android.ts +66 -0
- package/src/debugger/diagnostics.ts +43 -0
- package/src/debugger/guard.ts +31 -0
- package/src/debugger/java/DebuggerWorker.java +166 -0
- package/src/debugger/protocol.ts +37 -0
- package/src/debugger/sessions.ts +327 -0
- package/src/debugger/worker.ts +135 -0
- package/src/interact/adjustment/adjustment-context.ts +83 -0
- package/src/interact/adjustment/adjustment-gestures.ts +48 -0
- package/src/interact/adjustment/adjustment-plan.ts +23 -0
- package/src/interact/adjustment/adjustment-resolution.ts +104 -0
- package/src/interact/adjustment/adjustment-result.ts +34 -0
- package/src/interact/adjustment/adjustment-semantic.ts +53 -0
- package/src/interact/adjustment/adjustment-verification.ts +60 -0
- package/src/interact/adjustment/adjustment.ts +49 -0
- package/src/interact/adjustment/control-geometry.ts +120 -0
- package/src/interact/adjustment/control-values.ts +141 -0
- package/src/interact/adjustment/slider-labels.ts +84 -0
- package/src/interact/assertions/screen-assertion.ts +39 -0
- package/src/interact/assertions/state-assertion-context.ts +48 -0
- package/src/interact/assertions/state-assertion.ts +64 -0
- package/src/interact/assertions/state-comparison.ts +168 -0
- package/src/interact/assertions/visibility-assertion.ts +41 -0
- package/src/interact/conditional-action.ts +18 -12
- package/src/interact/index.ts +76 -3491
- package/src/interact/input-service.ts +11 -0
- package/src/interact/interaction-trace.ts +36 -0
- package/src/interact/model.ts +109 -0
- package/src/interact/observation.ts +7 -0
- package/src/interact/targeting/candidate-ancestor.ts +49 -0
- package/src/interact/targeting/candidate-result.ts +24 -0
- package/src/interact/targeting/candidate-scoring.ts +42 -0
- package/src/interact/targeting/element-helpers.ts +243 -0
- package/src/interact/targeting/find-element.ts +47 -0
- package/src/interact/targeting/nearby-control.ts +46 -0
- package/src/interact/targeting/resolved-elements.ts +20 -0
- package/src/interact/targeting/tap-target.ts +22 -0
- package/src/interact/targeting/target-scope.ts +121 -0
- package/src/interact/waits/legacy-log-wait.ts +30 -0
- package/src/interact/waits/legacy-wait-observations.ts +55 -0
- package/src/interact/waits/legacy-wait-state.ts +48 -0
- package/src/interact/waits/legacy-wait.ts +34 -0
- package/src/interact/waits/screen-wait.ts +37 -0
- package/src/interact/waits/ui-change-helpers.ts +184 -0
- package/src/interact/waits/ui-change-state.ts +26 -0
- package/src/interact/waits/ui-change-wait.ts +59 -0
- package/src/interact/waits/ui-wait-condition.ts +52 -0
- package/src/interact/waits/ui-wait-state.ts +55 -0
- package/src/interact/waits/ui-wait.ts +84 -0
- package/src/manage/android.ts +56 -50
- package/src/manage/index.ts +63 -93
- package/src/manage/ios/build-types.ts +14 -0
- package/src/manage/ios/build.ts +230 -0
- package/src/manage/ios/project-discovery.ts +26 -0
- package/src/manage/ios/scheme.ts +16 -0
- package/src/manage/ios.ts +65 -298
- package/src/observe/android/context.ts +1 -0
- package/src/observe/android/current-screen.ts +63 -0
- package/src/observe/android/fingerprint.ts +13 -0
- package/src/observe/android/log-stream.ts +127 -0
- package/src/observe/android/logs.ts +110 -0
- package/src/observe/android/screenshot.ts +79 -0
- package/src/observe/android/ui-tree.ts +113 -0
- package/src/observe/android.ts +17 -531
- package/src/observe/evidence.ts +130 -71
- package/src/observe/index.ts +24 -286
- package/src/observe/ios/context.ts +22 -0
- package/src/observe/ios/fingerprint.ts +12 -0
- package/src/observe/ios/log-stream.ts +103 -0
- package/src/observe/ios/logs.ts +179 -0
- package/src/observe/ios/screenshot.ts +26 -0
- package/src/observe/ios/ui-elements.ts +266 -0
- package/src/observe/ios/ui-tree.ts +143 -0
- package/src/observe/ios.ts +18 -749
- package/src/observe/logs/model.ts +11 -0
- package/src/{network → observe/network}/index.ts +2 -2
- package/src/observe/readiness-service.ts +10 -3
- package/src/observe/readiness.ts +49 -22
- package/src/observe/screenshot-encoding.ts +32 -0
- package/src/observe/snapshot-collection.ts +69 -0
- package/src/observe/snapshot-logs.ts +26 -0
- package/src/observe/snapshot-metadata.ts +51 -31
- package/src/observe/snapshot-semantic.ts +153 -0
- package/src/observe/system-trace-android.ts +133 -0
- package/src/observe/system-trace.ts +227 -0
- package/src/runtime/action-results.ts +376 -0
- package/src/runtime/clock.ts +9 -0
- package/src/server/common.ts +9 -359
- package/src/server/definitions/assert.ts +164 -0
- package/src/server/definitions/diagnostic.ts +120 -0
- package/src/server/definitions/input.ts +276 -0
- package/src/server/definitions/manage.ts +196 -0
- package/src/server/definitions/observe.ts +151 -0
- package/src/server/definitions/target.ts +140 -0
- package/src/server/definitions/wait.ts +107 -0
- package/src/server/handlers/action-execution.ts +164 -0
- package/src/server/handlers/correlation.ts +20 -0
- package/src/server/handlers/debugger-handlers.ts +39 -0
- package/src/server/handlers/debugger-response.ts +7 -0
- package/src/server/handlers/diagnostic.ts +79 -0
- package/src/server/handlers/handler-types.ts +7 -0
- package/src/server/handlers/input.ts +173 -0
- package/src/server/handlers/interact.ts +135 -0
- package/src/server/handlers/journey-assertions.ts +39 -0
- package/src/server/handlers/journey.ts +132 -0
- package/src/server/handlers/manage.ts +141 -0
- package/src/server/handlers/observe.ts +116 -0
- package/src/server/handlers/payload.ts +36 -0
- package/src/server/handlers/tool-handlers.ts +159 -0
- package/src/server/schemas/condition-schema.ts +2 -0
- package/src/server/schemas/debugger-schema.ts +43 -0
- package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +8 -22
- package/src/server/schemas/system-trace-schema.ts +17 -0
- package/src/server/tool-definitions.ts +69 -1140
- package/src/server-core.ts +4 -4
- package/src/server.ts +25 -1
- package/src/system/android-status.ts +124 -0
- package/src/system/android.ts +33 -21
- package/src/system/index.ts +21 -523
- package/src/system/ios-status.ts +147 -0
- package/src/system/ios.ts +14 -10
- package/src/system/model.ts +41 -0
- package/src/system/probes.ts +114 -0
- package/src/system/status-summary.ts +47 -0
- package/src/system/unsupported-status.ts +55 -0
- package/src/types.ts +16 -106
- package/src/utils/android/commands.ts +81 -0
- package/src/utils/android/gradle.ts +132 -0
- package/src/utils/android/logs.ts +103 -0
- package/src/utils/android/ui-tree.ts +239 -0
- package/src/utils/android/utils.ts +16 -556
- package/src/utils/cli/idb/check-idb.ts +4 -4
- package/src/utils/cli/idb/idb-helper.ts +6 -8
- package/src/utils/cli/idb/install-idb.ts +15 -9
- package/src/utils/cli/ios/preflight-ios.ts +39 -31
- package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
- package/src/utils/exec.ts +2 -0
- package/src/utils/ios/utils.ts +57 -61
- package/src/utils/java.ts +71 -51
- package/src/utils/operation-budget.ts +2 -0
- package/src/utils/resolve-device.ts +11 -6
- package/src/utils/ui/index.ts +28 -28
- package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
- package/test/device/automated/debugger/coroutine-support.ts +140 -0
- package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
- package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
- package/test/device/automated/debugger/runtime.android.test.ts +219 -0
- package/test/device/automated/interact/interaction.android.test.ts +56 -0
- package/test/device/automated/observe/system-trace.smoke.ts +104 -0
- package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
- package/test/fixtures/contracts/tool-definitions.json +3536 -0
- package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
- package/test/fixtures/coroutine-app/README.md +48 -0
- package/test/fixtures/coroutine-app/build.ts +35 -0
- package/test/fixtures/debugger-app/MainActivity.java +67 -0
- package/test/fixtures/debugger-app/MainActivity.kt +32 -0
- package/test/fixtures/debugger-app/NoLocals.java +7 -0
- package/test/fixtures/debugger-app/README.md +65 -0
- package/test/fixtures/debugger-app/build.ts +39 -0
- package/test/fixtures/latency-journey-app/README.md +2 -0
- package/test/fixtures/readiness-app/README.md +2 -0
- package/test/helpers/perfetto.ts +59 -0
- package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
- package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
- package/test/unit/debugger/diagnostics.test.ts +80 -0
- package/test/unit/debugger/guards.test.ts +50 -0
- package/test/unit/debugger/native.test.ts +33 -0
- package/test/unit/debugger/preparation.test.ts +32 -0
- package/test/unit/debugger/protocol.test.ts +15 -0
- package/test/unit/debugger/sessions.test.ts +83 -0
- package/test/unit/interact/adjustment_service.test.ts +42 -0
- package/test/unit/interact/adjustment_verification.test.ts +35 -0
- package/test/unit/interact/clock_dependencies.test.ts +63 -0
- package/test/unit/interact/element_helpers.test.ts +19 -0
- package/test/unit/interact/legacy_wait_service.test.ts +17 -0
- package/test/unit/interact/state_comparison.test.ts +20 -0
- package/test/unit/interact/target_scope.test.ts +35 -0
- package/test/unit/network/get_network_activity.test.ts +1 -1
- package/test/unit/observe/readiness.test.ts +3 -1
- package/test/unit/observe/screenshot-encoding.test.ts +30 -0
- package/test/unit/observe/snapshot-logs.test.ts +14 -0
- package/test/unit/observe/system-trace-android.test.ts +52 -0
- package/test/unit/observe/system-trace.test.ts +166 -0
- package/test/unit/quality/architecture.test.ts +60 -0
- package/test/unit/quality/code-quality.test.ts +58 -0
- package/test/unit/quality/interaction_boundaries.test.ts +24 -0
- package/test/unit/server/conditional_actions.test.ts +1 -1
- package/test/unit/server/conditional_schema.test.ts +2 -2
- package/test/unit/server/debugger-response.test.ts +63 -0
- package/test/unit/server/system-trace.test.ts +64 -0
- package/test/unit/server/tool_definitions_parity.test.ts +8 -0
- package/dist/server/tool-handlers.js +0 -893
- package/docs/ROADMAP.md +0 -646
- package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
- package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
- package/docs/specs/baseline-spec-v0.md +0 -312
- package/docs/specs/mcp-tooling-spec-v1.md +0 -446
- package/src/server/tool-handlers.ts +0 -923
- /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
- /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
- /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
- /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { _normalizeBounds, _isAdjustableControl, _findFirstMatchingElement, _isAdjustableActionable } from '../targeting/element-helpers.js';
|
|
2
|
+
import { planAdjustment } from './adjustment-plan.js';
|
|
3
|
+
import { failAdjustment, recordAdjustment, succeedAdjustment } from './adjustment-context.js';
|
|
4
|
+
function fallbackBounds(fallback) {
|
|
5
|
+
const bounds = fallback?.bounds;
|
|
6
|
+
if (Array.isArray(bounds))
|
|
7
|
+
return _normalizeBounds(bounds);
|
|
8
|
+
return bounds && typeof bounds === 'object' ? _normalizeBounds([Number(bounds.left), Number(bounds.top), Number(bounds.right), Number(bounds.bottom)]) : null;
|
|
9
|
+
}
|
|
10
|
+
function findFallback(elements, fallback) {
|
|
11
|
+
const bounds = fallbackBounds(fallback);
|
|
12
|
+
if (!bounds)
|
|
13
|
+
return null;
|
|
14
|
+
let index = elements.findIndex(el => {
|
|
15
|
+
const current = _normalizeBounds(el.bounds);
|
|
16
|
+
return !!current && current.every((value, i) => value === bounds[i]);
|
|
17
|
+
});
|
|
18
|
+
if (index === -1)
|
|
19
|
+
index = nearestControl(elements, bounds);
|
|
20
|
+
return index >= 0 && elements[index] ? { el: elements[index], idx: index } : null;
|
|
21
|
+
}
|
|
22
|
+
function nearestControl(elements, bounds) {
|
|
23
|
+
const x = Math.floor((bounds[0] + bounds[2]) / 2), y = Math.floor((bounds[1] + bounds[3]) / 2);
|
|
24
|
+
let best = Infinity, index = -1;
|
|
25
|
+
for (let i = 0; i < elements.length; i++) {
|
|
26
|
+
const el = elements[i];
|
|
27
|
+
if (!_isAdjustableControl(el))
|
|
28
|
+
continue;
|
|
29
|
+
const current = _normalizeBounds(el.bounds);
|
|
30
|
+
if (!current)
|
|
31
|
+
continue;
|
|
32
|
+
const distance = Math.abs(Math.floor((current[0] + current[2]) / 2) - x) + Math.abs(Math.floor((current[1] + current[3]) / 2) - y);
|
|
33
|
+
if (distance < best) {
|
|
34
|
+
best = distance;
|
|
35
|
+
index = i;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return index;
|
|
39
|
+
}
|
|
40
|
+
export async function resolveControl(s, deps) {
|
|
41
|
+
const tree = await deps.observe(s.platform, s.deviceId);
|
|
42
|
+
s.platform = tree?.device?.platform === 'ios' ? 'ios' : s.platform;
|
|
43
|
+
s.deviceId = tree?.device?.id || s.deviceId;
|
|
44
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
45
|
+
let match = null;
|
|
46
|
+
if (s.input.element_id) {
|
|
47
|
+
const stored = deps.getHandle(s.input.element_id);
|
|
48
|
+
if (!stored)
|
|
49
|
+
return null;
|
|
50
|
+
const current = deps.currentHandle(elements, s.platform, s.deviceId, stored);
|
|
51
|
+
if (!current)
|
|
52
|
+
return null;
|
|
53
|
+
match = { el: current.el, idx: current.index };
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
if (s.fallback)
|
|
57
|
+
match = findFallback(elements, s.fallback);
|
|
58
|
+
if (!match && s.input.selector)
|
|
59
|
+
match = _findFirstMatchingElement(elements, s.input.selector);
|
|
60
|
+
}
|
|
61
|
+
return match ? { tree, device: tree?.device, match, resolvedTarget: deps.target(s.platform, s.deviceId, match) } : null;
|
|
62
|
+
}
|
|
63
|
+
export async function initializeControl(s, deps) {
|
|
64
|
+
const { selector, element_id } = s.input;
|
|
65
|
+
if ((!selector && !element_id) || (selector && element_id)) {
|
|
66
|
+
return failAdjustment(s, 'ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', false, null, undefined, null, 0, 'gesture');
|
|
67
|
+
}
|
|
68
|
+
if (!selector || element_id)
|
|
69
|
+
return null;
|
|
70
|
+
const wait = await deps.wait({ selector, condition: 'clickable', timeout_ms: 5000, poll_interval_ms: 300, platform: s.platform, deviceId: s.deviceId });
|
|
71
|
+
if (wait?.status === 'success' && wait?.element?.elementId) {
|
|
72
|
+
s.input.element_id = wait.element.elementId;
|
|
73
|
+
s.fallback = null;
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
return initializeFallback(s, deps, wait, selector);
|
|
77
|
+
}
|
|
78
|
+
async function initializeFallback(s, deps, wait, selector) {
|
|
79
|
+
const query = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? '';
|
|
80
|
+
if (!query)
|
|
81
|
+
return failAdjustment(s, wait?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT', wait?.error?.message ?? 'adjustable control not found', wait?.error?.code === 'ELEMENT_NOT_FOUND', null, wait?.device, null, 0, 'gesture');
|
|
82
|
+
const fallback = await deps.find({ query, exact: false, timeoutMs: 3000, platform: s.platform, deviceId: s.deviceId });
|
|
83
|
+
if (!fallback.found || !fallback.element)
|
|
84
|
+
return failAdjustment(s, 'ELEMENT_NOT_FOUND', wait?.error?.message ?? 'adjustable control not found', true, null, wait?.device, null, 0, 'gesture');
|
|
85
|
+
s.fallback = fallback.element;
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
export async function prepareIteration(s, deps) {
|
|
89
|
+
const resolved = s.cached ? { tree: null, device: s.device, match: s.cached, resolvedTarget: deps.target(s.platform, s.deviceId, s.cached) } : await resolveControl(s, deps);
|
|
90
|
+
if (!resolved?.match || !resolved.resolvedTarget)
|
|
91
|
+
return failAdjustment(s, 'STALE_REFERENCE', 'adjustable control could not be resolved', true);
|
|
92
|
+
s.device = resolved.device;
|
|
93
|
+
s.target = resolved.resolvedTarget;
|
|
94
|
+
s.cached = resolved.match;
|
|
95
|
+
const element = resolved.match.el;
|
|
96
|
+
const actionable = _isAdjustableActionable(element, s.stored?.stable_id);
|
|
97
|
+
if (!actionable.actionable)
|
|
98
|
+
return failAdjustment(s, actionable.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', actionable.reason ?? 'adjustable control is not actionable', true);
|
|
99
|
+
const elements = Array.isArray(resolved.tree?.elements) ? resolved.tree.elements : s.cached ? [s.cached.el] : [element];
|
|
100
|
+
const plan = planAdjustment(element, elements, s.input.targetValue, s.input.property, s.tolerance);
|
|
101
|
+
s.observed = plan.actualState;
|
|
102
|
+
if (!s.trace.some(step => step.stage === 'resolve'))
|
|
103
|
+
recordAdjustment(s, 'resolve', 'success', { resolved_target: s.target, current_value: plan.currentValue, adjustment_mode: s.mode });
|
|
104
|
+
const iteration = { ...plan, element, index: resolved.match.idx, actionDevice: s.device, observed: plan.actualState,
|
|
105
|
+
verification: { verification: null, observedState: plan.actualState, withinTolerance: false } };
|
|
106
|
+
const invalid = validateControl(s, iteration);
|
|
107
|
+
if (invalid)
|
|
108
|
+
return invalid;
|
|
109
|
+
if (plan.currentValue !== null && Math.abs(plan.currentValue - plan.normalizedTargetValue) <= plan.effectiveTolerance) {
|
|
110
|
+
recordAdjustment(s, 'verify', 'success', { property: s.input.property, target_value: plan.normalizedTargetValue, actual_state: plan.actualState, reason: 'control already within tolerance' });
|
|
111
|
+
return succeedAdjustment(s, iteration, deps, 'control already within tolerance', 'semantic', s.device);
|
|
112
|
+
}
|
|
113
|
+
if (!plan.bounds)
|
|
114
|
+
return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds');
|
|
115
|
+
return iteration;
|
|
116
|
+
}
|
|
117
|
+
function validateControl(s, p) {
|
|
118
|
+
if (s.input.property !== 'value' && s.input.property !== 'raw_value')
|
|
119
|
+
return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only');
|
|
120
|
+
if (!_isAdjustableControl(p.element) && !p.inferredSliderState)
|
|
121
|
+
return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control');
|
|
122
|
+
if (p.currentValue === null)
|
|
123
|
+
return failAdjustment(s, 'UNKNOWN', 'current numeric value is unavailable');
|
|
124
|
+
if (p.valueRange && (s.input.targetValue < p.valueRange.min || s.input.targetValue > p.valueRange.max))
|
|
125
|
+
return failAdjustment(s, 'CONTROL_CONVERGENCE_FAILED', `targetValue ${s.input.targetValue} is outside the control range ${p.valueRange.min}..${p.valueRange.max}`);
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { buildActionExecutionResult } from '../../runtime/action-results.js';
|
|
2
|
+
/** Keep the action envelope and top-level control result in agreement. */
|
|
3
|
+
export function completeAdjustment(input) {
|
|
4
|
+
const base = buildActionExecutionResult(input);
|
|
5
|
+
const details = input.details;
|
|
6
|
+
return {
|
|
7
|
+
...base,
|
|
8
|
+
target_state: {
|
|
9
|
+
property: details.property,
|
|
10
|
+
target_value: details.target_value,
|
|
11
|
+
tolerance: details.tolerance
|
|
12
|
+
},
|
|
13
|
+
actual_state: details.actual_state,
|
|
14
|
+
within_tolerance: true,
|
|
15
|
+
converged: true,
|
|
16
|
+
attempts: details.attempts,
|
|
17
|
+
adjustment_mode: details.adjustment_mode
|
|
18
|
+
};
|
|
19
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { _hasDirectAdjustmentAction, _adjustmentDirections } from './control-values.js';
|
|
2
|
+
import { verifyAdjustment } from './adjustment-verification.js';
|
|
3
|
+
import { recordAdjustment, succeedAdjustment } from './adjustment-context.js';
|
|
4
|
+
export async function verifyControl(s, p, deps, fallback) {
|
|
5
|
+
p.verification = await verifyAdjustment({ labeled: !!p.inferredSliderState, currentEl: p.element, actualState: p.actualState,
|
|
6
|
+
property: s.input.property, target: p.normalizedTargetValue, tolerance: p.effectiveTolerance,
|
|
7
|
+
expectation: { element_id: s.target?.elementId ?? s.input.element_id, selector: s.input.selector ?? undefined,
|
|
8
|
+
property: s.input.property, expected: p.normalizedTargetValue, platform: s.platform, deviceId: s.deviceId }
|
|
9
|
+
}, { observe: () => deps.observe(s.platform, s.deviceId), expectState: deps.expectState });
|
|
10
|
+
p.observed = p.verification.observedState;
|
|
11
|
+
s.observed = p.observed;
|
|
12
|
+
recordAdjustment(s, 'verify', p.verification.withinTolerance ? 'success' : 'retry', {
|
|
13
|
+
attempt: s.attempts, property: s.input.property, target_value: p.normalizedTargetValue,
|
|
14
|
+
actual_state: p.observed, reason: p.verification.verification?.reason ?? fallback
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
export async function directAdjustment(s, p, deps) {
|
|
18
|
+
if (!_hasDirectAdjustmentAction(p.element) || s.attempts >= s.limit)
|
|
19
|
+
return 'next';
|
|
20
|
+
const { interact, resolved } = await deps.service(s.platform, s.deviceId);
|
|
21
|
+
if (typeof interact.setAdjustableValue !== 'function')
|
|
22
|
+
return 'next';
|
|
23
|
+
s.mode = 'semantic';
|
|
24
|
+
recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts + 1, mode: 'semantic', strategy: 'direct_set_value', target_value: p.normalizedTargetValue });
|
|
25
|
+
const action = await interact.setAdjustableValue({ deviceId: resolved.id, target: s.target, property: s.input.property, value: p.normalizedTargetValue });
|
|
26
|
+
s.attempts++;
|
|
27
|
+
p.actionDevice = action?.device ?? p.actionDevice;
|
|
28
|
+
if (action?.success === false)
|
|
29
|
+
return 'next';
|
|
30
|
+
await verifyControl(s, p, deps, 'direct adjustment did not converge yet');
|
|
31
|
+
if (p.verification.withinTolerance)
|
|
32
|
+
return succeedAdjustment(s, p, deps, p.verification.verification?.reason ?? 'control converged to target value');
|
|
33
|
+
s.cached = null;
|
|
34
|
+
return s.attempts < s.limit ? 'retry' : 'next';
|
|
35
|
+
}
|
|
36
|
+
export async function incrementalAdjustment(s, p, deps) {
|
|
37
|
+
const directions = _adjustmentDirections(p.element);
|
|
38
|
+
if ((!directions.increment && !directions.decrement) || s.attempts >= s.limit)
|
|
39
|
+
return 'next';
|
|
40
|
+
const { interact, resolved } = await deps.service(s.platform, s.deviceId);
|
|
41
|
+
const adjust = typeof interact.adjustAccessibleValue === 'function' ? interact.adjustAccessibleValue.bind(interact) : null;
|
|
42
|
+
while (adjust && s.attempts < s.limit) {
|
|
43
|
+
const direction = p.normalizedTargetValue > (p.observed?.value ?? p.currentValue) ? 'increment' : 'decrement';
|
|
44
|
+
if (!directionAllowed(direction, directions))
|
|
45
|
+
break;
|
|
46
|
+
s.mode = 'semantic';
|
|
47
|
+
recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts + 1, mode: 'semantic', strategy: 'accessibility_increment_decrement', direction });
|
|
48
|
+
const action = await adjust({ deviceId: resolved.id, target: s.target, property: s.input.property, direction });
|
|
49
|
+
s.attempts++;
|
|
50
|
+
p.actionDevice = action?.device ?? p.actionDevice;
|
|
51
|
+
if (action?.success === false)
|
|
52
|
+
break;
|
|
53
|
+
await verifyControl(s, p, deps, 'incremental adjustment did not converge yet');
|
|
54
|
+
if (p.verification.withinTolerance)
|
|
55
|
+
return succeedAdjustment(s, p, deps, p.verification.verification?.reason ?? 'control converged to target value');
|
|
56
|
+
s.cached = null;
|
|
57
|
+
if (s.attempts < s.limit)
|
|
58
|
+
return 'retry';
|
|
59
|
+
}
|
|
60
|
+
return 'next';
|
|
61
|
+
}
|
|
62
|
+
function directionAllowed(direction, directions) {
|
|
63
|
+
return !((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement));
|
|
64
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { _inferLabeledSliderState } from './slider-labels.js';
|
|
2
|
+
import { _parseNumericControlValue } from './control-values.js';
|
|
3
|
+
/** Every verification obtains fresh evidence through its supplied observation boundary. */
|
|
4
|
+
export async function verifyAdjustment(input, dependencies) {
|
|
5
|
+
if (input.labeled)
|
|
6
|
+
return verifyLabeledControl(input, dependencies);
|
|
7
|
+
const verification = await dependencies.expectState(input.expectation);
|
|
8
|
+
const observedValue = _parseNumericControlValue(verification?.observed_state?.value) ??
|
|
9
|
+
_parseNumericControlValue(verification?.observed_state?.raw_value);
|
|
10
|
+
const observedState = observedValue !== null ? {
|
|
11
|
+
property: input.property, value: observedValue,
|
|
12
|
+
raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
|
|
13
|
+
} : input.actualState;
|
|
14
|
+
return { verification, observedState, withinTolerance: withinTolerance(observedValue, input) };
|
|
15
|
+
}
|
|
16
|
+
async function verifyLabeledControl(input, dependencies) {
|
|
17
|
+
const tree = await dependencies.observe();
|
|
18
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
19
|
+
const refreshed = _inferLabeledSliderState(elements, input.currentEl);
|
|
20
|
+
const observedValue = refreshed?.current ?? null;
|
|
21
|
+
const observedState = observedValue !== null ? {
|
|
22
|
+
property: input.property, value: observedValue, raw_value: observedValue
|
|
23
|
+
} : input.actualState;
|
|
24
|
+
const matched = withinTolerance(observedValue, input);
|
|
25
|
+
return {
|
|
26
|
+
verification: {
|
|
27
|
+
success: matched, observed_state: observedState,
|
|
28
|
+
reason: observedValue !== null ? 'visible labeled value read from nearby slider labels' : 'visible labeled value unavailable'
|
|
29
|
+
},
|
|
30
|
+
observedState, withinTolerance: matched
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function withinTolerance(value, input) {
|
|
34
|
+
return value !== null && Math.abs(value - input.target) <= input.tolerance;
|
|
35
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { createAdjustment, failAdjustment, recordAdjustment, succeedAdjustment } from './adjustment-context.js';
|
|
2
|
+
import { initializeControl, prepareIteration } from './adjustment-resolution.js';
|
|
3
|
+
import { directAdjustment, incrementalAdjustment } from './adjustment-semantic.js';
|
|
4
|
+
import { coordinateAdjustment, gestureAdjustment } from './adjustment-gestures.js';
|
|
5
|
+
function cacheObservation(s, p) {
|
|
6
|
+
s.cached = p.inferredSliderState ? null : {
|
|
7
|
+
el: { ...p.element, state: { ...(p.element.state ?? null), ...(p.observed ? {
|
|
8
|
+
[p.observed.property]: p.observed.value, raw_value: p.observed.raw_value ?? p.observed.value
|
|
9
|
+
} : {}) } }, idx: p.index
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
async function attempt(s, p, deps) {
|
|
13
|
+
for (const strategy of [directAdjustment, incrementalAdjustment]) {
|
|
14
|
+
const outcome = await strategy(s, p, deps);
|
|
15
|
+
if (outcome !== 'next')
|
|
16
|
+
return outcome;
|
|
17
|
+
}
|
|
18
|
+
if (!p.bounds)
|
|
19
|
+
return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds');
|
|
20
|
+
if (!p.valueRange)
|
|
21
|
+
return failAdjustment(s, 'UNKNOWN', 'value_range unavailable');
|
|
22
|
+
for (const strategy of [coordinateAdjustment, gestureAdjustment]) {
|
|
23
|
+
const outcome = await strategy(s, p, deps);
|
|
24
|
+
if (outcome !== 'next')
|
|
25
|
+
return outcome;
|
|
26
|
+
}
|
|
27
|
+
s.observed = p.observed;
|
|
28
|
+
if (p.verification.withinTolerance) {
|
|
29
|
+
const reason = p.verification.verification?.reason ?? 'control converged to target value';
|
|
30
|
+
recordAdjustment(s, 'verify', 'success', { attempt: s.attempts, property: s.input.property,
|
|
31
|
+
target_value: p.normalizedTargetValue, actual_state: p.observed, reason });
|
|
32
|
+
return succeedAdjustment(s, p, deps, reason);
|
|
33
|
+
}
|
|
34
|
+
cacheObservation(s, p);
|
|
35
|
+
return 'next';
|
|
36
|
+
}
|
|
37
|
+
export async function adjustControl(input, deps) {
|
|
38
|
+
const state = await createAdjustment(input, deps);
|
|
39
|
+
const invalid = await initializeControl(state, deps);
|
|
40
|
+
if (invalid)
|
|
41
|
+
return invalid;
|
|
42
|
+
for (let index = 0; index < state.limit && state.attempts < state.limit; index++) {
|
|
43
|
+
const prepared = await prepareIteration(state, deps);
|
|
44
|
+
if ('success' in prepared)
|
|
45
|
+
return prepared;
|
|
46
|
+
const outcome = await attempt(state, prepared, deps);
|
|
47
|
+
if (outcome !== 'retry' && outcome !== 'next')
|
|
48
|
+
return outcome;
|
|
49
|
+
}
|
|
50
|
+
const after = await deps.fingerprint(state.platform, state.deviceId);
|
|
51
|
+
return failAdjustment(state, 'CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', true, state.target, state.device, state.observed, state.attempts, state.mode, after);
|
|
52
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { _normalize } from '../targeting/element-helpers.js';
|
|
2
|
+
export function _buildControlPoint(bounds, ratio, axis) {
|
|
3
|
+
const clampedRatio = Math.max(0, Math.min(1, ratio));
|
|
4
|
+
const [left, top, right, bottom] = bounds;
|
|
5
|
+
const width = Math.max(1, right - left);
|
|
6
|
+
const height = Math.max(1, bottom - top);
|
|
7
|
+
const insetX = Math.max(8, Math.floor(width * 0.08));
|
|
8
|
+
const insetY = Math.max(8, Math.floor(height * 0.08));
|
|
9
|
+
if (axis === 'vertical') {
|
|
10
|
+
const usableHeight = Math.max(1, height - (insetY * 2));
|
|
11
|
+
return {
|
|
12
|
+
x: Math.floor((left + right) / 2),
|
|
13
|
+
y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
const usableWidth = Math.max(1, width - (insetX * 2));
|
|
17
|
+
return {
|
|
18
|
+
x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
|
|
19
|
+
y: Math.floor((top + bottom) / 2)
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export function _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis) {
|
|
23
|
+
const range = Math.max(1, max - min);
|
|
24
|
+
const targetRatio = (targetValue - min) / range;
|
|
25
|
+
const stepRatio = 1 / range;
|
|
26
|
+
const centerBias = stepRatio / 2;
|
|
27
|
+
const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue);
|
|
28
|
+
const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0]);
|
|
29
|
+
const edgeWindow = Math.max(3, Math.floor(range * 0.1));
|
|
30
|
+
const isNearLowEdge = targetValue - min <= edgeWindow;
|
|
31
|
+
const isNearHighEdge = max - targetValue <= edgeWindow;
|
|
32
|
+
const directionBias = direction > 0
|
|
33
|
+
? -stepRatio * 0.15
|
|
34
|
+
: direction < 0
|
|
35
|
+
? stepRatio * 0.65
|
|
36
|
+
: 0;
|
|
37
|
+
const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx));
|
|
38
|
+
const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin);
|
|
39
|
+
const edgeBias = isNearLowEdge
|
|
40
|
+
? endpointMargin
|
|
41
|
+
: isNearHighEdge
|
|
42
|
+
? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
|
|
43
|
+
: 0;
|
|
44
|
+
const safeRatio = Math.min(1 - (endpointMargin * 0.25), Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias));
|
|
45
|
+
return _buildControlPoint(bounds, safeRatio, axis);
|
|
46
|
+
}
|
|
47
|
+
export function _buildAdjustmentProbePoints(bounds, targetValue, currentValue, min, max, axis) {
|
|
48
|
+
const targetPoint = _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis);
|
|
49
|
+
const currentPoint = currentValue !== null
|
|
50
|
+
? _buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
|
|
51
|
+
: _buildControlPoint(bounds, 0.5, axis);
|
|
52
|
+
const [left, top, right, bottom] = bounds;
|
|
53
|
+
const width = Math.max(1, right - left);
|
|
54
|
+
const height = Math.max(1, bottom - top);
|
|
55
|
+
const crossAxisBumps = axis === 'horizontal'
|
|
56
|
+
? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
|
|
57
|
+
: [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))];
|
|
58
|
+
const clampPoint = (point) => ({
|
|
59
|
+
x: axis === 'horizontal'
|
|
60
|
+
? Math.max(left, Math.min(right, point.x))
|
|
61
|
+
: Math.max(left, Math.min(right + Math.max(width, height), point.x)),
|
|
62
|
+
y: axis === 'vertical'
|
|
63
|
+
? Math.max(top, Math.min(bottom, point.y))
|
|
64
|
+
: Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
|
|
65
|
+
});
|
|
66
|
+
const probes = [targetPoint, currentPoint];
|
|
67
|
+
for (const bump of crossAxisBumps) {
|
|
68
|
+
if (axis === 'horizontal') {
|
|
69
|
+
probes.push({ x: targetPoint.x, y: bottom + bump }, { x: currentPoint.x, y: bottom + bump });
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
probes.push({ x: right + bump, y: targetPoint.y }, { x: right + bump, y: currentPoint.y });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return Array.from(new Map(probes
|
|
76
|
+
.map(clampPoint)
|
|
77
|
+
.map((point) => [`${point.x}:${point.y}`, point])).values());
|
|
78
|
+
}
|
|
79
|
+
export function _controlAxis(el, bounds) {
|
|
80
|
+
const type = _normalize(el.type ?? el.class ?? '');
|
|
81
|
+
const role = _normalize(el.role ?? '');
|
|
82
|
+
if (/vertical/.test(type) || /vertical/.test(role))
|
|
83
|
+
return 'vertical';
|
|
84
|
+
if (/horizontal/.test(type) || /horizontal/.test(role))
|
|
85
|
+
return 'horizontal';
|
|
86
|
+
return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal';
|
|
87
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { _normalize } from '../targeting/element-helpers.js';
|
|
2
|
+
export function _readNumericControlValue(el, property) {
|
|
3
|
+
if (!el?.state)
|
|
4
|
+
return null;
|
|
5
|
+
const stateValue = el.state[property];
|
|
6
|
+
if (typeof stateValue === 'number' && Number.isFinite(stateValue))
|
|
7
|
+
return stateValue;
|
|
8
|
+
const parsedStateValue = _parseNumericControlValue(stateValue);
|
|
9
|
+
if (parsedStateValue !== null)
|
|
10
|
+
return parsedStateValue;
|
|
11
|
+
if (property === 'value' || property === 'raw_value') {
|
|
12
|
+
const fallback = el.state.raw_value ?? el.state.value;
|
|
13
|
+
if (typeof fallback === 'number' && Number.isFinite(fallback))
|
|
14
|
+
return fallback;
|
|
15
|
+
return _parseNumericControlValue(fallback);
|
|
16
|
+
}
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
export function _parseNumericControlValue(value) {
|
|
20
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
21
|
+
return value;
|
|
22
|
+
if (typeof value !== 'string')
|
|
23
|
+
return null;
|
|
24
|
+
const trimmed = value.trim();
|
|
25
|
+
if (!trimmed)
|
|
26
|
+
return null;
|
|
27
|
+
const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/);
|
|
28
|
+
if (percent)
|
|
29
|
+
return Number(percent[1]);
|
|
30
|
+
const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i);
|
|
31
|
+
if (fraction)
|
|
32
|
+
return Number(fraction[1]);
|
|
33
|
+
const plain = Number(trimmed);
|
|
34
|
+
if (Number.isFinite(plain))
|
|
35
|
+
return plain;
|
|
36
|
+
const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/);
|
|
37
|
+
if (trailing)
|
|
38
|
+
return Number(trailing[1]);
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
export function _readNumericControlRange(el) {
|
|
42
|
+
const range = el?.state?.value_range;
|
|
43
|
+
const min = typeof range?.min === 'number' ? range.min : null;
|
|
44
|
+
const max = typeof range?.max === 'number' ? range.max : null;
|
|
45
|
+
if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
|
|
46
|
+
return { min, max };
|
|
47
|
+
}
|
|
48
|
+
return rangeFromFraction(el?.state?.value ?? el?.state?.raw_value);
|
|
49
|
+
}
|
|
50
|
+
function rangeFromFraction(value) {
|
|
51
|
+
if (typeof value === 'string') {
|
|
52
|
+
const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i);
|
|
53
|
+
if (fraction) {
|
|
54
|
+
const parsedMax = Number(fraction[1]);
|
|
55
|
+
if (Number.isFinite(parsedMax) && parsedMax > 0)
|
|
56
|
+
return { min: 0, max: parsedMax };
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
export function _readControlStep(el) {
|
|
62
|
+
const candidates = [
|
|
63
|
+
el?.state?.step,
|
|
64
|
+
el?.state?.value_range?.step
|
|
65
|
+
];
|
|
66
|
+
for (const candidate of candidates) {
|
|
67
|
+
if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0)
|
|
68
|
+
return candidate;
|
|
69
|
+
}
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
export function _quantizeTargetValue(targetValue, min, max, step) {
|
|
73
|
+
if (!step)
|
|
74
|
+
return targetValue;
|
|
75
|
+
const quantized = min + (Math.round((targetValue - min) / step) * step);
|
|
76
|
+
const bounded = Math.max(min, Math.min(max, quantized));
|
|
77
|
+
const precision = _decimalPrecisionForStep(step);
|
|
78
|
+
return Number(bounded.toFixed(precision));
|
|
79
|
+
}
|
|
80
|
+
export function _decimalPrecisionForStep(step) {
|
|
81
|
+
const normalized = String(step).toLowerCase();
|
|
82
|
+
const [coefficient, exponentPart] = normalized.split('e');
|
|
83
|
+
const coefficientDecimals = coefficient.includes('.')
|
|
84
|
+
? coefficient.split('.')[1]?.length ?? 0
|
|
85
|
+
: 0;
|
|
86
|
+
const exponent = exponentPart !== undefined ? Number(exponentPart) : 0;
|
|
87
|
+
const precision = exponent < 0
|
|
88
|
+
? coefficientDecimals + Math.abs(exponent)
|
|
89
|
+
: Math.max(0, coefficientDecimals - exponent);
|
|
90
|
+
return Math.max(0, Math.min(12, precision));
|
|
91
|
+
}
|
|
92
|
+
export function _defaultAdjustTolerance(range, step) {
|
|
93
|
+
if (step)
|
|
94
|
+
return 0;
|
|
95
|
+
if (!range)
|
|
96
|
+
return 0;
|
|
97
|
+
return Math.max(0, 0.01 * Math.abs(range.max - range.min));
|
|
98
|
+
}
|
|
99
|
+
export function _runtimeActionSet(el) {
|
|
100
|
+
const values = [
|
|
101
|
+
...(Array.isArray(el?.actions) ? el.actions : []),
|
|
102
|
+
...(Array.isArray(el?.accessibilityActions) ? el.accessibilityActions : []),
|
|
103
|
+
...(Array.isArray(el?.runtimeActions) ? el.runtimeActions : []),
|
|
104
|
+
...(Array.isArray(el?.platformActions) ? el.platformActions : [])
|
|
105
|
+
];
|
|
106
|
+
return new Set(values.map((value) => _normalize(value)).filter(Boolean));
|
|
107
|
+
}
|
|
108
|
+
export function _hasDirectAdjustmentAction(el) {
|
|
109
|
+
const actions = _runtimeActionSet(el);
|
|
110
|
+
return [
|
|
111
|
+
'action_set_progress',
|
|
112
|
+
'set_progress',
|
|
113
|
+
'setprogress',
|
|
114
|
+
'semanticsactions.setprogress',
|
|
115
|
+
'set_value',
|
|
116
|
+
'setvalue',
|
|
117
|
+
'direct_set_value'
|
|
118
|
+
].some((action) => actions.has(action));
|
|
119
|
+
}
|
|
120
|
+
export function _adjustmentDirections(el) {
|
|
121
|
+
const actions = _runtimeActionSet(el);
|
|
122
|
+
return {
|
|
123
|
+
increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
|
|
124
|
+
decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
export function _textFromElement(el) {
|
|
128
|
+
return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim();
|
|
129
|
+
}
|
|
130
|
+
export function _parseLeadingNumber(text) {
|
|
131
|
+
const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/);
|
|
132
|
+
if (!match)
|
|
133
|
+
return null;
|
|
134
|
+
const value = Number(match[1]);
|
|
135
|
+
return Number.isFinite(value) ? value : null;
|
|
136
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { _normalize, _normalizeBounds } from '../targeting/element-helpers.js';
|
|
2
|
+
import { _textFromElement, _parseLeadingNumber } from './control-values.js';
|
|
3
|
+
export function _inferLabeledSliderState(elements, track) {
|
|
4
|
+
const trackBounds = _normalizeBounds(track?.bounds);
|
|
5
|
+
if (!track || !trackBounds)
|
|
6
|
+
return null;
|
|
7
|
+
const type = _normalize(track.type ?? track.class ?? '');
|
|
8
|
+
const role = _normalize(track.role ?? '');
|
|
9
|
+
const width = trackBounds[2] - trackBounds[0];
|
|
10
|
+
const height = trackBounds[3] - trackBounds[1];
|
|
11
|
+
const wideTrackLike = width >= 240 && height <= 220;
|
|
12
|
+
const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role);
|
|
13
|
+
if (!wideTrackLike && !knownTrackLike)
|
|
14
|
+
return null;
|
|
15
|
+
const trackTop = trackBounds[1];
|
|
16
|
+
const trackBottom = trackBounds[3];
|
|
17
|
+
const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2;
|
|
18
|
+
const labels = elements
|
|
19
|
+
.map((el, index) => {
|
|
20
|
+
const bounds = _normalizeBounds(el.bounds);
|
|
21
|
+
const text = _textFromElement(el);
|
|
22
|
+
const value = _parseLeadingNumber(text);
|
|
23
|
+
return bounds && text && value !== null
|
|
24
|
+
? {
|
|
25
|
+
index,
|
|
26
|
+
el,
|
|
27
|
+
text,
|
|
28
|
+
value,
|
|
29
|
+
bounds,
|
|
30
|
+
centerX: (bounds[0] + bounds[2]) / 2,
|
|
31
|
+
centerY: (bounds[1] + bounds[3]) / 2,
|
|
32
|
+
hasUnit: /[a-zA-Z%]/.test(text)
|
|
33
|
+
}
|
|
34
|
+
: null;
|
|
35
|
+
})
|
|
36
|
+
.filter((label) => !!label);
|
|
37
|
+
const rangeLabels = labels
|
|
38
|
+
.filter((label) => {
|
|
39
|
+
const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260;
|
|
40
|
+
const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120;
|
|
41
|
+
return verticallyNearTrack && horizontallyNearTrack && label.hasUnit;
|
|
42
|
+
})
|
|
43
|
+
.sort((a, b) => a.centerX - b.centerX);
|
|
44
|
+
if (rangeLabels.length < 2)
|
|
45
|
+
return null;
|
|
46
|
+
const minLabel = rangeLabels[0];
|
|
47
|
+
const maxLabel = rangeLabels[rangeLabels.length - 1];
|
|
48
|
+
if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value)
|
|
49
|
+
return null;
|
|
50
|
+
const currentCandidates = labels
|
|
51
|
+
.filter((label) => {
|
|
52
|
+
if (label.index === minLabel.index || label.index === maxLabel.index)
|
|
53
|
+
return false;
|
|
54
|
+
if (label.value < minLabel.value || label.value > maxLabel.value)
|
|
55
|
+
return false;
|
|
56
|
+
if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20)
|
|
57
|
+
return false;
|
|
58
|
+
return label.centerX >= trackCenterX - 80;
|
|
59
|
+
})
|
|
60
|
+
.sort((a, b) => {
|
|
61
|
+
const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY);
|
|
62
|
+
if (yDistance !== 0)
|
|
63
|
+
return yDistance;
|
|
64
|
+
return b.centerX - a.centerX;
|
|
65
|
+
});
|
|
66
|
+
const currentLabel = currentCandidates[0];
|
|
67
|
+
if (!currentLabel)
|
|
68
|
+
return null;
|
|
69
|
+
const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop;
|
|
70
|
+
const labelDerivedBounds = [
|
|
71
|
+
Math.min(minLabel.bounds[0], trackBounds[0]),
|
|
72
|
+
Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
|
|
73
|
+
Math.max(maxLabel.bounds[2], trackBounds[2]),
|
|
74
|
+
Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
|
|
75
|
+
];
|
|
76
|
+
return {
|
|
77
|
+
current: currentLabel.value,
|
|
78
|
+
min: minLabel.value,
|
|
79
|
+
max: maxLabel.value,
|
|
80
|
+
value_range: { min: minLabel.value, max: maxLabel.value },
|
|
81
|
+
touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
|
|
82
|
+
source: 'nearby_labels'
|
|
83
|
+
};
|
|
84
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { buildObservationTrace } from '../interaction-trace.js';
|
|
2
|
+
function compareScreen(input, observed, label) {
|
|
3
|
+
const { fingerprint, screen } = input;
|
|
4
|
+
if (fingerprint) {
|
|
5
|
+
const success = observed.fingerprint === fingerprint;
|
|
6
|
+
return { success, basis: 'fingerprint', reason: success ? `observed fingerprint matches expected fingerprint ${fingerprint}` : `expected fingerprint ${fingerprint} but observed ${observed.fingerprint ?? 'null'}` };
|
|
7
|
+
}
|
|
8
|
+
if (screen) {
|
|
9
|
+
const candidates = new Set();
|
|
10
|
+
if (observed.screen)
|
|
11
|
+
candidates.add(observed.screen);
|
|
12
|
+
if (label)
|
|
13
|
+
candidates.add(label);
|
|
14
|
+
const success = candidates.has(screen);
|
|
15
|
+
return { success, basis: 'screen', reason: success ? `observed screen matches expected screen ${screen}` : `expected screen ${screen} but observed ${label ?? observed.screen ?? 'null'}` };
|
|
16
|
+
}
|
|
17
|
+
return { success: false, basis: 'none', reason: 'No fingerprint or screen expectation provided' };
|
|
18
|
+
}
|
|
19
|
+
export async function expectScreen(input, deps) {
|
|
20
|
+
const observed = await deps.fingerprint({ platform: input.platform, deviceId: input.deviceId });
|
|
21
|
+
const screen = { fingerprint: observed?.fingerprint ?? null, screen: observed?.activity ?? null };
|
|
22
|
+
let label = screen.screen;
|
|
23
|
+
if (!input.fingerprint && input.screen && input.platform !== 'ios') {
|
|
24
|
+
try {
|
|
25
|
+
const current = await deps.current({ deviceId: input.deviceId });
|
|
26
|
+
label = current?.shortActivity || current?.activity || label;
|
|
27
|
+
}
|
|
28
|
+
catch { /* Keep the fingerprint-derived activity if current-screen lookup is unavailable. */ }
|
|
29
|
+
}
|
|
30
|
+
const comparison = compareScreen(input, screen, label);
|
|
31
|
+
const expected = { fingerprint: input.fingerprint ?? null, screen: input.screen ?? null };
|
|
32
|
+
const observedScreen = { fingerprint: screen.fingerprint, screen: label };
|
|
33
|
+
return { success: comparison.success, observed_screen: observedScreen, expected_screen: expected,
|
|
34
|
+
confidence: comparison.success ? 1 : 0, comparison: { basis: comparison.basis, matched: comparison.success, reason: comparison.reason },
|
|
35
|
+
trace: buildObservationTrace({ actionType: 'expect_screen', stage: 'verify', success: comparison.success, attempts: 1,
|
|
36
|
+
metadata: { expected_screen: expected, observed_screen: observedScreen, comparison: { basis: comparison.basis, matched: comparison.success } } }) };
|
|
37
|
+
}
|