mobile-debug-mcp 0.34.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/mcp-builder/SKILL.md +44 -64
- package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
- package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
- package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
- package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
- package/.agents/skills/test-authoring/SKILL.md +7 -6
- package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
- package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
- package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
- package/.github/copilot-instructions.md +14 -36
- package/.github/workflows/ci.yml +4 -4
- package/AGENTS.md +12 -3
- package/CONTRIBUTING.md +79 -0
- package/README.md +156 -129
- package/agent-plugin/.claude-plugin/plugin.json +2 -2
- package/agent-plugin/.codex-plugin/plugin.json +1 -1
- package/agent-plugin/.cursor-plugin/plugin.json +2 -2
- package/agent-plugin/.mcp.json +1 -1
- package/agent-plugin/README.md +16 -6
- package/agent-plugin/mcp.json +1 -1
- package/agent-plugin/plugin.json +1 -1
- package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +27 -2
- package/dist/contracts/execution-controls.js +24 -0
- package/dist/contracts/selector.js +8 -0
- package/dist/contracts/target.js +15 -0
- package/dist/contracts/ui.js +1 -0
- package/dist/debugger/android.js +2 -1
- package/dist/debugger/diagnostics.js +54 -0
- package/dist/debugger/protocol.js +33 -0
- package/dist/debugger/sessions.js +104 -62
- package/dist/debugger/worker.js +53 -42
- package/dist/interact/adjustment/adjustment-context.js +41 -0
- package/dist/interact/adjustment/adjustment-gestures.js +55 -0
- package/dist/interact/adjustment/adjustment-plan.js +18 -0
- package/dist/interact/adjustment/adjustment-resolution.js +127 -0
- package/dist/interact/adjustment/adjustment-result.js +19 -0
- package/dist/interact/adjustment/adjustment-semantic.js +64 -0
- package/dist/interact/adjustment/adjustment-verification.js +35 -0
- package/dist/interact/adjustment/adjustment.js +52 -0
- package/dist/interact/adjustment/control-geometry.js +87 -0
- package/dist/interact/adjustment/control-values.js +136 -0
- package/dist/interact/adjustment/slider-labels.js +84 -0
- package/dist/interact/assertions/screen-assertion.js +37 -0
- package/dist/interact/assertions/state-assertion-context.js +35 -0
- package/dist/interact/assertions/state-assertion.js +76 -0
- package/dist/interact/assertions/state-comparison.js +141 -0
- package/dist/interact/assertions/visibility-assertion.js +34 -0
- package/dist/interact/conditional-action.js +12 -7
- package/dist/interact/index.js +65 -3090
- package/dist/interact/input-service.js +1 -0
- package/dist/interact/interaction-trace.js +21 -0
- package/dist/interact/model.js +1 -0
- package/dist/interact/observation.js +1 -0
- package/dist/interact/targeting/candidate-ancestor.js +58 -0
- package/dist/interact/targeting/candidate-result.js +23 -0
- package/dist/interact/targeting/candidate-scoring.js +50 -0
- package/dist/interact/targeting/element-helpers.js +216 -0
- package/dist/interact/targeting/find-element.js +65 -0
- package/dist/interact/targeting/nearby-control.js +56 -0
- package/dist/interact/targeting/resolved-elements.js +21 -0
- package/dist/interact/targeting/tap-target.js +17 -0
- package/dist/interact/targeting/target-scope.js +104 -0
- package/dist/interact/waits/legacy-log-wait.js +38 -0
- package/dist/interact/waits/legacy-wait-observations.js +103 -0
- package/dist/interact/waits/legacy-wait-state.js +34 -0
- package/dist/interact/waits/legacy-wait.js +48 -0
- package/dist/interact/waits/screen-wait.js +49 -0
- package/dist/interact/waits/ui-change-helpers.js +157 -0
- package/dist/interact/waits/ui-change-state.js +18 -0
- package/dist/interact/waits/ui-change-wait.js +77 -0
- package/dist/interact/waits/ui-wait-condition.js +56 -0
- package/dist/interact/waits/ui-wait-state.js +38 -0
- package/dist/interact/waits/ui-wait.js +106 -0
- package/dist/manage/android.js +55 -51
- package/dist/manage/index.js +38 -75
- package/dist/manage/ios/build-types.js +1 -0
- package/dist/manage/ios/build.js +210 -0
- package/dist/manage/ios/project-discovery.js +29 -0
- package/dist/manage/ios/scheme.js +18 -0
- package/dist/manage/ios.js +42 -256
- package/dist/observe/android/context.js +1 -0
- package/dist/observe/android/current-screen.js +55 -0
- package/dist/observe/android/fingerprint.js +13 -0
- package/dist/observe/android/log-stream.js +132 -0
- package/dist/observe/android/logs.js +107 -0
- package/dist/observe/android/screenshot.js +70 -0
- package/dist/observe/android/ui-tree.js +107 -0
- package/dist/observe/android.js +16 -520
- package/dist/observe/evidence.js +114 -75
- package/dist/observe/index.js +16 -263
- package/dist/observe/ios/context.js +16 -0
- package/dist/observe/ios/fingerprint.js +12 -0
- package/dist/observe/ios/log-stream.js +105 -0
- package/dist/observe/ios/logs.js +168 -0
- package/dist/observe/ios/screenshot.js +22 -0
- package/dist/observe/ios/ui-elements.js +241 -0
- package/dist/observe/ios/ui-tree.js +137 -0
- package/dist/observe/ios.js +17 -715
- package/dist/observe/logs/model.js +1 -0
- package/dist/{network → observe/network}/index.js +2 -2
- package/dist/observe/readiness-service.js +4 -2
- package/dist/observe/readiness.js +29 -10
- package/dist/observe/screenshot-encoding.js +41 -0
- package/dist/observe/snapshot-collection.js +79 -0
- package/dist/observe/snapshot-logs.js +30 -0
- package/dist/observe/snapshot-metadata.js +43 -31
- package/dist/observe/snapshot-semantic.js +115 -0
- package/dist/observe/system-trace.js +22 -13
- package/dist/runtime/action-results.js +270 -0
- package/dist/runtime/clock.js +4 -0
- package/dist/server/common.js +1 -256
- package/dist/server/definitions/assert.js +162 -0
- package/dist/server/definitions/diagnostic.js +118 -0
- package/dist/server/definitions/input.js +274 -0
- package/dist/server/definitions/manage.js +194 -0
- package/dist/server/definitions/observe.js +149 -0
- package/dist/server/definitions/target.js +138 -0
- package/dist/server/definitions/wait.js +105 -0
- package/dist/server/handlers/action-execution.js +163 -0
- package/dist/server/handlers/correlation.js +23 -0
- package/dist/server/{debugger-handlers.js → handlers/debugger-handlers.js} +22 -12
- package/dist/server/handlers/debugger-response.js +9 -0
- package/dist/server/handlers/diagnostic.js +69 -0
- package/dist/server/handlers/handler-types.js +1 -0
- package/dist/server/handlers/input.js +154 -0
- package/dist/server/handlers/interact.js +113 -0
- package/dist/server/handlers/journey-assertions.js +35 -0
- package/dist/server/handlers/journey.js +147 -0
- package/dist/server/handlers/manage.js +119 -0
- package/dist/server/handlers/observe.js +95 -0
- package/dist/server/handlers/payload.js +4 -0
- package/dist/server/handlers/tool-handlers.js +160 -0
- package/dist/server/schemas/condition-schema.js +2 -0
- package/dist/server/{debugger-schema.js → schemas/debugger-schema.js} +11 -6
- package/dist/server/{journey-schema.js → schemas/journey-schema.js} +7 -22
- package/dist/server/tool-definitions.js +58 -1135
- package/dist/server-core.js +2 -2
- package/dist/system/android-status.js +101 -0
- package/dist/system/android.js +37 -27
- package/dist/system/index.js +13 -439
- package/dist/system/ios-status.js +132 -0
- package/dist/system/ios.js +11 -8
- package/dist/system/model.js +1 -0
- package/dist/system/probes.js +105 -0
- package/dist/system/status-summary.js +43 -0
- package/dist/system/unsupported-status.js +52 -0
- package/dist/utils/android/commands.js +89 -0
- package/dist/utils/android/gradle.js +139 -0
- package/dist/utils/android/logs.js +103 -0
- package/dist/utils/android/ui-tree.js +233 -0
- package/dist/utils/android/utils.js +6 -564
- package/dist/utils/cli/idb/idb-helper.js +4 -6
- package/dist/utils/cli/idb/install-idb.js +13 -9
- package/dist/utils/cli/ios/preflight-ios.js +42 -38
- package/dist/utils/ios/utils.js +45 -46
- package/dist/utils/java.js +65 -49
- package/dist/utils/resolve-device.js +4 -2
- package/dist/utils/ui/index.js +26 -26
- package/docs/CHANGELOG.md +44 -0
- package/docs/README.md +38 -0
- package/docs/benchmarks/README.md +18 -0
- package/docs/benchmarks/interaction-latency/README.md +14 -0
- package/docs/benchmarks/interaction-latency/final.md +4 -1
- package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
- package/docs/benchmarks/ui-tree-performance/README.md +24 -16
- package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
- package/docs/code-structure.md +118 -0
- package/docs/specs/001-state-verification/spec.md +16 -14
- package/docs/specs/002-richer-element-identity/spec.md +12 -10
- package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
- package/docs/specs/004-action-verification-routing/spec.md +3 -3
- package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
- package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
- package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
- package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
- package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
- package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
- package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
- package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
- package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
- package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
- package/docs/specs/014-actionability-resolution/spec.md +20 -18
- package/docs/specs/015-adjustable-control-support/spec.md +17 -15
- package/docs/specs/016-environment-tools/spec.md +20 -20
- package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
- package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
- package/docs/specs/020-conditional-actions/intent.md +5 -3
- package/docs/specs/020-conditional-actions/plan.md +8 -9
- package/docs/specs/020-conditional-actions/spec.md +4 -2
- package/docs/specs/021-app-readiness-signals/intent.md +4 -2
- package/docs/specs/021-app-readiness-signals/plan.md +7 -6
- package/docs/specs/021-app-readiness-signals/spec.md +3 -1
- package/docs/specs/022-correlated-evidence/intent.md +4 -4
- package/docs/specs/022-correlated-evidence/plan.md +10 -11
- package/docs/specs/022-correlated-evidence/spec.md +3 -1
- package/docs/specs/023-android-system-traces/intent.md +3 -2
- package/docs/specs/023-android-system-traces/plan.md +5 -7
- package/docs/specs/023-android-system-traces/spec.md +3 -1
- package/docs/specs/024-android-heap-dump-capture/intent.md +3 -1
- package/docs/specs/025-android-runtime-debugger/intent.md +4 -3
- package/docs/specs/025-android-runtime-debugger/plan.md +3 -1
- package/docs/specs/025-android-runtime-debugger/spec.md +4 -3
- package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
- package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
- package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
- package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
- package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
- package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
- package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
- package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
- package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
- package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
- package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
- package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
- package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
- package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
- package/docs/specs/030-android-coroutine-validation/report.md +240 -0
- package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
- package/docs/tools/TOOLS.md +31 -18
- package/docs/tools/{evidence.md → correlated-evidence.md} +9 -1
- package/docs/tools/debugger.md +76 -0
- package/docs/tools/interact.md +42 -20
- package/docs/tools/manage.md +25 -44
- package/docs/tools/observe.md +55 -23
- package/docs/tools/system-traces.md +9 -1
- package/docs/tools/system.md +12 -5
- package/package.json +11 -8
- package/scripts/architecture-graph.mjs +69 -0
- package/scripts/check-architecture.mjs +47 -0
- package/scripts/code-quality-baseline.json +1 -0
- package/scripts/code-quality.mjs +161 -0
- package/src/contracts/execution-controls.ts +26 -0
- package/src/contracts/selector.ts +17 -0
- package/src/contracts/target.ts +19 -0
- package/src/contracts/ui.ts +94 -0
- package/src/debugger/android.ts +2 -1
- package/src/debugger/diagnostics.ts +43 -0
- package/src/debugger/guard.ts +6 -1
- package/{worker → src/debugger/java}/DebuggerWorker.java +9 -3
- package/src/debugger/protocol.ts +37 -0
- package/src/debugger/sessions.ts +314 -155
- package/src/debugger/worker.ts +51 -38
- package/src/interact/adjustment/adjustment-context.ts +83 -0
- package/src/interact/adjustment/adjustment-gestures.ts +48 -0
- package/src/interact/adjustment/adjustment-plan.ts +23 -0
- package/src/interact/adjustment/adjustment-resolution.ts +104 -0
- package/src/interact/adjustment/adjustment-result.ts +34 -0
- package/src/interact/adjustment/adjustment-semantic.ts +53 -0
- package/src/interact/adjustment/adjustment-verification.ts +60 -0
- package/src/interact/adjustment/adjustment.ts +49 -0
- package/src/interact/adjustment/control-geometry.ts +120 -0
- package/src/interact/adjustment/control-values.ts +141 -0
- package/src/interact/adjustment/slider-labels.ts +84 -0
- package/src/interact/assertions/screen-assertion.ts +39 -0
- package/src/interact/assertions/state-assertion-context.ts +48 -0
- package/src/interact/assertions/state-assertion.ts +64 -0
- package/src/interact/assertions/state-comparison.ts +168 -0
- package/src/interact/assertions/visibility-assertion.ts +41 -0
- package/src/interact/conditional-action.ts +18 -12
- package/src/interact/index.ts +76 -3492
- package/src/interact/input-service.ts +11 -0
- package/src/interact/interaction-trace.ts +36 -0
- package/src/interact/model.ts +109 -0
- package/src/interact/observation.ts +7 -0
- package/src/interact/targeting/candidate-ancestor.ts +49 -0
- package/src/interact/targeting/candidate-result.ts +24 -0
- package/src/interact/targeting/candidate-scoring.ts +42 -0
- package/src/interact/targeting/element-helpers.ts +243 -0
- package/src/interact/targeting/find-element.ts +47 -0
- package/src/interact/targeting/nearby-control.ts +46 -0
- package/src/interact/targeting/resolved-elements.ts +20 -0
- package/src/interact/targeting/tap-target.ts +22 -0
- package/src/interact/targeting/target-scope.ts +121 -0
- package/src/interact/waits/legacy-log-wait.ts +30 -0
- package/src/interact/waits/legacy-wait-observations.ts +55 -0
- package/src/interact/waits/legacy-wait-state.ts +48 -0
- package/src/interact/waits/legacy-wait.ts +34 -0
- package/src/interact/waits/screen-wait.ts +37 -0
- package/src/interact/waits/ui-change-helpers.ts +184 -0
- package/src/interact/waits/ui-change-state.ts +26 -0
- package/src/interact/waits/ui-change-wait.ts +59 -0
- package/src/interact/waits/ui-wait-condition.ts +52 -0
- package/src/interact/waits/ui-wait-state.ts +55 -0
- package/src/interact/waits/ui-wait.ts +84 -0
- package/src/manage/android.ts +56 -50
- package/src/manage/index.ts +63 -93
- package/src/manage/ios/build-types.ts +14 -0
- package/src/manage/ios/build.ts +230 -0
- package/src/manage/ios/project-discovery.ts +26 -0
- package/src/manage/ios/scheme.ts +16 -0
- package/src/manage/ios.ts +65 -298
- package/src/observe/android/context.ts +1 -0
- package/src/observe/android/current-screen.ts +63 -0
- package/src/observe/android/fingerprint.ts +13 -0
- package/src/observe/android/log-stream.ts +127 -0
- package/src/observe/android/logs.ts +110 -0
- package/src/observe/android/screenshot.ts +79 -0
- package/src/observe/android/ui-tree.ts +113 -0
- package/src/observe/android.ts +17 -531
- package/src/observe/evidence.ts +122 -70
- package/src/observe/index.ts +24 -286
- package/src/observe/ios/context.ts +22 -0
- package/src/observe/ios/fingerprint.ts +12 -0
- package/src/observe/ios/log-stream.ts +103 -0
- package/src/observe/ios/logs.ts +179 -0
- package/src/observe/ios/screenshot.ts +26 -0
- package/src/observe/ios/ui-elements.ts +266 -0
- package/src/observe/ios/ui-tree.ts +143 -0
- package/src/observe/ios.ts +18 -749
- package/src/observe/logs/model.ts +11 -0
- package/src/{network → observe/network}/index.ts +2 -2
- package/src/observe/readiness-service.ts +5 -3
- package/src/observe/readiness.ts +39 -18
- package/src/observe/screenshot-encoding.ts +32 -0
- package/src/observe/snapshot-collection.ts +69 -0
- package/src/observe/snapshot-logs.ts +26 -0
- package/src/observe/snapshot-metadata.ts +51 -31
- package/src/observe/snapshot-semantic.ts +153 -0
- package/src/observe/system-trace.ts +24 -15
- package/src/runtime/action-results.ts +376 -0
- package/src/runtime/clock.ts +9 -0
- package/src/server/common.ts +9 -359
- package/src/server/definitions/assert.ts +164 -0
- package/src/server/definitions/diagnostic.ts +120 -0
- package/src/server/definitions/input.ts +276 -0
- package/src/server/definitions/manage.ts +196 -0
- package/src/server/definitions/observe.ts +151 -0
- package/src/server/definitions/target.ts +140 -0
- package/src/server/definitions/wait.ts +107 -0
- package/src/server/handlers/action-execution.ts +164 -0
- package/src/server/handlers/correlation.ts +20 -0
- package/src/server/handlers/debugger-handlers.ts +39 -0
- package/src/server/handlers/debugger-response.ts +7 -0
- package/src/server/handlers/diagnostic.ts +79 -0
- package/src/server/handlers/handler-types.ts +7 -0
- package/src/server/handlers/input.ts +173 -0
- package/src/server/handlers/interact.ts +135 -0
- package/src/server/handlers/journey-assertions.ts +39 -0
- package/src/server/handlers/journey.ts +132 -0
- package/src/server/handlers/manage.ts +141 -0
- package/src/server/handlers/observe.ts +116 -0
- package/src/server/handlers/payload.ts +36 -0
- package/src/server/handlers/tool-handlers.ts +159 -0
- package/src/server/schemas/condition-schema.ts +2 -0
- package/src/server/{debugger-schema.ts → schemas/debugger-schema.ts} +13 -7
- package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +7 -22
- package/src/server/{system-trace-schema.ts → schemas/system-trace-schema.ts} +3 -3
- package/src/server/tool-definitions.ts +67 -1143
- package/src/server-core.ts +2 -2
- package/src/server.ts +1 -1
- package/src/system/android-status.ts +124 -0
- package/src/system/android.ts +33 -21
- package/src/system/index.ts +21 -525
- package/src/system/ios-status.ts +147 -0
- package/src/system/ios.ts +14 -10
- package/src/system/model.ts +41 -0
- package/src/system/probes.ts +114 -0
- package/src/system/status-summary.ts +47 -0
- package/src/system/unsupported-status.ts +55 -0
- package/src/types.ts +16 -106
- package/src/utils/android/commands.ts +81 -0
- package/src/utils/android/gradle.ts +132 -0
- package/src/utils/android/logs.ts +103 -0
- package/src/utils/android/ui-tree.ts +239 -0
- package/src/utils/android/utils.ts +16 -556
- package/src/utils/cli/idb/check-idb.ts +4 -4
- package/src/utils/cli/idb/idb-helper.ts +6 -8
- package/src/utils/cli/idb/install-idb.ts +15 -9
- package/src/utils/cli/ios/preflight-ios.ts +39 -31
- package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
- package/src/utils/ios/utils.ts +57 -61
- package/src/utils/java.ts +62 -43
- package/src/utils/resolve-device.ts +9 -6
- package/src/utils/ui/index.ts +28 -28
- package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
- package/test/device/automated/debugger/coroutine-support.ts +140 -0
- package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
- package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
- package/test/device/automated/debugger/runtime.android.test.ts +19 -3
- package/test/device/automated/interact/interaction.android.test.ts +56 -0
- package/test/device/automated/observe/system-trace.smoke.ts +1 -1
- package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
- package/test/fixtures/contracts/tool-definitions.json +3536 -0
- package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
- package/test/fixtures/coroutine-app/README.md +48 -0
- package/test/fixtures/coroutine-app/build.ts +35 -0
- package/test/fixtures/debugger-app/MainActivity.java +21 -0
- package/test/fixtures/debugger-app/MainActivity.kt +32 -0
- package/test/fixtures/debugger-app/README.md +52 -0
- package/test/fixtures/debugger-app/build.ts +25 -15
- package/test/fixtures/latency-journey-app/README.md +2 -0
- package/test/fixtures/readiness-app/README.md +2 -0
- package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
- package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
- package/test/unit/debugger/diagnostics.test.ts +80 -0
- package/test/unit/debugger/guards.test.ts +4 -0
- package/test/unit/debugger/preparation.test.ts +32 -0
- package/test/unit/debugger/protocol.test.ts +15 -0
- package/test/unit/debugger/sessions.test.ts +1 -1
- package/test/unit/interact/adjustment_service.test.ts +42 -0
- package/test/unit/interact/adjustment_verification.test.ts +35 -0
- package/test/unit/interact/clock_dependencies.test.ts +63 -0
- package/test/unit/interact/element_helpers.test.ts +19 -0
- package/test/unit/interact/legacy_wait_service.test.ts +17 -0
- package/test/unit/interact/state_comparison.test.ts +20 -0
- package/test/unit/interact/target_scope.test.ts +35 -0
- package/test/unit/network/get_network_activity.test.ts +1 -1
- package/test/unit/observe/screenshot-encoding.test.ts +30 -0
- package/test/unit/observe/snapshot-logs.test.ts +14 -0
- package/test/unit/quality/architecture.test.ts +60 -0
- package/test/unit/quality/code-quality.test.ts +58 -0
- package/test/unit/quality/interaction_boundaries.test.ts +24 -0
- package/test/unit/server/conditional_actions.test.ts +1 -1
- package/test/unit/server/conditional_schema.test.ts +2 -2
- package/test/unit/server/debugger-response.test.ts +63 -0
- package/test/unit/server/tool_definitions_parity.test.ts +8 -0
- package/dist/server/tool-handlers.js +0 -1000
- package/docs/ROADMAP.md +0 -646
- package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
- package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
- package/docs/specs/baseline-spec-v0.md +0 -312
- package/docs/specs/mcp-tooling-spec-v1.md +0 -446
- package/src/server/debugger-handlers.ts +0 -29
- package/src/server/tool-handlers.ts +0 -1008
- /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
- /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
- /package/dist/server/{system-trace-schema.js → schemas/system-trace-schema.js} +0 -0
- /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
- /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
package/dist/interact/index.js
CHANGED
|
@@ -1,180 +1,29 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { resolveTap } from './targeting/tap-target.js';
|
|
2
|
+
import { waitForUICore } from './waits/legacy-wait.js';
|
|
3
|
+
import { expectState } from './assertions/state-assertion.js';
|
|
4
|
+
import { waitForUIChange } from './waits/ui-change-wait.js';
|
|
5
|
+
import { waitForScreenChange } from './waits/screen-wait.js';
|
|
6
|
+
import { expectScreen } from './assertions/screen-assertion.js';
|
|
7
|
+
import { expectVisible } from './assertions/visibility-assertion.js';
|
|
8
|
+
import { findElement } from './targeting/find-element.js';
|
|
9
|
+
import { waitForUI } from './waits/ui-wait.js';
|
|
10
|
+
import { adjustControl } from './adjustment/adjustment.js';
|
|
11
|
+
import { ResolvedElements } from './targeting/resolved-elements.js';
|
|
12
|
+
import { findScopedElement, resolveUiChangeScope } from './targeting/target-scope.js';
|
|
13
|
+
import { _normalizeBounds, _computeElementId, _resolvedTargetFromElement } from './targeting/element-helpers.js';
|
|
14
|
+
import { _collectSubtreeIndices } from './waits/ui-change-helpers.js';
|
|
4
15
|
import { isConditionBlocked } from './conditional-action.js';
|
|
5
|
-
import { remainingBudget } from '../utils/operation-budget.js';
|
|
6
|
-
import { createHash } from 'crypto';
|
|
7
16
|
import { AndroidInteract } from './android.js';
|
|
8
17
|
import { iOSInteract } from './ios.js';
|
|
9
18
|
export { AndroidInteract, iOSInteract };
|
|
10
19
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
11
20
|
import { ToolsObserve } from '../observe/index.js';
|
|
12
|
-
import {
|
|
13
|
-
import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js';
|
|
14
|
-
function buildObservationTrace({ actionType, stage, success, attempts, metadata }) {
|
|
15
|
-
const now = Date.now();
|
|
16
|
-
const actionId = nextActionId(actionType, now);
|
|
17
|
-
const steps = [
|
|
18
|
-
createTraceStep({
|
|
19
|
-
stage,
|
|
20
|
-
timestamp: now,
|
|
21
|
-
result: success ? 'success' : 'failure',
|
|
22
|
-
attemptIndex: 0,
|
|
23
|
-
metadata
|
|
24
|
-
})
|
|
25
|
-
];
|
|
26
|
-
return {
|
|
27
|
-
action_id: actionId,
|
|
28
|
-
steps,
|
|
29
|
-
final_outcome: success ? 'success' : 'failure',
|
|
30
|
-
attempts: Math.max(1, Math.floor(attempts || 1))
|
|
31
|
-
};
|
|
32
|
-
}
|
|
21
|
+
import { buildActionExecutionResult } from '../runtime/action-results.js';
|
|
33
22
|
export class ToolsInteract {
|
|
34
23
|
static _maxResolvedUiElements = 256;
|
|
35
|
-
static
|
|
36
|
-
static
|
|
37
|
-
static
|
|
38
|
-
static _sliderPositiveGapLimitPx = 640;
|
|
39
|
-
static _sliderTrackMinLengthPx = 220;
|
|
40
|
-
static _sliderTrackMaxThicknessPx = 180;
|
|
41
|
-
static _sliderTrackLengthRatio = 0.18;
|
|
42
|
-
static _sliderTrackThicknessRatio = 0.08;
|
|
43
|
-
static _sliderLabelWidthRatio = 1.5;
|
|
44
|
-
static _resolvedUiElements = new Map();
|
|
45
|
-
static _normalize(s) {
|
|
46
|
-
if (s === null || s === undefined)
|
|
47
|
-
return '';
|
|
48
|
-
try {
|
|
49
|
-
return String(s).toLowerCase().trim();
|
|
50
|
-
}
|
|
51
|
-
catch {
|
|
52
|
-
return '';
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
static _normalizeBounds(bounds) {
|
|
56
|
-
if (!Array.isArray(bounds) || bounds.length < 4)
|
|
57
|
-
return null;
|
|
58
|
-
const normalized = bounds.slice(0, 4).map((value) => Number(value));
|
|
59
|
-
if (normalized.some((value) => Number.isNaN(value)))
|
|
60
|
-
return null;
|
|
61
|
-
return normalized;
|
|
62
|
-
}
|
|
63
|
-
static _hash(value) {
|
|
64
|
-
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
65
|
-
}
|
|
66
|
-
static _matchesSelector(el, selector) {
|
|
67
|
-
if (!selector)
|
|
68
|
-
return false;
|
|
69
|
-
const normalize = ToolsInteract._normalize;
|
|
70
|
-
const containsFlag = !!selector.contains;
|
|
71
|
-
const text = normalize(el.text ?? el.label ?? el.value ?? '');
|
|
72
|
-
const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
|
|
73
|
-
const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '');
|
|
74
|
-
if (selector.text !== undefined && selector.text !== null) {
|
|
75
|
-
const q = normalize(selector.text);
|
|
76
|
-
if (containsFlag ? !text.includes(q) : text !== q)
|
|
77
|
-
return false;
|
|
78
|
-
}
|
|
79
|
-
if (selector.resource_id !== undefined && selector.resource_id !== null) {
|
|
80
|
-
const q = normalize(selector.resource_id);
|
|
81
|
-
if (containsFlag ? !resourceId.includes(q) : resourceId !== q)
|
|
82
|
-
return false;
|
|
83
|
-
}
|
|
84
|
-
if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
|
|
85
|
-
const q = normalize(selector.accessibility_id);
|
|
86
|
-
if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q)
|
|
87
|
-
return false;
|
|
88
|
-
}
|
|
89
|
-
return true;
|
|
90
|
-
}
|
|
91
|
-
static _findFirstMatchingElement(elements, selector) {
|
|
92
|
-
if (!selector)
|
|
93
|
-
return null;
|
|
94
|
-
for (let i = 0; i < elements.length; i++) {
|
|
95
|
-
const el = elements[i];
|
|
96
|
-
if (!el)
|
|
97
|
-
continue;
|
|
98
|
-
if (ToolsInteract._matchesSelector(el, selector))
|
|
99
|
-
return { el, idx: i };
|
|
100
|
-
}
|
|
101
|
-
return null;
|
|
102
|
-
}
|
|
103
|
-
static _resolveParentIndex(elements, parentId) {
|
|
104
|
-
if (parentId === undefined || parentId === null)
|
|
105
|
-
return null;
|
|
106
|
-
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
107
|
-
return parentId;
|
|
108
|
-
}
|
|
109
|
-
if (typeof parentId === 'string') {
|
|
110
|
-
const normalized = ToolsInteract._normalize(parentId);
|
|
111
|
-
if (!normalized)
|
|
112
|
-
return null;
|
|
113
|
-
if (/^\d+$/.test(normalized)) {
|
|
114
|
-
const index = Number(normalized);
|
|
115
|
-
if (index >= 0 && index < elements.length)
|
|
116
|
-
return index;
|
|
117
|
-
}
|
|
118
|
-
const foundIndex = elements.findIndex((el) => {
|
|
119
|
-
if (!el)
|
|
120
|
-
return false;
|
|
121
|
-
return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
122
|
-
ToolsInteract._normalize(el.stable_id ?? '') === normalized;
|
|
123
|
-
});
|
|
124
|
-
return foundIndex >= 0 ? foundIndex : null;
|
|
125
|
-
}
|
|
126
|
-
return null;
|
|
127
|
-
}
|
|
128
|
-
static _isVisibleElement(el) {
|
|
129
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
130
|
-
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
|
|
131
|
-
}
|
|
132
|
-
static _isTapActionable(el, storedStableId, platform) {
|
|
133
|
-
if (!ToolsInteract._isVisibleElement(el)) {
|
|
134
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
135
|
-
}
|
|
136
|
-
if (el.enabled === false) {
|
|
137
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
138
|
-
}
|
|
139
|
-
const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
|
|
140
|
-
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap')));
|
|
141
|
-
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
142
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
|
|
143
|
-
}
|
|
144
|
-
if (storedStableId) {
|
|
145
|
-
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
146
|
-
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
return { actionable: true };
|
|
150
|
-
}
|
|
151
|
-
static _isAdjustableActionable(el, storedStableId) {
|
|
152
|
-
if (!ToolsInteract._isVisibleElement(el)) {
|
|
153
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
154
|
-
}
|
|
155
|
-
if (el.enabled === false) {
|
|
156
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
157
|
-
}
|
|
158
|
-
if (storedStableId) {
|
|
159
|
-
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
160
|
-
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
return { actionable: true };
|
|
164
|
-
}
|
|
165
|
-
static _computeElementId(platform, deviceId, el, index) {
|
|
166
|
-
const identity = {
|
|
167
|
-
platform,
|
|
168
|
-
deviceId: deviceId || '',
|
|
169
|
-
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
170
|
-
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
171
|
-
accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
|
|
172
|
-
class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
173
|
-
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
174
|
-
index
|
|
175
|
-
};
|
|
176
|
-
return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`;
|
|
177
|
-
}
|
|
24
|
+
static _resolvedUiElements = new ResolvedElements(ToolsInteract._maxResolvedUiElements);
|
|
25
|
+
static _normalizeBounds = _normalizeBounds;
|
|
26
|
+
static _computeElementId = _computeElementId;
|
|
178
27
|
static _buildResolvedElement(platform, deviceId, el, index) {
|
|
179
28
|
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
180
29
|
const elementId = ToolsInteract._computeElementId(platform, deviceId, el, index);
|
|
@@ -203,224 +52,14 @@ export class ToolsInteract {
|
|
|
203
52
|
};
|
|
204
53
|
}
|
|
205
54
|
static _resolveUiChangeScope(tree, scope, target) {
|
|
206
|
-
|
|
207
|
-
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
|
|
208
|
-
if (normalizedScope === 'screen') {
|
|
209
|
-
return {
|
|
210
|
-
elements,
|
|
211
|
-
resolution: {
|
|
212
|
-
scope: 'screen',
|
|
213
|
-
target: null,
|
|
214
|
-
resolved: true,
|
|
215
|
-
resolvedIndex: null,
|
|
216
|
-
resolvedStableId: null,
|
|
217
|
-
reason: 'screen scope'
|
|
218
|
-
}
|
|
219
|
-
};
|
|
220
|
-
}
|
|
221
|
-
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
|
|
222
|
-
if (!requestedTarget) {
|
|
223
|
-
return {
|
|
224
|
-
elements: [],
|
|
225
|
-
resolution: {
|
|
226
|
-
scope: 'subtree',
|
|
227
|
-
target: null,
|
|
228
|
-
resolved: false,
|
|
229
|
-
resolvedIndex: null,
|
|
230
|
-
resolvedStableId: null,
|
|
231
|
-
reason: 'subtree scope requires a target element id'
|
|
232
|
-
},
|
|
233
|
-
error: {
|
|
234
|
-
code: 'INVALID_SCOPE',
|
|
235
|
-
message: 'scope=subtree requires a target element_id'
|
|
236
|
-
}
|
|
237
|
-
};
|
|
238
|
-
}
|
|
239
|
-
const resolved = ToolsInteract._findScopedElement(tree, requestedTarget);
|
|
240
|
-
if (!resolved) {
|
|
241
|
-
return {
|
|
242
|
-
elements: [],
|
|
243
|
-
resolution: {
|
|
244
|
-
scope: 'subtree',
|
|
245
|
-
target: requestedTarget,
|
|
246
|
-
resolved: false,
|
|
247
|
-
resolvedIndex: null,
|
|
248
|
-
resolvedStableId: null,
|
|
249
|
-
reason: 'target element could not be resolved'
|
|
250
|
-
},
|
|
251
|
-
error: {
|
|
252
|
-
code: 'ELEMENT_NOT_FOUND',
|
|
253
|
-
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
254
|
-
}
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
|
-
const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index);
|
|
258
|
-
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
|
|
259
|
-
return {
|
|
260
|
-
elements: scopedElements,
|
|
261
|
-
resolution: {
|
|
262
|
-
scope: 'subtree',
|
|
263
|
-
target: requestedTarget,
|
|
264
|
-
resolved: true,
|
|
265
|
-
resolvedIndex: resolved.index,
|
|
266
|
-
resolvedStableId: resolved.stableId,
|
|
267
|
-
reason: resolved.reason
|
|
268
|
-
}
|
|
269
|
-
};
|
|
270
|
-
}
|
|
271
|
-
static _findScopedElement(tree, targetElementId) {
|
|
272
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
273
|
-
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
|
|
274
|
-
const deviceId = tree?.device?.id ?? undefined;
|
|
275
|
-
const normalizedTarget = ToolsInteract._normalize(targetElementId);
|
|
276
|
-
for (let i = 0; i < elements.length; i++) {
|
|
277
|
-
const el = elements[i];
|
|
278
|
-
if (!el)
|
|
279
|
-
continue;
|
|
280
|
-
const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i);
|
|
281
|
-
if (computedElementId === targetElementId) {
|
|
282
|
-
return {
|
|
283
|
-
index: i,
|
|
284
|
-
stableId: el.stable_id ?? null,
|
|
285
|
-
reason: 'element_id_match'
|
|
286
|
-
};
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
for (let i = 0; i < elements.length; i++) {
|
|
290
|
-
const el = elements[i];
|
|
291
|
-
if (!el)
|
|
292
|
-
continue;
|
|
293
|
-
if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
|
|
294
|
-
return {
|
|
295
|
-
index: i,
|
|
296
|
-
stableId: el.stable_id,
|
|
297
|
-
reason: 'stable_id_match'
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId);
|
|
302
|
-
if (storedContext?.stable_id) {
|
|
303
|
-
const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id);
|
|
304
|
-
for (let i = 0; i < elements.length; i++) {
|
|
305
|
-
const el = elements[i];
|
|
306
|
-
if (!el?.stable_id)
|
|
307
|
-
continue;
|
|
308
|
-
if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
|
|
309
|
-
return {
|
|
310
|
-
index: i,
|
|
311
|
-
stableId: el.stable_id,
|
|
312
|
-
reason: 'stored_stable_id_match'
|
|
313
|
-
};
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
return null;
|
|
318
|
-
}
|
|
319
|
-
static _collectSubtreeIndices(elements, rootIndex) {
|
|
320
|
-
if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length)
|
|
321
|
-
return [];
|
|
322
|
-
const visited = new Set();
|
|
323
|
-
const stack = [rootIndex];
|
|
324
|
-
const result = [];
|
|
325
|
-
while (stack.length > 0) {
|
|
326
|
-
const index = stack.pop();
|
|
327
|
-
if (index === undefined || visited.has(index) || index < 0 || index >= elements.length)
|
|
328
|
-
continue;
|
|
329
|
-
visited.add(index);
|
|
330
|
-
result.push(index);
|
|
331
|
-
const element = elements[index];
|
|
332
|
-
if (!element)
|
|
333
|
-
continue;
|
|
334
|
-
const directChildren = new Set();
|
|
335
|
-
if (Array.isArray(element.children)) {
|
|
336
|
-
for (const childIndex of element.children) {
|
|
337
|
-
if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
|
|
338
|
-
directChildren.add(childIndex);
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
for (let i = 0; i < elements.length; i++) {
|
|
343
|
-
if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
|
|
344
|
-
directChildren.add(i);
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
for (const childIndex of directChildren) {
|
|
348
|
-
if (!visited.has(childIndex))
|
|
349
|
-
stack.push(childIndex);
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
return result.sort((left, right) => left - right);
|
|
55
|
+
return resolveUiChangeScope(tree, scope, target, (snapshot, id) => ToolsInteract._findScopedElement(snapshot, id));
|
|
353
56
|
}
|
|
354
|
-
static
|
|
355
|
-
|
|
356
|
-
if (stableId)
|
|
357
|
-
return `stable:${stableId}`;
|
|
358
|
-
return `fallback:${ToolsInteract._hash({
|
|
359
|
-
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
360
|
-
contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
|
|
361
|
-
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
362
|
-
type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
363
|
-
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
364
|
-
index
|
|
365
|
-
})}`;
|
|
366
|
-
}
|
|
367
|
-
static _summarizeUiChangeDelta(initialElements, currentElements) {
|
|
368
|
-
const buildMap = (elements) => {
|
|
369
|
-
const map = new Map();
|
|
370
|
-
for (let i = 0; i < elements.length; i++) {
|
|
371
|
-
const element = elements[i];
|
|
372
|
-
if (!element)
|
|
373
|
-
continue;
|
|
374
|
-
const key = ToolsInteract._changeIdentityForElement(element, i);
|
|
375
|
-
map.set(key, ToolsInteract._hash({
|
|
376
|
-
text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
|
|
377
|
-
contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
|
|
378
|
-
resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
|
|
379
|
-
type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
|
|
380
|
-
bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
|
|
381
|
-
state: element.state ?? null,
|
|
382
|
-
visible: !!element.visible,
|
|
383
|
-
enabled: !!element.enabled,
|
|
384
|
-
clickable: !!element.clickable
|
|
385
|
-
}));
|
|
386
|
-
}
|
|
387
|
-
return map;
|
|
388
|
-
};
|
|
389
|
-
const initialMap = buildMap(initialElements);
|
|
390
|
-
const currentMap = buildMap(currentElements);
|
|
391
|
-
let added = 0;
|
|
392
|
-
let removed = 0;
|
|
393
|
-
let mutated = 0;
|
|
394
|
-
for (const [key, value] of currentMap.entries()) {
|
|
395
|
-
if (!initialMap.has(key)) {
|
|
396
|
-
added++;
|
|
397
|
-
}
|
|
398
|
-
else if (initialMap.get(key) !== value) {
|
|
399
|
-
mutated++;
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
for (const key of initialMap.keys()) {
|
|
403
|
-
if (!currentMap.has(key))
|
|
404
|
-
removed++;
|
|
405
|
-
}
|
|
406
|
-
return {
|
|
407
|
-
total_elements: currentElements.length,
|
|
408
|
-
added_elements: added,
|
|
409
|
-
removed_elements: removed,
|
|
410
|
-
mutated_elements: mutated
|
|
411
|
-
};
|
|
57
|
+
static _findScopedElement(tree, id) {
|
|
58
|
+
return findScopedElement(tree, id, ToolsInteract._resolvedUiElements);
|
|
412
59
|
}
|
|
60
|
+
static _collectSubtreeIndices = _collectSubtreeIndices;
|
|
413
61
|
static _rememberResolvedElement(elementId, context) {
|
|
414
|
-
|
|
415
|
-
ToolsInteract._resolvedUiElements.delete(elementId);
|
|
416
|
-
}
|
|
417
|
-
ToolsInteract._resolvedUiElements.set(elementId, context);
|
|
418
|
-
while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
|
|
419
|
-
const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value;
|
|
420
|
-
if (!oldestElementId)
|
|
421
|
-
break;
|
|
422
|
-
ToolsInteract._resolvedUiElements.delete(oldestElementId);
|
|
423
|
-
}
|
|
62
|
+
ToolsInteract._resolvedUiElements.remember(elementId, context);
|
|
424
63
|
}
|
|
425
64
|
static async _captureFingerprint(platform, deviceId) {
|
|
426
65
|
if (isConditionBlocked())
|
|
@@ -433,402 +72,7 @@ export class ToolsInteract {
|
|
|
433
72
|
return null;
|
|
434
73
|
}
|
|
435
74
|
}
|
|
436
|
-
static
|
|
437
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
438
|
-
const textPayload = [];
|
|
439
|
-
const statePayload = [];
|
|
440
|
-
for (const el of elements) {
|
|
441
|
-
textPayload.push({
|
|
442
|
-
text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
|
|
443
|
-
contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
|
|
444
|
-
resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
|
|
445
|
-
});
|
|
446
|
-
statePayload.push({
|
|
447
|
-
checked: el?.state?.checked ?? null,
|
|
448
|
-
selected: el?.state?.selected ?? null,
|
|
449
|
-
focused: el?.state?.focused ?? null,
|
|
450
|
-
expanded: el?.state?.expanded ?? null,
|
|
451
|
-
enabled: el?.state?.enabled ?? null,
|
|
452
|
-
text_value: el?.state?.text_value ?? null,
|
|
453
|
-
value: el?.state?.value ?? null,
|
|
454
|
-
raw_value: el?.state?.raw_value ?? null,
|
|
455
|
-
value_range: el?.state?.value_range ?? null
|
|
456
|
-
});
|
|
457
|
-
}
|
|
458
|
-
return {
|
|
459
|
-
hierarchy: computeSnapshotSignature(tree),
|
|
460
|
-
text: ToolsInteract._hash({
|
|
461
|
-
screen: ToolsInteract._normalize(tree?.screen),
|
|
462
|
-
elements: textPayload
|
|
463
|
-
}),
|
|
464
|
-
state: ToolsInteract._hash({
|
|
465
|
-
screen: ToolsInteract._normalize(tree?.screen),
|
|
466
|
-
elements: statePayload
|
|
467
|
-
})
|
|
468
|
-
};
|
|
469
|
-
}
|
|
470
|
-
static _matchesUiChange(expected, initial, current) {
|
|
471
|
-
const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds;
|
|
472
|
-
for (const changeKind of candidates) {
|
|
473
|
-
if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy)
|
|
474
|
-
return changeKind;
|
|
475
|
-
if (changeKind === 'text_change' && initial.text !== current.text)
|
|
476
|
-
return changeKind;
|
|
477
|
-
if (changeKind === 'state_change' && initial.state !== current.state)
|
|
478
|
-
return changeKind;
|
|
479
|
-
}
|
|
480
|
-
return null;
|
|
481
|
-
}
|
|
482
|
-
static _uiChangeSignaturesEqual(left, right) {
|
|
483
|
-
return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state;
|
|
484
|
-
}
|
|
485
|
-
static _resolvedTargetFromElement(elementId, element, index) {
|
|
486
|
-
return {
|
|
487
|
-
elementId,
|
|
488
|
-
text: element.text ?? null,
|
|
489
|
-
resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
|
|
490
|
-
accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
|
|
491
|
-
class: element.type ?? element.class ?? null,
|
|
492
|
-
bounds: ToolsInteract._normalizeBounds(element.bounds),
|
|
493
|
-
index,
|
|
494
|
-
state: element.state ?? null,
|
|
495
|
-
stable_id: element.stable_id ?? null,
|
|
496
|
-
role: element.role ?? null,
|
|
497
|
-
test_tag: element.test_tag ?? null,
|
|
498
|
-
selector: element.selector ?? null,
|
|
499
|
-
semantic: element.semantic ?? null
|
|
500
|
-
};
|
|
501
|
-
}
|
|
502
|
-
static _summarizeResolutionCandidate(candidate) {
|
|
503
|
-
const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds);
|
|
504
|
-
return {
|
|
505
|
-
text: candidate.el.text ?? null,
|
|
506
|
-
resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
|
|
507
|
-
accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
|
|
508
|
-
class: candidate.el.type ?? candidate.el.class ?? null,
|
|
509
|
-
bounds: bounds
|
|
510
|
-
? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
|
|
511
|
-
: null,
|
|
512
|
-
clickable: !!candidate.el.clickable,
|
|
513
|
-
enabled: !!candidate.el.enabled,
|
|
514
|
-
score: candidate.score,
|
|
515
|
-
reason: candidate.reason
|
|
516
|
-
};
|
|
517
|
-
}
|
|
518
|
-
static _isAdjustableControl(el) {
|
|
519
|
-
if (!el)
|
|
520
|
-
return false;
|
|
521
|
-
const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
|
|
522
|
-
const role = ToolsInteract._normalize(el.role ?? '');
|
|
523
|
-
return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
|
|
524
|
-
}
|
|
525
|
-
static _isSemanticActionable(el) {
|
|
526
|
-
if (!el?.semantic)
|
|
527
|
-
return false;
|
|
528
|
-
if (el.semantic.adjustable)
|
|
529
|
-
return true;
|
|
530
|
-
return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0;
|
|
531
|
-
}
|
|
532
|
-
static _readNumericControlValue(el, property) {
|
|
533
|
-
if (!el?.state)
|
|
534
|
-
return null;
|
|
535
|
-
const stateValue = el.state[property];
|
|
536
|
-
if (typeof stateValue === 'number' && Number.isFinite(stateValue))
|
|
537
|
-
return stateValue;
|
|
538
|
-
const parsedStateValue = ToolsInteract._parseNumericControlValue(stateValue);
|
|
539
|
-
if (parsedStateValue !== null)
|
|
540
|
-
return parsedStateValue;
|
|
541
|
-
if (property === 'value' || property === 'raw_value') {
|
|
542
|
-
const fallback = el.state.raw_value ?? el.state.value;
|
|
543
|
-
if (typeof fallback === 'number' && Number.isFinite(fallback))
|
|
544
|
-
return fallback;
|
|
545
|
-
return ToolsInteract._parseNumericControlValue(fallback);
|
|
546
|
-
}
|
|
547
|
-
return null;
|
|
548
|
-
}
|
|
549
|
-
static _parseNumericControlValue(value) {
|
|
550
|
-
if (typeof value === 'number' && Number.isFinite(value))
|
|
551
|
-
return value;
|
|
552
|
-
if (typeof value !== 'string')
|
|
553
|
-
return null;
|
|
554
|
-
const trimmed = value.trim();
|
|
555
|
-
if (!trimmed)
|
|
556
|
-
return null;
|
|
557
|
-
const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/);
|
|
558
|
-
if (percent)
|
|
559
|
-
return Number(percent[1]);
|
|
560
|
-
const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i);
|
|
561
|
-
if (fraction)
|
|
562
|
-
return Number(fraction[1]);
|
|
563
|
-
const plain = Number(trimmed);
|
|
564
|
-
if (Number.isFinite(plain))
|
|
565
|
-
return plain;
|
|
566
|
-
const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/);
|
|
567
|
-
if (trailing)
|
|
568
|
-
return Number(trailing[1]);
|
|
569
|
-
return null;
|
|
570
|
-
}
|
|
571
|
-
static _readNumericControlRange(el) {
|
|
572
|
-
const min = typeof el?.state?.value_range?.min === 'number' ? el.state.value_range.min : null;
|
|
573
|
-
const max = typeof el?.state?.value_range?.max === 'number' ? el.state.value_range.max : null;
|
|
574
|
-
if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
|
|
575
|
-
return { min, max };
|
|
576
|
-
}
|
|
577
|
-
const value = el?.state?.value ?? el?.state?.raw_value;
|
|
578
|
-
if (typeof value === 'string') {
|
|
579
|
-
const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i);
|
|
580
|
-
if (fraction) {
|
|
581
|
-
const parsedMax = Number(fraction[1]);
|
|
582
|
-
if (Number.isFinite(parsedMax) && parsedMax > 0)
|
|
583
|
-
return { min: 0, max: parsedMax };
|
|
584
|
-
}
|
|
585
|
-
}
|
|
586
|
-
return null;
|
|
587
|
-
}
|
|
588
|
-
static _readControlStep(el) {
|
|
589
|
-
const candidates = [
|
|
590
|
-
el?.state?.step,
|
|
591
|
-
el?.state?.value_range?.step
|
|
592
|
-
];
|
|
593
|
-
for (const candidate of candidates) {
|
|
594
|
-
if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0)
|
|
595
|
-
return candidate;
|
|
596
|
-
}
|
|
597
|
-
return null;
|
|
598
|
-
}
|
|
599
|
-
static _quantizeTargetValue(targetValue, min, max, step) {
|
|
600
|
-
if (!step)
|
|
601
|
-
return targetValue;
|
|
602
|
-
const quantized = min + (Math.round((targetValue - min) / step) * step);
|
|
603
|
-
const bounded = Math.max(min, Math.min(max, quantized));
|
|
604
|
-
const precision = ToolsInteract._decimalPrecisionForStep(step);
|
|
605
|
-
return Number(bounded.toFixed(precision));
|
|
606
|
-
}
|
|
607
|
-
static _decimalPrecisionForStep(step) {
|
|
608
|
-
const normalized = String(step).toLowerCase();
|
|
609
|
-
const [coefficient, exponentPart] = normalized.split('e');
|
|
610
|
-
const coefficientDecimals = coefficient.includes('.')
|
|
611
|
-
? coefficient.split('.')[1]?.length ?? 0
|
|
612
|
-
: 0;
|
|
613
|
-
const exponent = exponentPart !== undefined ? Number(exponentPart) : 0;
|
|
614
|
-
const precision = exponent < 0
|
|
615
|
-
? coefficientDecimals + Math.abs(exponent)
|
|
616
|
-
: Math.max(0, coefficientDecimals - exponent);
|
|
617
|
-
return Math.max(0, Math.min(12, precision));
|
|
618
|
-
}
|
|
619
|
-
static _defaultAdjustTolerance(range, step) {
|
|
620
|
-
if (step)
|
|
621
|
-
return 0;
|
|
622
|
-
if (!range)
|
|
623
|
-
return 0;
|
|
624
|
-
return Math.max(0, 0.01 * Math.abs(range.max - range.min));
|
|
625
|
-
}
|
|
626
|
-
static _runtimeActionSet(el) {
|
|
627
|
-
const values = [
|
|
628
|
-
...(Array.isArray(el?.actions) ? el.actions : []),
|
|
629
|
-
...(Array.isArray(el?.accessibilityActions) ? el.accessibilityActions : []),
|
|
630
|
-
...(Array.isArray(el?.runtimeActions) ? el.runtimeActions : []),
|
|
631
|
-
...(Array.isArray(el?.platformActions) ? el.platformActions : [])
|
|
632
|
-
];
|
|
633
|
-
return new Set(values.map((value) => ToolsInteract._normalize(value)).filter(Boolean));
|
|
634
|
-
}
|
|
635
|
-
static _hasDirectAdjustmentAction(el) {
|
|
636
|
-
const actions = ToolsInteract._runtimeActionSet(el);
|
|
637
|
-
return [
|
|
638
|
-
'action_set_progress',
|
|
639
|
-
'set_progress',
|
|
640
|
-
'setprogress',
|
|
641
|
-
'semanticsactions.setprogress',
|
|
642
|
-
'set_value',
|
|
643
|
-
'setvalue',
|
|
644
|
-
'direct_set_value'
|
|
645
|
-
].some((action) => actions.has(action));
|
|
646
|
-
}
|
|
647
|
-
static _adjustmentDirections(el) {
|
|
648
|
-
const actions = ToolsInteract._runtimeActionSet(el);
|
|
649
|
-
return {
|
|
650
|
-
increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
|
|
651
|
-
decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
|
|
652
|
-
};
|
|
653
|
-
}
|
|
654
|
-
static _textFromElement(el) {
|
|
655
|
-
return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim();
|
|
656
|
-
}
|
|
657
|
-
static _parseLeadingNumber(text) {
|
|
658
|
-
const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/);
|
|
659
|
-
if (!match)
|
|
660
|
-
return null;
|
|
661
|
-
const value = Number(match[1]);
|
|
662
|
-
return Number.isFinite(value) ? value : null;
|
|
663
|
-
}
|
|
664
|
-
static _inferLabeledSliderState(elements, track) {
|
|
665
|
-
const trackBounds = ToolsInteract._normalizeBounds(track?.bounds);
|
|
666
|
-
if (!track || !trackBounds)
|
|
667
|
-
return null;
|
|
668
|
-
const type = ToolsInteract._normalize(track.type ?? track.class ?? '');
|
|
669
|
-
const role = ToolsInteract._normalize(track.role ?? '');
|
|
670
|
-
const width = trackBounds[2] - trackBounds[0];
|
|
671
|
-
const height = trackBounds[3] - trackBounds[1];
|
|
672
|
-
const wideTrackLike = width >= 240 && height <= 220;
|
|
673
|
-
const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role);
|
|
674
|
-
if (!wideTrackLike && !knownTrackLike)
|
|
675
|
-
return null;
|
|
676
|
-
const trackTop = trackBounds[1];
|
|
677
|
-
const trackBottom = trackBounds[3];
|
|
678
|
-
const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2;
|
|
679
|
-
const labels = elements
|
|
680
|
-
.map((el, index) => {
|
|
681
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
682
|
-
const text = ToolsInteract._textFromElement(el);
|
|
683
|
-
const value = ToolsInteract._parseLeadingNumber(text);
|
|
684
|
-
return bounds && text && value !== null
|
|
685
|
-
? {
|
|
686
|
-
index,
|
|
687
|
-
el,
|
|
688
|
-
text,
|
|
689
|
-
value,
|
|
690
|
-
bounds,
|
|
691
|
-
centerX: (bounds[0] + bounds[2]) / 2,
|
|
692
|
-
centerY: (bounds[1] + bounds[3]) / 2,
|
|
693
|
-
hasUnit: /[a-zA-Z%]/.test(text)
|
|
694
|
-
}
|
|
695
|
-
: null;
|
|
696
|
-
})
|
|
697
|
-
.filter((label) => !!label);
|
|
698
|
-
const rangeLabels = labels
|
|
699
|
-
.filter((label) => {
|
|
700
|
-
const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260;
|
|
701
|
-
const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120;
|
|
702
|
-
return verticallyNearTrack && horizontallyNearTrack && label.hasUnit;
|
|
703
|
-
})
|
|
704
|
-
.sort((a, b) => a.centerX - b.centerX);
|
|
705
|
-
if (rangeLabels.length < 2)
|
|
706
|
-
return null;
|
|
707
|
-
const minLabel = rangeLabels[0];
|
|
708
|
-
const maxLabel = rangeLabels[rangeLabels.length - 1];
|
|
709
|
-
if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value)
|
|
710
|
-
return null;
|
|
711
|
-
const currentCandidates = labels
|
|
712
|
-
.filter((label) => {
|
|
713
|
-
if (label.index === minLabel.index || label.index === maxLabel.index)
|
|
714
|
-
return false;
|
|
715
|
-
if (label.value < minLabel.value || label.value > maxLabel.value)
|
|
716
|
-
return false;
|
|
717
|
-
if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20)
|
|
718
|
-
return false;
|
|
719
|
-
return label.centerX >= trackCenterX - 80;
|
|
720
|
-
})
|
|
721
|
-
.sort((a, b) => {
|
|
722
|
-
const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY);
|
|
723
|
-
if (yDistance !== 0)
|
|
724
|
-
return yDistance;
|
|
725
|
-
return b.centerX - a.centerX;
|
|
726
|
-
});
|
|
727
|
-
const currentLabel = currentCandidates[0];
|
|
728
|
-
if (!currentLabel)
|
|
729
|
-
return null;
|
|
730
|
-
const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop;
|
|
731
|
-
const labelDerivedBounds = [
|
|
732
|
-
Math.min(minLabel.bounds[0], trackBounds[0]),
|
|
733
|
-
Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
|
|
734
|
-
Math.max(maxLabel.bounds[2], trackBounds[2]),
|
|
735
|
-
Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
|
|
736
|
-
];
|
|
737
|
-
return {
|
|
738
|
-
current: currentLabel.value,
|
|
739
|
-
min: minLabel.value,
|
|
740
|
-
max: maxLabel.value,
|
|
741
|
-
value_range: { min: minLabel.value, max: maxLabel.value },
|
|
742
|
-
touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
|
|
743
|
-
source: 'nearby_labels'
|
|
744
|
-
};
|
|
745
|
-
}
|
|
746
|
-
static _buildControlPoint(bounds, ratio, axis) {
|
|
747
|
-
const clampedRatio = Math.max(0, Math.min(1, ratio));
|
|
748
|
-
const [left, top, right, bottom] = bounds;
|
|
749
|
-
const width = Math.max(1, right - left);
|
|
750
|
-
const height = Math.max(1, bottom - top);
|
|
751
|
-
const insetX = Math.max(8, Math.floor(width * 0.08));
|
|
752
|
-
const insetY = Math.max(8, Math.floor(height * 0.08));
|
|
753
|
-
if (axis === 'vertical') {
|
|
754
|
-
const usableHeight = Math.max(1, height - (insetY * 2));
|
|
755
|
-
return {
|
|
756
|
-
x: Math.floor((left + right) / 2),
|
|
757
|
-
y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
|
|
758
|
-
};
|
|
759
|
-
}
|
|
760
|
-
const usableWidth = Math.max(1, width - (insetX * 2));
|
|
761
|
-
return {
|
|
762
|
-
x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
|
|
763
|
-
y: Math.floor((top + bottom) / 2)
|
|
764
|
-
};
|
|
765
|
-
}
|
|
766
|
-
static _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis) {
|
|
767
|
-
const range = Math.max(1, max - min);
|
|
768
|
-
const targetRatio = (targetValue - min) / range;
|
|
769
|
-
const stepRatio = 1 / range;
|
|
770
|
-
const centerBias = stepRatio / 2;
|
|
771
|
-
const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue);
|
|
772
|
-
const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0]);
|
|
773
|
-
const edgeWindow = Math.max(3, Math.floor(range * 0.1));
|
|
774
|
-
const isNearLowEdge = targetValue - min <= edgeWindow;
|
|
775
|
-
const isNearHighEdge = max - targetValue <= edgeWindow;
|
|
776
|
-
const directionBias = direction > 0
|
|
777
|
-
? -stepRatio * 0.15
|
|
778
|
-
: direction < 0
|
|
779
|
-
? stepRatio * 0.65
|
|
780
|
-
: 0;
|
|
781
|
-
const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx));
|
|
782
|
-
const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin);
|
|
783
|
-
const edgeBias = isNearLowEdge
|
|
784
|
-
? endpointMargin
|
|
785
|
-
: isNearHighEdge
|
|
786
|
-
? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
|
|
787
|
-
: 0;
|
|
788
|
-
const safeRatio = Math.min(1 - (endpointMargin * 0.25), Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias));
|
|
789
|
-
return ToolsInteract._buildControlPoint(bounds, safeRatio, axis);
|
|
790
|
-
}
|
|
791
|
-
static _buildAdjustmentProbePoints(bounds, targetValue, currentValue, min, max, axis) {
|
|
792
|
-
const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis);
|
|
793
|
-
const currentPoint = currentValue !== null
|
|
794
|
-
? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
|
|
795
|
-
: ToolsInteract._buildControlPoint(bounds, 0.5, axis);
|
|
796
|
-
const [left, top, right, bottom] = bounds;
|
|
797
|
-
const width = Math.max(1, right - left);
|
|
798
|
-
const height = Math.max(1, bottom - top);
|
|
799
|
-
const crossAxisBumps = axis === 'horizontal'
|
|
800
|
-
? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
|
|
801
|
-
: [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))];
|
|
802
|
-
const clampPoint = (point) => ({
|
|
803
|
-
x: axis === 'horizontal'
|
|
804
|
-
? Math.max(left, Math.min(right, point.x))
|
|
805
|
-
: Math.max(left, Math.min(right + Math.max(width, height), point.x)),
|
|
806
|
-
y: axis === 'vertical'
|
|
807
|
-
? Math.max(top, Math.min(bottom, point.y))
|
|
808
|
-
: Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
|
|
809
|
-
});
|
|
810
|
-
const probes = [targetPoint, currentPoint];
|
|
811
|
-
for (const bump of crossAxisBumps) {
|
|
812
|
-
if (axis === 'horizontal') {
|
|
813
|
-
probes.push({ x: targetPoint.x, y: bottom + bump }, { x: currentPoint.x, y: bottom + bump });
|
|
814
|
-
}
|
|
815
|
-
else {
|
|
816
|
-
probes.push({ x: right + bump, y: targetPoint.y }, { x: right + bump, y: currentPoint.y });
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
return Array.from(new Map(probes
|
|
820
|
-
.map(clampPoint)
|
|
821
|
-
.map((point) => [`${point.x}:${point.y}`, point])).values());
|
|
822
|
-
}
|
|
823
|
-
static _controlAxis(el, bounds) {
|
|
824
|
-
const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
|
|
825
|
-
const role = ToolsInteract._normalize(el.role ?? '');
|
|
826
|
-
if (/vertical/.test(type) || /vertical/.test(role))
|
|
827
|
-
return 'vertical';
|
|
828
|
-
if (/horizontal/.test(type) || /horizontal/.test(role))
|
|
829
|
-
return 'horizontal';
|
|
830
|
-
return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal';
|
|
831
|
-
}
|
|
75
|
+
static _resolvedTargetFromElement = _resolvedTargetFromElement;
|
|
832
76
|
static _actionFailure(actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter, sourceModule = 'interact') {
|
|
833
77
|
return buildActionExecutionResult({
|
|
834
78
|
actionType,
|
|
@@ -851,111 +95,11 @@ export class ToolsInteract {
|
|
|
851
95
|
}
|
|
852
96
|
return null;
|
|
853
97
|
}
|
|
854
|
-
static _resolveActionableAncestor(elements, chosen) {
|
|
855
|
-
if (!chosen)
|
|
856
|
-
return null;
|
|
857
|
-
if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el))
|
|
858
|
-
return chosen;
|
|
859
|
-
let current = chosen;
|
|
860
|
-
let safety = 0;
|
|
861
|
-
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
862
|
-
const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId);
|
|
863
|
-
if (parentIndex !== null && elements[parentIndex]) {
|
|
864
|
-
current = { el: elements[parentIndex], idx: parentIndex };
|
|
865
|
-
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
|
|
866
|
-
return current;
|
|
867
|
-
}
|
|
868
|
-
else {
|
|
869
|
-
break;
|
|
870
|
-
}
|
|
871
|
-
safety++;
|
|
872
|
-
}
|
|
873
|
-
const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
|
|
874
|
-
if (!childBounds)
|
|
875
|
-
return null;
|
|
876
|
-
const [cl, ct, cr, cb] = childBounds;
|
|
877
|
-
let best = null;
|
|
878
|
-
let bestArea = Infinity;
|
|
879
|
-
for (let i = 0; i < elements.length; i++) {
|
|
880
|
-
const el = elements[i];
|
|
881
|
-
if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el)))
|
|
882
|
-
continue;
|
|
883
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
884
|
-
if (!bounds)
|
|
885
|
-
continue;
|
|
886
|
-
const [pl, pt, pr, pb] = bounds;
|
|
887
|
-
if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
|
|
888
|
-
const area = (pr - pl) * (pb - pt);
|
|
889
|
-
if (area < bestArea) {
|
|
890
|
-
bestArea = area;
|
|
891
|
-
best = { el, idx: i };
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
return best;
|
|
896
|
-
}
|
|
897
|
-
static _resolveNearbyActionableControl(elements, chosen, screen) {
|
|
898
|
-
if (!chosen)
|
|
899
|
-
return null;
|
|
900
|
-
const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
|
|
901
|
-
if (!labelBounds)
|
|
902
|
-
return null;
|
|
903
|
-
const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds;
|
|
904
|
-
const labelWidth = labelRight - labelLeft;
|
|
905
|
-
const labelHeight = labelBottom - labelTop;
|
|
906
|
-
const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0;
|
|
907
|
-
const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0;
|
|
908
|
-
const minTrackLengthPx = Math.max(ToolsInteract._sliderTrackMinLengthPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0);
|
|
909
|
-
const maxTrackThicknessPx = Math.max(ToolsInteract._sliderTrackMaxThicknessPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0);
|
|
910
|
-
let best = null;
|
|
911
|
-
let bestScore = Infinity;
|
|
912
|
-
for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
|
|
913
|
-
const candidate = elements[i];
|
|
914
|
-
if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false)
|
|
915
|
-
continue;
|
|
916
|
-
const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds);
|
|
917
|
-
if (!candidateBounds)
|
|
918
|
-
continue;
|
|
919
|
-
const [left, top, right] = candidateBounds;
|
|
920
|
-
const width = right - left;
|
|
921
|
-
const height = candidateBounds[3] - top;
|
|
922
|
-
const verticalGap = top - labelBottom;
|
|
923
|
-
if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx)
|
|
924
|
-
continue;
|
|
925
|
-
const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left);
|
|
926
|
-
if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx)
|
|
927
|
-
continue;
|
|
928
|
-
const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '');
|
|
929
|
-
const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '');
|
|
930
|
-
const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '');
|
|
931
|
-
let score = verticalGap;
|
|
932
|
-
const horizontalTrackLike = width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
|
|
933
|
-
height <= maxTrackThicknessPx;
|
|
934
|
-
const verticalTrackLike = height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
|
|
935
|
-
width <= maxTrackThicknessPx;
|
|
936
|
-
const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike;
|
|
937
|
-
if (!candidateText && !candidateContent)
|
|
938
|
-
score -= 18;
|
|
939
|
-
if (trackLike)
|
|
940
|
-
score -= 30;
|
|
941
|
-
if (/view|layout|group|frame/i.test(candidateClass))
|
|
942
|
-
score -= 10;
|
|
943
|
-
if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio)
|
|
944
|
-
score -= 8;
|
|
945
|
-
if (candidateText || candidateContent)
|
|
946
|
-
score += 20;
|
|
947
|
-
if (score < bestScore) {
|
|
948
|
-
bestScore = score;
|
|
949
|
-
best = { el: candidate, idx: i, sliderLike: trackLike };
|
|
950
|
-
}
|
|
951
|
-
}
|
|
952
|
-
return best;
|
|
953
|
-
}
|
|
954
98
|
static async getInteractionService(platform, deviceId) {
|
|
955
99
|
const effectivePlatform = platform || 'android';
|
|
956
100
|
const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
|
|
957
101
|
const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract();
|
|
958
|
-
return { interact
|
|
102
|
+
return { interact, resolved, platform: effectivePlatform };
|
|
959
103
|
}
|
|
960
104
|
static async tapHandler({ platform, x, y, deviceId, timeoutMs }) {
|
|
961
105
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
@@ -970,24 +114,14 @@ export class ToolsInteract {
|
|
|
970
114
|
}
|
|
971
115
|
const fingerprintBefore = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null;
|
|
972
116
|
const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
|
|
973
|
-
const
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
if (
|
|
978
|
-
return ToolsInteract._actionFailure(actionType, selector,
|
|
979
|
-
|
|
980
|
-
const
|
|
981
|
-
const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform);
|
|
982
|
-
if (!tapActionability.actionable) {
|
|
983
|
-
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
984
|
-
}
|
|
985
|
-
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
|
|
986
|
-
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
|
|
987
|
-
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
988
|
-
}
|
|
989
|
-
const x = Math.floor((bounds[0] + bounds[2]) / 2);
|
|
990
|
-
const y = Math.floor((bounds[1] + bounds[3]) / 2);
|
|
117
|
+
const tap = resolveTap(tree, resolved, {
|
|
118
|
+
current: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
|
|
119
|
+
target: (element, index) => ToolsInteract._resolvedTargetFromElement(resolved.elementId, element, index)
|
|
120
|
+
});
|
|
121
|
+
if ('failure' in tap)
|
|
122
|
+
return ToolsInteract._actionFailure(actionType, selector, tap.target, tap.failure, true, fingerprintBefore);
|
|
123
|
+
const resolvedTarget = tap.target;
|
|
124
|
+
const { x, y } = tap.point;
|
|
991
125
|
const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId, timeoutMs });
|
|
992
126
|
if (!tapResult.success) {
|
|
993
127
|
const fingerprintAfterFailure = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null;
|
|
@@ -1005,763 +139,17 @@ export class ToolsInteract {
|
|
|
1005
139
|
sourceModule: 'interact'
|
|
1006
140
|
});
|
|
1007
141
|
}
|
|
1008
|
-
static async adjustControlHandler(
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
:
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1020
|
-
let semanticFallbackElement = null;
|
|
1021
|
-
const traceSteps = [];
|
|
1022
|
-
let traceAttemptIndex = 0;
|
|
1023
|
-
const recordTraceStep = (stage, result, metadata) => {
|
|
1024
|
-
traceSteps.push(createTraceStep({
|
|
1025
|
-
stage,
|
|
1026
|
-
timestamp: Date.now(),
|
|
1027
|
-
result,
|
|
1028
|
-
attemptIndex: traceAttemptIndex++,
|
|
1029
|
-
metadata
|
|
1030
|
-
}));
|
|
1031
|
-
};
|
|
1032
|
-
const buildFailure = (failureCode, reason, resolved, device, actualState, attempts, adjustmentMode = 'gesture', retryable = false, uiFingerprintAfter = null) => {
|
|
1033
|
-
if (!traceSteps.some((step) => step.stage === 'resolve')) {
|
|
1034
|
-
recordTraceStep('resolve', 'failure', {
|
|
1035
|
-
reason,
|
|
1036
|
-
failure_code: failureCode
|
|
1037
|
-
});
|
|
1038
|
-
}
|
|
1039
|
-
if (!traceSteps.some((step) => step.stage === 'recover')) {
|
|
1040
|
-
recordTraceStep('recover', retryable ? 'retry' : 'failure', {
|
|
1041
|
-
reason,
|
|
1042
|
-
failure_code: failureCode,
|
|
1043
|
-
retry_allowed: retryable,
|
|
1044
|
-
recovery_attempts: attempts,
|
|
1045
|
-
retry_depth: attempts
|
|
1046
|
-
});
|
|
1047
|
-
}
|
|
1048
|
-
const responseTolerance = requestedTolerance ?? 0;
|
|
1049
|
-
const base = buildActionExecutionResult({
|
|
1050
|
-
actionType,
|
|
1051
|
-
sourceModule: 'interact',
|
|
1052
|
-
device,
|
|
1053
|
-
selector: targetSelector,
|
|
1054
|
-
resolved,
|
|
1055
|
-
success: false,
|
|
1056
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1057
|
-
uiFingerprintAfter,
|
|
1058
|
-
failure: { failureCode, retryable },
|
|
1059
|
-
details: {
|
|
1060
|
-
target_value: targetValue,
|
|
1061
|
-
tolerance: responseTolerance,
|
|
1062
|
-
property,
|
|
1063
|
-
attempts,
|
|
1064
|
-
adjustment_mode: adjustmentMode,
|
|
1065
|
-
actual_state: actualState,
|
|
1066
|
-
converged: false,
|
|
1067
|
-
within_tolerance: false,
|
|
1068
|
-
reason
|
|
1069
|
-
},
|
|
1070
|
-
traceSteps
|
|
1071
|
-
});
|
|
1072
|
-
return {
|
|
1073
|
-
...base,
|
|
1074
|
-
target_state: {
|
|
1075
|
-
property,
|
|
1076
|
-
target_value: targetValue,
|
|
1077
|
-
tolerance: responseTolerance
|
|
1078
|
-
},
|
|
1079
|
-
actual_state: actualState,
|
|
1080
|
-
within_tolerance: false,
|
|
1081
|
-
converged: false,
|
|
1082
|
-
attempts,
|
|
1083
|
-
adjustment_mode: adjustmentMode
|
|
1084
|
-
};
|
|
1085
|
-
};
|
|
1086
|
-
const resolveCurrentMatch = async () => {
|
|
1087
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
|
|
1088
|
-
resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform;
|
|
1089
|
-
resolvedDeviceId = tree?.device?.id || resolvedDeviceId;
|
|
1090
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
1091
|
-
if (element_id) {
|
|
1092
|
-
const stored = ToolsInteract._resolvedUiElements.get(element_id);
|
|
1093
|
-
if (!stored) {
|
|
1094
|
-
return null;
|
|
1095
|
-
}
|
|
1096
|
-
const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored);
|
|
1097
|
-
if (!current) {
|
|
1098
|
-
return null;
|
|
1099
|
-
}
|
|
1100
|
-
return {
|
|
1101
|
-
tree,
|
|
1102
|
-
device: tree?.device,
|
|
1103
|
-
match: { el: current.el, idx: current.index },
|
|
1104
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index), current.el, current.index)
|
|
1105
|
-
};
|
|
1106
|
-
}
|
|
1107
|
-
if (semanticFallbackElement) {
|
|
1108
|
-
const fallbackBounds = ToolsInteract._normalizeBounds(Array.isArray(semanticFallbackElement.bounds)
|
|
1109
|
-
? semanticFallbackElement.bounds
|
|
1110
|
-
: semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
|
|
1111
|
-
? [
|
|
1112
|
-
Number(semanticFallbackElement.bounds.left),
|
|
1113
|
-
Number(semanticFallbackElement.bounds.top),
|
|
1114
|
-
Number(semanticFallbackElement.bounds.right),
|
|
1115
|
-
Number(semanticFallbackElement.bounds.bottom)
|
|
1116
|
-
]
|
|
1117
|
-
: null);
|
|
1118
|
-
let matchedIndex = -1;
|
|
1119
|
-
if (fallbackBounds) {
|
|
1120
|
-
matchedIndex = elements.findIndex((el) => {
|
|
1121
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
1122
|
-
return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3];
|
|
1123
|
-
});
|
|
1124
|
-
}
|
|
1125
|
-
if (matchedIndex === -1 && fallbackBounds) {
|
|
1126
|
-
const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2);
|
|
1127
|
-
const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2);
|
|
1128
|
-
let bestDistance = Infinity;
|
|
1129
|
-
for (let i = 0; i < elements.length; i++) {
|
|
1130
|
-
const el = elements[i];
|
|
1131
|
-
if (!ToolsInteract._isAdjustableControl(el))
|
|
1132
|
-
continue;
|
|
1133
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
1134
|
-
if (!bounds)
|
|
1135
|
-
continue;
|
|
1136
|
-
const centerX = Math.floor((bounds[0] + bounds[2]) / 2);
|
|
1137
|
-
const centerY = Math.floor((bounds[1] + bounds[3]) / 2);
|
|
1138
|
-
const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY);
|
|
1139
|
-
if (distance < bestDistance) {
|
|
1140
|
-
bestDistance = distance;
|
|
1141
|
-
matchedIndex = i;
|
|
1142
|
-
}
|
|
1143
|
-
}
|
|
1144
|
-
}
|
|
1145
|
-
if (matchedIndex >= 0 && elements[matchedIndex]) {
|
|
1146
|
-
const matched = { el: elements[matchedIndex], idx: matchedIndex };
|
|
1147
|
-
return {
|
|
1148
|
-
tree,
|
|
1149
|
-
device: tree?.device,
|
|
1150
|
-
match: matched,
|
|
1151
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
|
|
1152
|
-
};
|
|
1153
|
-
}
|
|
1154
|
-
}
|
|
1155
|
-
if (selector) {
|
|
1156
|
-
const matched = ToolsInteract._findFirstMatchingElement(elements, selector);
|
|
1157
|
-
if (!matched) {
|
|
1158
|
-
return null;
|
|
1159
|
-
}
|
|
1160
|
-
return {
|
|
1161
|
-
tree,
|
|
1162
|
-
device: tree?.device,
|
|
1163
|
-
match: matched,
|
|
1164
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
|
|
1165
|
-
};
|
|
1166
|
-
}
|
|
1167
|
-
return null;
|
|
1168
|
-
};
|
|
1169
|
-
if ((!selector && !element_id) || (selector && element_id)) {
|
|
1170
|
-
return buildFailure('ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', null, undefined, null, 0, 'gesture', false);
|
|
1171
|
-
}
|
|
1172
|
-
if (selector && !element_id) {
|
|
1173
|
-
const waitResult = await ToolsInteract.waitForUIHandler({
|
|
1174
|
-
selector,
|
|
1175
|
-
condition: 'clickable',
|
|
1176
|
-
timeout_ms: 5000,
|
|
1177
|
-
poll_interval_ms: 300,
|
|
1178
|
-
platform: resolvedPlatform,
|
|
1179
|
-
deviceId: resolvedDeviceId
|
|
1180
|
-
});
|
|
1181
|
-
if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
|
|
1182
|
-
const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? '';
|
|
1183
|
-
if (!semanticQuery) {
|
|
1184
|
-
return buildFailure(waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', waitResult?.error?.code === 'ELEMENT_NOT_FOUND');
|
|
1185
|
-
}
|
|
1186
|
-
const fallback = await ToolsInteract.findElementHandler({
|
|
1187
|
-
query: semanticQuery,
|
|
1188
|
-
exact: false,
|
|
1189
|
-
timeoutMs: 3000,
|
|
1190
|
-
platform: resolvedPlatform,
|
|
1191
|
-
deviceId: resolvedDeviceId
|
|
1192
|
-
});
|
|
1193
|
-
if (!fallback.found || !fallback.element) {
|
|
1194
|
-
return buildFailure('ELEMENT_NOT_FOUND', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', true);
|
|
1195
|
-
}
|
|
1196
|
-
semanticFallbackElement = fallback.element;
|
|
1197
|
-
}
|
|
1198
|
-
else {
|
|
1199
|
-
element_id = waitResult.element.elementId;
|
|
1200
|
-
semanticFallbackElement = null;
|
|
1201
|
-
}
|
|
1202
|
-
}
|
|
1203
|
-
let lastObservedState = null;
|
|
1204
|
-
let lastAdjustmentMode = 'gesture';
|
|
1205
|
-
let resolvedTarget = null;
|
|
1206
|
-
let currentDevice = undefined;
|
|
1207
|
-
let attemptCount = 0;
|
|
1208
|
-
let cachedResolvedMatch = null;
|
|
1209
|
-
let coordinateTapsAwaitingFreshGesture = false;
|
|
1210
|
-
adjustmentAttempts: for (let attempt = 0; attempt < attemptsLimit && attemptCount < attemptsLimit; attempt++) {
|
|
1211
|
-
const resolved = cachedResolvedMatch
|
|
1212
|
-
? {
|
|
1213
|
-
tree: null,
|
|
1214
|
-
device: currentDevice,
|
|
1215
|
-
match: cachedResolvedMatch,
|
|
1216
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx), cachedResolvedMatch.el, cachedResolvedMatch.idx)
|
|
1217
|
-
}
|
|
1218
|
-
: await resolveCurrentMatch();
|
|
1219
|
-
if (!resolved || !resolved.match || !resolved.resolvedTarget) {
|
|
1220
|
-
return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
|
|
1221
|
-
}
|
|
1222
|
-
currentDevice = resolved.device;
|
|
1223
|
-
resolvedTarget = resolved.resolvedTarget;
|
|
1224
|
-
const currentEl = resolved.match.el;
|
|
1225
|
-
cachedResolvedMatch = resolved.match;
|
|
1226
|
-
const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id);
|
|
1227
|
-
if (!adjustableActionability.actionable) {
|
|
1228
|
-
return buildFailure(adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', adjustableActionability.reason ?? 'adjustable control is not actionable', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
|
|
1229
|
-
}
|
|
1230
|
-
const elementsForInference = Array.isArray(resolved.tree?.elements)
|
|
1231
|
-
? resolved.tree.elements
|
|
1232
|
-
: cachedResolvedMatch
|
|
1233
|
-
? [cachedResolvedMatch.el]
|
|
1234
|
-
: [currentEl];
|
|
1235
|
-
const inferredSliderState = ToolsInteract._inferLabeledSliderState(elementsForInference, currentEl);
|
|
1236
|
-
const nativeBounds = ToolsInteract._normalizeBounds(currentEl.bounds);
|
|
1237
|
-
const bounds = inferredSliderState?.touch_bounds ?? nativeBounds;
|
|
1238
|
-
const valueRange = ToolsInteract._readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null;
|
|
1239
|
-
const step = ToolsInteract._readControlStep(currentEl);
|
|
1240
|
-
const normalizedTargetValue = valueRange
|
|
1241
|
-
? ToolsInteract._quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step)
|
|
1242
|
-
: targetValue;
|
|
1243
|
-
const effectiveTolerance = requestedTolerance ?? ToolsInteract._defaultAdjustTolerance(valueRange, step);
|
|
1244
|
-
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null;
|
|
1245
|
-
const actualState = currentValue !== null
|
|
1246
|
-
? {
|
|
1247
|
-
property,
|
|
1248
|
-
value: currentValue,
|
|
1249
|
-
raw_value: typeof currentEl.state?.raw_value === 'number'
|
|
1250
|
-
? currentEl.state.raw_value
|
|
1251
|
-
: inferredSliderState?.current
|
|
1252
|
-
}
|
|
1253
|
-
: null;
|
|
1254
|
-
lastObservedState = actualState;
|
|
1255
|
-
if (!traceSteps.some((step) => step.stage === 'resolve')) {
|
|
1256
|
-
recordTraceStep('resolve', 'success', {
|
|
1257
|
-
resolved_target: resolvedTarget,
|
|
1258
|
-
current_value: currentValue,
|
|
1259
|
-
adjustment_mode: lastAdjustmentMode
|
|
1260
|
-
});
|
|
1261
|
-
}
|
|
1262
|
-
if (property !== 'value' && property !== 'raw_value') {
|
|
1263
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1264
|
-
}
|
|
1265
|
-
const isAdjustableControl = ToolsInteract._isAdjustableControl(currentEl) || !!inferredSliderState;
|
|
1266
|
-
if (!isAdjustableControl) {
|
|
1267
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1268
|
-
}
|
|
1269
|
-
if (currentValue === null) {
|
|
1270
|
-
return buildFailure('UNKNOWN', 'current numeric value is unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1271
|
-
}
|
|
1272
|
-
if (valueRange && (targetValue < valueRange.min || targetValue > valueRange.max)) {
|
|
1273
|
-
return buildFailure('CONTROL_CONVERGENCE_FAILED', `targetValue ${targetValue} is outside the control range ${valueRange.min}..${valueRange.max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1274
|
-
}
|
|
1275
|
-
if (currentValue !== null && Math.abs(currentValue - normalizedTargetValue) <= effectiveTolerance) {
|
|
1276
|
-
recordTraceStep('verify', 'success', {
|
|
1277
|
-
property,
|
|
1278
|
-
target_value: normalizedTargetValue,
|
|
1279
|
-
actual_state: actualState,
|
|
1280
|
-
reason: 'control already within tolerance'
|
|
1281
|
-
});
|
|
1282
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1283
|
-
const base = buildActionExecutionResult({
|
|
1284
|
-
actionType,
|
|
1285
|
-
sourceModule: 'interact',
|
|
1286
|
-
device: currentDevice,
|
|
1287
|
-
selector: targetSelector,
|
|
1288
|
-
resolved: resolvedTarget,
|
|
1289
|
-
success: true,
|
|
1290
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1291
|
-
uiFingerprintAfter,
|
|
1292
|
-
details: {
|
|
1293
|
-
target_value: normalizedTargetValue,
|
|
1294
|
-
tolerance: effectiveTolerance,
|
|
1295
|
-
property,
|
|
1296
|
-
attempts: attemptCount,
|
|
1297
|
-
adjustment_mode: 'semantic',
|
|
1298
|
-
actual_state: actualState,
|
|
1299
|
-
converged: true,
|
|
1300
|
-
within_tolerance: true,
|
|
1301
|
-
reason: 'control already within tolerance'
|
|
1302
|
-
},
|
|
1303
|
-
traceSteps
|
|
1304
|
-
});
|
|
1305
|
-
return {
|
|
1306
|
-
...base,
|
|
1307
|
-
target_state: {
|
|
1308
|
-
property,
|
|
1309
|
-
target_value: normalizedTargetValue,
|
|
1310
|
-
tolerance: effectiveTolerance
|
|
1311
|
-
},
|
|
1312
|
-
actual_state: actualState,
|
|
1313
|
-
within_tolerance: true,
|
|
1314
|
-
converged: true,
|
|
1315
|
-
attempts: attemptCount,
|
|
1316
|
-
adjustment_mode: 'semantic'
|
|
1317
|
-
};
|
|
1318
|
-
}
|
|
1319
|
-
if (!bounds) {
|
|
1320
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1321
|
-
}
|
|
1322
|
-
const runVerification = async () => {
|
|
1323
|
-
if (inferredSliderState) {
|
|
1324
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
|
|
1325
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
1326
|
-
const refreshed = ToolsInteract._inferLabeledSliderState(elements, currentEl);
|
|
1327
|
-
const observedValue = refreshed?.current ?? null;
|
|
1328
|
-
const observedState = observedValue !== null
|
|
1329
|
-
? { property, value: observedValue, raw_value: observedValue }
|
|
1330
|
-
: actualState;
|
|
1331
|
-
return {
|
|
1332
|
-
verification: {
|
|
1333
|
-
success: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance,
|
|
1334
|
-
observed_state: observedState,
|
|
1335
|
-
reason: observedValue !== null
|
|
1336
|
-
? 'visible labeled value read from nearby slider labels'
|
|
1337
|
-
: 'visible labeled value unavailable'
|
|
1338
|
-
},
|
|
1339
|
-
observedState,
|
|
1340
|
-
withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
|
|
1341
|
-
};
|
|
1342
|
-
}
|
|
1343
|
-
const verification = await ToolsInteract.expectStateHandler({
|
|
1344
|
-
element_id: resolvedTarget?.elementId ?? element_id,
|
|
1345
|
-
selector: selector ?? undefined,
|
|
1346
|
-
property,
|
|
1347
|
-
expected: normalizedTargetValue,
|
|
1348
|
-
platform: resolvedPlatform,
|
|
1349
|
-
deviceId: resolvedDeviceId
|
|
1350
|
-
});
|
|
1351
|
-
const observedValue = ToolsInteract._parseNumericControlValue(verification?.observed_state?.value) ??
|
|
1352
|
-
ToolsInteract._parseNumericControlValue(verification?.observed_state?.raw_value);
|
|
1353
|
-
const observedState = observedValue !== null
|
|
1354
|
-
? {
|
|
1355
|
-
property,
|
|
1356
|
-
value: observedValue,
|
|
1357
|
-
raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
|
|
1358
|
-
}
|
|
1359
|
-
: actualState;
|
|
1360
|
-
return {
|
|
1361
|
-
verification,
|
|
1362
|
-
observedState,
|
|
1363
|
-
withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
|
|
1364
|
-
};
|
|
1365
|
-
};
|
|
1366
|
-
let actionDevice = currentDevice;
|
|
1367
|
-
let observedState = actualState;
|
|
1368
|
-
let verification = null;
|
|
1369
|
-
let verificationResult = { verification: null, observedState: actualState, withinTolerance: false };
|
|
1370
|
-
const directActionSupported = ToolsInteract._hasDirectAdjustmentAction(currentEl);
|
|
1371
|
-
if (directActionSupported && attemptCount < attemptsLimit) {
|
|
1372
|
-
const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId);
|
|
1373
|
-
if (typeof interact.setAdjustableValue === 'function') {
|
|
1374
|
-
lastAdjustmentMode = 'semantic';
|
|
1375
|
-
recordTraceStep('execute', 'retry', {
|
|
1376
|
-
attempt: attemptCount + 1,
|
|
1377
|
-
mode: 'semantic',
|
|
1378
|
-
strategy: 'direct_set_value',
|
|
1379
|
-
target_value: normalizedTargetValue
|
|
1380
|
-
});
|
|
1381
|
-
const actionResult = await interact.setAdjustableValue({
|
|
1382
|
-
deviceId: actionResolved.id,
|
|
1383
|
-
target: resolvedTarget,
|
|
1384
|
-
property,
|
|
1385
|
-
value: normalizedTargetValue
|
|
1386
|
-
});
|
|
1387
|
-
attemptCount++;
|
|
1388
|
-
actionDevice = actionResult?.device ?? actionDevice;
|
|
1389
|
-
if (actionResult?.success !== false) {
|
|
1390
|
-
verificationResult = await runVerification();
|
|
1391
|
-
observedState = verificationResult.observedState;
|
|
1392
|
-
lastObservedState = observedState;
|
|
1393
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1394
|
-
attempt: attemptCount,
|
|
1395
|
-
property,
|
|
1396
|
-
target_value: normalizedTargetValue,
|
|
1397
|
-
actual_state: observedState,
|
|
1398
|
-
reason: verificationResult.verification?.reason ?? 'direct adjustment did not converge yet'
|
|
1399
|
-
});
|
|
1400
|
-
if (verificationResult.withinTolerance) {
|
|
1401
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1402
|
-
const base = buildActionExecutionResult({
|
|
1403
|
-
actionType,
|
|
1404
|
-
sourceModule: 'interact',
|
|
1405
|
-
device: actionDevice ?? currentDevice,
|
|
1406
|
-
selector: targetSelector,
|
|
1407
|
-
resolved: resolvedTarget,
|
|
1408
|
-
success: true,
|
|
1409
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1410
|
-
uiFingerprintAfter,
|
|
1411
|
-
details: {
|
|
1412
|
-
target_value: normalizedTargetValue,
|
|
1413
|
-
tolerance: effectiveTolerance,
|
|
1414
|
-
property,
|
|
1415
|
-
attempts: attemptCount,
|
|
1416
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1417
|
-
actual_state: observedState,
|
|
1418
|
-
converged: true,
|
|
1419
|
-
within_tolerance: true,
|
|
1420
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1421
|
-
},
|
|
1422
|
-
traceSteps
|
|
1423
|
-
});
|
|
1424
|
-
return {
|
|
1425
|
-
...base,
|
|
1426
|
-
target_state: {
|
|
1427
|
-
property,
|
|
1428
|
-
target_value: normalizedTargetValue,
|
|
1429
|
-
tolerance: effectiveTolerance
|
|
1430
|
-
},
|
|
1431
|
-
actual_state: observedState,
|
|
1432
|
-
within_tolerance: true,
|
|
1433
|
-
converged: true,
|
|
1434
|
-
attempts: attemptCount,
|
|
1435
|
-
adjustment_mode: lastAdjustmentMode
|
|
1436
|
-
};
|
|
1437
|
-
}
|
|
1438
|
-
cachedResolvedMatch = null;
|
|
1439
|
-
if (attemptCount < attemptsLimit) {
|
|
1440
|
-
continue;
|
|
1441
|
-
}
|
|
1442
|
-
}
|
|
1443
|
-
}
|
|
1444
|
-
}
|
|
1445
|
-
const directions = ToolsInteract._adjustmentDirections(currentEl);
|
|
1446
|
-
if ((directions.increment || directions.decrement) && attemptCount < attemptsLimit) {
|
|
1447
|
-
const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId);
|
|
1448
|
-
const adjustByDirection = typeof interact.adjustAccessibleValue === 'function'
|
|
1449
|
-
? interact.adjustAccessibleValue.bind(interact)
|
|
1450
|
-
: null;
|
|
1451
|
-
while (adjustByDirection && attemptCount < attemptsLimit) {
|
|
1452
|
-
const direction = normalizedTargetValue > (observedState?.value ?? currentValue) ? 'increment' : 'decrement';
|
|
1453
|
-
if ((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement))
|
|
1454
|
-
break;
|
|
1455
|
-
lastAdjustmentMode = 'semantic';
|
|
1456
|
-
recordTraceStep('execute', 'retry', {
|
|
1457
|
-
attempt: attemptCount + 1,
|
|
1458
|
-
mode: 'semantic',
|
|
1459
|
-
strategy: 'accessibility_increment_decrement',
|
|
1460
|
-
direction
|
|
1461
|
-
});
|
|
1462
|
-
const actionResult = await adjustByDirection({
|
|
1463
|
-
deviceId: actionResolved.id,
|
|
1464
|
-
target: resolvedTarget,
|
|
1465
|
-
property,
|
|
1466
|
-
direction
|
|
1467
|
-
});
|
|
1468
|
-
attemptCount++;
|
|
1469
|
-
actionDevice = actionResult?.device ?? actionDevice;
|
|
1470
|
-
if (actionResult?.success === false)
|
|
1471
|
-
break;
|
|
1472
|
-
verificationResult = await runVerification();
|
|
1473
|
-
observedState = verificationResult.observedState;
|
|
1474
|
-
lastObservedState = observedState;
|
|
1475
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1476
|
-
attempt: attemptCount,
|
|
1477
|
-
property,
|
|
1478
|
-
target_value: normalizedTargetValue,
|
|
1479
|
-
actual_state: observedState,
|
|
1480
|
-
reason: verificationResult.verification?.reason ?? 'incremental adjustment did not converge yet'
|
|
1481
|
-
});
|
|
1482
|
-
if (verificationResult.withinTolerance) {
|
|
1483
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1484
|
-
const base = buildActionExecutionResult({
|
|
1485
|
-
actionType,
|
|
1486
|
-
sourceModule: 'interact',
|
|
1487
|
-
device: actionDevice ?? currentDevice,
|
|
1488
|
-
selector: targetSelector,
|
|
1489
|
-
resolved: resolvedTarget,
|
|
1490
|
-
success: true,
|
|
1491
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1492
|
-
uiFingerprintAfter,
|
|
1493
|
-
details: {
|
|
1494
|
-
target_value: normalizedTargetValue,
|
|
1495
|
-
tolerance: effectiveTolerance,
|
|
1496
|
-
property,
|
|
1497
|
-
attempts: attemptCount,
|
|
1498
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1499
|
-
actual_state: observedState,
|
|
1500
|
-
converged: true,
|
|
1501
|
-
within_tolerance: true,
|
|
1502
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1503
|
-
},
|
|
1504
|
-
traceSteps
|
|
1505
|
-
});
|
|
1506
|
-
return {
|
|
1507
|
-
...base,
|
|
1508
|
-
target_state: {
|
|
1509
|
-
property,
|
|
1510
|
-
target_value: normalizedTargetValue,
|
|
1511
|
-
tolerance: effectiveTolerance
|
|
1512
|
-
},
|
|
1513
|
-
actual_state: observedState,
|
|
1514
|
-
within_tolerance: true,
|
|
1515
|
-
converged: true,
|
|
1516
|
-
attempts: attemptCount,
|
|
1517
|
-
adjustment_mode: lastAdjustmentMode
|
|
1518
|
-
};
|
|
1519
|
-
}
|
|
1520
|
-
cachedResolvedMatch = null;
|
|
1521
|
-
if (attemptCount < attemptsLimit) {
|
|
1522
|
-
continue adjustmentAttempts;
|
|
1523
|
-
}
|
|
1524
|
-
}
|
|
1525
|
-
}
|
|
1526
|
-
if (!bounds) {
|
|
1527
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1528
|
-
}
|
|
1529
|
-
if (!valueRange) {
|
|
1530
|
-
return buildFailure('UNKNOWN', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
1531
|
-
}
|
|
1532
|
-
const min = valueRange.min;
|
|
1533
|
-
const max = valueRange.max;
|
|
1534
|
-
const axis = ToolsInteract._controlAxis(currentEl, bounds);
|
|
1535
|
-
const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, normalizedTargetValue, currentValue, min, max, axis);
|
|
1536
|
-
const currentPoint = ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis);
|
|
1537
|
-
if (!coordinateTapsAwaitingFreshGesture) {
|
|
1538
|
-
const probePoints = inferredSliderState
|
|
1539
|
-
? [targetPoint]
|
|
1540
|
-
: ToolsInteract._buildAdjustmentProbePoints(bounds, normalizedTargetValue, currentValue, min, max, axis);
|
|
1541
|
-
for (let i = 0; i < probePoints.length && attemptCount < attemptsLimit; i++) {
|
|
1542
|
-
const probePoint = probePoints[i];
|
|
1543
|
-
lastAdjustmentMode = 'coordinate';
|
|
1544
|
-
recordTraceStep('execute', 'retry', {
|
|
1545
|
-
attempt: attemptCount + 1,
|
|
1546
|
-
mode: 'coordinate',
|
|
1547
|
-
point: probePoint
|
|
1548
|
-
});
|
|
1549
|
-
const actionResult = await ToolsInteract.tapHandler({
|
|
1550
|
-
platform: resolvedPlatform,
|
|
1551
|
-
x: probePoint.x,
|
|
1552
|
-
y: probePoint.y,
|
|
1553
|
-
deviceId: resolvedDeviceId
|
|
1554
|
-
});
|
|
1555
|
-
attemptCount++;
|
|
1556
|
-
actionDevice = actionResult.device ?? actionDevice;
|
|
1557
|
-
if (!actionResult.success) {
|
|
1558
|
-
recordTraceStep('execute', 'retry', {
|
|
1559
|
-
attempt: attemptCount,
|
|
1560
|
-
mode: 'coordinate',
|
|
1561
|
-
point: probePoint,
|
|
1562
|
-
success: false
|
|
1563
|
-
});
|
|
1564
|
-
continue;
|
|
1565
|
-
}
|
|
1566
|
-
verificationResult = await runVerification();
|
|
1567
|
-
observedState = verificationResult.observedState;
|
|
1568
|
-
lastObservedState = observedState;
|
|
1569
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1570
|
-
attempt: attemptCount,
|
|
1571
|
-
property,
|
|
1572
|
-
target_value: normalizedTargetValue,
|
|
1573
|
-
actual_state: observedState,
|
|
1574
|
-
reason: verificationResult.verification?.reason ?? 'control did not converge yet'
|
|
1575
|
-
});
|
|
1576
|
-
if (verificationResult.withinTolerance) {
|
|
1577
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1578
|
-
const base = buildActionExecutionResult({
|
|
1579
|
-
actionType,
|
|
1580
|
-
sourceModule: 'interact',
|
|
1581
|
-
device: actionDevice ?? currentDevice,
|
|
1582
|
-
selector: targetSelector,
|
|
1583
|
-
resolved: resolvedTarget,
|
|
1584
|
-
success: true,
|
|
1585
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1586
|
-
uiFingerprintAfter,
|
|
1587
|
-
details: {
|
|
1588
|
-
target_value: normalizedTargetValue,
|
|
1589
|
-
tolerance: effectiveTolerance,
|
|
1590
|
-
property,
|
|
1591
|
-
attempts: attemptCount,
|
|
1592
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1593
|
-
actual_state: observedState,
|
|
1594
|
-
converged: true,
|
|
1595
|
-
within_tolerance: true,
|
|
1596
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1597
|
-
},
|
|
1598
|
-
traceSteps
|
|
1599
|
-
});
|
|
1600
|
-
return {
|
|
1601
|
-
...base,
|
|
1602
|
-
target_state: {
|
|
1603
|
-
property,
|
|
1604
|
-
target_value: normalizedTargetValue,
|
|
1605
|
-
tolerance: effectiveTolerance
|
|
1606
|
-
},
|
|
1607
|
-
actual_state: observedState,
|
|
1608
|
-
within_tolerance: true,
|
|
1609
|
-
converged: true,
|
|
1610
|
-
attempts: attemptCount,
|
|
1611
|
-
adjustment_mode: lastAdjustmentMode
|
|
1612
|
-
};
|
|
1613
|
-
}
|
|
1614
|
-
coordinateTapsAwaitingFreshGesture = true;
|
|
1615
|
-
cachedResolvedMatch = null;
|
|
1616
|
-
if (attemptCount < attemptsLimit) {
|
|
1617
|
-
continue adjustmentAttempts;
|
|
1618
|
-
}
|
|
1619
|
-
}
|
|
1620
|
-
}
|
|
1621
|
-
if (attemptCount < attemptsLimit) {
|
|
1622
|
-
coordinateTapsAwaitingFreshGesture = false;
|
|
1623
|
-
lastAdjustmentMode = 'gesture';
|
|
1624
|
-
recordTraceStep('execute', 'retry', {
|
|
1625
|
-
attempt: attemptCount + 1,
|
|
1626
|
-
mode: 'gesture',
|
|
1627
|
-
start: currentPoint,
|
|
1628
|
-
end: targetPoint
|
|
1629
|
-
});
|
|
1630
|
-
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
1631
|
-
platform: resolvedPlatform,
|
|
1632
|
-
x1: currentPoint.x,
|
|
1633
|
-
y1: currentPoint.y,
|
|
1634
|
-
x2: targetPoint.x,
|
|
1635
|
-
y2: targetPoint.y,
|
|
1636
|
-
duration: inferredSliderState ? 520 : 220,
|
|
1637
|
-
deviceId: resolvedDeviceId
|
|
1638
|
-
});
|
|
1639
|
-
attemptCount++;
|
|
1640
|
-
if (!fallbackActionResult.success) {
|
|
1641
|
-
recordTraceStep('execute', 'failure', {
|
|
1642
|
-
attempt: attemptCount,
|
|
1643
|
-
mode: 'gesture',
|
|
1644
|
-
start: currentPoint,
|
|
1645
|
-
end: targetPoint,
|
|
1646
|
-
success: false
|
|
1647
|
-
});
|
|
1648
|
-
return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false);
|
|
1649
|
-
}
|
|
1650
|
-
actionDevice = fallbackActionResult.device ?? actionDevice;
|
|
1651
|
-
verificationResult = await runVerification();
|
|
1652
|
-
observedState = verificationResult.observedState;
|
|
1653
|
-
lastObservedState = observedState;
|
|
1654
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1655
|
-
attempt: attemptCount,
|
|
1656
|
-
property,
|
|
1657
|
-
target_value: normalizedTargetValue,
|
|
1658
|
-
actual_state: observedState,
|
|
1659
|
-
reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
|
|
1660
|
-
});
|
|
1661
|
-
if (verificationResult.withinTolerance) {
|
|
1662
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1663
|
-
const base = buildActionExecutionResult({
|
|
1664
|
-
actionType,
|
|
1665
|
-
sourceModule: 'interact',
|
|
1666
|
-
device: actionDevice ?? currentDevice,
|
|
1667
|
-
selector: targetSelector,
|
|
1668
|
-
resolved: resolvedTarget,
|
|
1669
|
-
success: true,
|
|
1670
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1671
|
-
uiFingerprintAfter,
|
|
1672
|
-
details: {
|
|
1673
|
-
target_value: normalizedTargetValue,
|
|
1674
|
-
tolerance: effectiveTolerance,
|
|
1675
|
-
property,
|
|
1676
|
-
attempts: attemptCount,
|
|
1677
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1678
|
-
actual_state: observedState,
|
|
1679
|
-
converged: true,
|
|
1680
|
-
within_tolerance: true,
|
|
1681
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1682
|
-
},
|
|
1683
|
-
traceSteps
|
|
1684
|
-
});
|
|
1685
|
-
return {
|
|
1686
|
-
...base,
|
|
1687
|
-
target_state: {
|
|
1688
|
-
property,
|
|
1689
|
-
target_value: normalizedTargetValue,
|
|
1690
|
-
tolerance: effectiveTolerance
|
|
1691
|
-
},
|
|
1692
|
-
actual_state: observedState,
|
|
1693
|
-
within_tolerance: true,
|
|
1694
|
-
converged: true,
|
|
1695
|
-
attempts: attemptCount,
|
|
1696
|
-
adjustment_mode: lastAdjustmentMode
|
|
1697
|
-
};
|
|
1698
|
-
}
|
|
1699
|
-
}
|
|
1700
|
-
verification = verificationResult.verification;
|
|
1701
|
-
lastObservedState = observedState;
|
|
1702
|
-
if (verificationResult.withinTolerance) {
|
|
1703
|
-
recordTraceStep('verify', 'success', {
|
|
1704
|
-
attempt: attemptCount,
|
|
1705
|
-
property,
|
|
1706
|
-
target_value: normalizedTargetValue,
|
|
1707
|
-
actual_state: observedState,
|
|
1708
|
-
reason: verification?.reason ?? 'control converged to target value'
|
|
1709
|
-
});
|
|
1710
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1711
|
-
const base = buildActionExecutionResult({
|
|
1712
|
-
actionType,
|
|
1713
|
-
sourceModule: 'interact',
|
|
1714
|
-
device: actionDevice ?? currentDevice,
|
|
1715
|
-
selector: targetSelector,
|
|
1716
|
-
resolved: resolvedTarget,
|
|
1717
|
-
success: true,
|
|
1718
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1719
|
-
uiFingerprintAfter,
|
|
1720
|
-
details: {
|
|
1721
|
-
target_value: normalizedTargetValue,
|
|
1722
|
-
tolerance: effectiveTolerance,
|
|
1723
|
-
property,
|
|
1724
|
-
attempts: attemptCount,
|
|
1725
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1726
|
-
actual_state: observedState,
|
|
1727
|
-
converged: true,
|
|
1728
|
-
within_tolerance: true,
|
|
1729
|
-
reason: verification?.reason ?? 'control converged to target value'
|
|
1730
|
-
},
|
|
1731
|
-
traceSteps
|
|
1732
|
-
});
|
|
1733
|
-
return {
|
|
1734
|
-
...base,
|
|
1735
|
-
target_state: {
|
|
1736
|
-
property,
|
|
1737
|
-
target_value: normalizedTargetValue,
|
|
1738
|
-
tolerance: effectiveTolerance
|
|
1739
|
-
},
|
|
1740
|
-
actual_state: observedState,
|
|
1741
|
-
within_tolerance: true,
|
|
1742
|
-
converged: true,
|
|
1743
|
-
attempts: attemptCount,
|
|
1744
|
-
adjustment_mode: lastAdjustmentMode
|
|
1745
|
-
};
|
|
1746
|
-
}
|
|
1747
|
-
cachedResolvedMatch = inferredSliderState
|
|
1748
|
-
? null
|
|
1749
|
-
: {
|
|
1750
|
-
el: {
|
|
1751
|
-
...currentEl,
|
|
1752
|
-
state: {
|
|
1753
|
-
...(currentEl.state ?? null),
|
|
1754
|
-
...(observedState ? {
|
|
1755
|
-
[observedState.property]: observedState.value,
|
|
1756
|
-
raw_value: observedState.raw_value ?? observedState.value
|
|
1757
|
-
} : {})
|
|
1758
|
-
}
|
|
1759
|
-
},
|
|
1760
|
-
idx: resolved.match.idx
|
|
1761
|
-
};
|
|
1762
|
-
}
|
|
1763
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
1764
|
-
return buildFailure('CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter);
|
|
142
|
+
static async adjustControlHandler(input) {
|
|
143
|
+
return adjustControl(input, {
|
|
144
|
+
observe: (platform, deviceId) => ToolsObserve.getUITreeHandler({ platform, deviceId }),
|
|
145
|
+
fingerprint: (platform, deviceId) => ToolsInteract._captureFingerprint(platform, deviceId),
|
|
146
|
+
getHandle: id => ToolsInteract._resolvedUiElements.get(id),
|
|
147
|
+
currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
|
|
148
|
+
target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx),
|
|
149
|
+
wait: input => ToolsInteract.waitForUIHandler(input), find: input => ToolsInteract.findElementHandler(input),
|
|
150
|
+
expectState: input => ToolsInteract.expectStateHandler(input), service: (platform, deviceId) => ToolsInteract.getInteractionService(platform, deviceId),
|
|
151
|
+
tap: input => ToolsInteract.tapHandler(input), swipe: input => ToolsInteract.swipeHandler(input)
|
|
152
|
+
});
|
|
1765
153
|
}
|
|
1766
154
|
static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId, timeoutMs }) {
|
|
1767
155
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
@@ -1779,1446 +167,33 @@ export class ToolsInteract {
|
|
|
1779
167
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
1780
168
|
return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id);
|
|
1781
169
|
}
|
|
1782
|
-
static async findElementHandler(
|
|
1783
|
-
|
|
1784
|
-
const start = Date.now();
|
|
1785
|
-
const deadline = start + timeoutMs;
|
|
1786
|
-
const normalize = ToolsInteract._normalize;
|
|
1787
|
-
const q = normalize(query);
|
|
1788
|
-
if (!q)
|
|
1789
|
-
return { found: false, error: 'Empty query' };
|
|
1790
|
-
let best = null;
|
|
1791
|
-
let bestTree = null;
|
|
1792
|
-
let bestIterationCandidates = [];
|
|
1793
|
-
let shouldStop = false;
|
|
1794
|
-
const scoreElement = (el, idx) => {
|
|
1795
|
-
if (!el || !el.visible)
|
|
1796
|
-
return null;
|
|
1797
|
-
const bounds = el.bounds || [0, 0, 0, 0];
|
|
1798
|
-
if (!Array.isArray(bounds) || bounds.length < 4)
|
|
1799
|
-
return null;
|
|
1800
|
-
const [l, t, r, b] = bounds;
|
|
1801
|
-
if (r <= l || b <= t)
|
|
1802
|
-
return null;
|
|
1803
|
-
// Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
|
|
1804
|
-
const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el));
|
|
1805
|
-
const text = normalize(el.text ?? el.label ?? el.value ?? '');
|
|
1806
|
-
const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
|
|
1807
|
-
const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
|
|
1808
|
-
const className = normalize(el.type ?? el.class ?? '');
|
|
1809
|
-
const semanticRole = normalize(el.semantic?.semantic_role ?? '');
|
|
1810
|
-
const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : [];
|
|
1811
|
-
let score = 0;
|
|
1812
|
-
let reason = 'best_scoring_candidate';
|
|
1813
|
-
if (exact) {
|
|
1814
|
-
if (text && text === q) {
|
|
1815
|
-
score = 1.0;
|
|
1816
|
-
reason = 'exact_text_match';
|
|
1817
|
-
}
|
|
1818
|
-
else if (content && content === q) {
|
|
1819
|
-
score = 0.95;
|
|
1820
|
-
reason = 'exact_content_desc_match';
|
|
1821
|
-
}
|
|
1822
|
-
else if (resourceId && resourceId === q) {
|
|
1823
|
-
score = 0.92;
|
|
1824
|
-
reason = 'exact_resource_id_match';
|
|
1825
|
-
}
|
|
1826
|
-
else if (className && className === q) {
|
|
1827
|
-
score = 0.3;
|
|
1828
|
-
reason = 'exact_class_match';
|
|
1829
|
-
}
|
|
1830
|
-
}
|
|
1831
|
-
else {
|
|
1832
|
-
if (text && text === q) {
|
|
1833
|
-
score = 1.0;
|
|
1834
|
-
reason = 'exact_text_match';
|
|
1835
|
-
}
|
|
1836
|
-
else if (content && content === q) {
|
|
1837
|
-
score = 0.95;
|
|
1838
|
-
reason = 'exact_content_desc_match';
|
|
1839
|
-
}
|
|
1840
|
-
else if (resourceId && resourceId === q) {
|
|
1841
|
-
score = 0.92;
|
|
1842
|
-
reason = 'exact_resource_id_match';
|
|
1843
|
-
}
|
|
1844
|
-
else if (text && text.includes(q)) {
|
|
1845
|
-
score = 0.6;
|
|
1846
|
-
reason = 'partial_text_match';
|
|
1847
|
-
}
|
|
1848
|
-
else if (content && content.includes(q)) {
|
|
1849
|
-
score = 0.55;
|
|
1850
|
-
reason = 'partial_content_desc_match';
|
|
1851
|
-
}
|
|
1852
|
-
else if (resourceId && resourceId.includes(q)) {
|
|
1853
|
-
score = 0.7;
|
|
1854
|
-
reason = 'partial_resource_id_match';
|
|
1855
|
-
}
|
|
1856
|
-
else if (className && className.includes(q)) {
|
|
1857
|
-
score = 0.3;
|
|
1858
|
-
reason = 'partial_class_match';
|
|
1859
|
-
}
|
|
1860
|
-
}
|
|
1861
|
-
if (!exact) {
|
|
1862
|
-
if (!score && semanticRole && semanticRole.includes(q)) {
|
|
1863
|
-
score = 0.5;
|
|
1864
|
-
reason = 'semantic_role_match';
|
|
1865
|
-
}
|
|
1866
|
-
if (semanticActions.some((action) => action.includes(q))) {
|
|
1867
|
-
score = Math.max(score, score > 0 ? 0.65 : 0.6);
|
|
1868
|
-
reason = 'semantic_action_match';
|
|
1869
|
-
}
|
|
1870
|
-
if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
|
|
1871
|
-
score = 0.45;
|
|
1872
|
-
reason = 'semantic_control_match';
|
|
1873
|
-
}
|
|
1874
|
-
}
|
|
1875
|
-
else {
|
|
1876
|
-
if (!score && semanticRole && semanticRole === q) {
|
|
1877
|
-
score = 0.5;
|
|
1878
|
-
reason = 'semantic_role_match';
|
|
1879
|
-
}
|
|
1880
|
-
if (semanticActions.some((action) => action === q)) {
|
|
1881
|
-
score = Math.max(score, score > 0 ? 0.65 : 0.6);
|
|
1882
|
-
reason = 'semantic_action_match';
|
|
1883
|
-
}
|
|
1884
|
-
}
|
|
1885
|
-
if (score > 0 && interactable)
|
|
1886
|
-
score += 0.05;
|
|
1887
|
-
if (score <= 0)
|
|
1888
|
-
return null;
|
|
1889
|
-
return { el, idx, score, reason, interactable };
|
|
1890
|
-
};
|
|
1891
|
-
while (Date.now() <= deadline) {
|
|
1892
|
-
try {
|
|
1893
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
1894
|
-
if (tree && Array.isArray(tree.elements)) {
|
|
1895
|
-
const elements = tree.elements;
|
|
1896
|
-
const iterationCandidates = [];
|
|
1897
|
-
let iterationImprovedBest = false;
|
|
1898
|
-
for (let i = 0; i < elements.length; i++) {
|
|
1899
|
-
const el = elements[i];
|
|
1900
|
-
try {
|
|
1901
|
-
const candidate = scoreElement(el, i);
|
|
1902
|
-
if (!candidate)
|
|
1903
|
-
continue;
|
|
1904
|
-
iterationCandidates.push(candidate);
|
|
1905
|
-
if (!best || candidate.score > best.score) {
|
|
1906
|
-
best = candidate;
|
|
1907
|
-
bestTree = tree;
|
|
1908
|
-
iterationImprovedBest = true;
|
|
1909
|
-
if (best.score >= 0.95) {
|
|
1910
|
-
shouldStop = true;
|
|
1911
|
-
break;
|
|
1912
|
-
}
|
|
1913
|
-
}
|
|
1914
|
-
}
|
|
1915
|
-
catch (e) {
|
|
1916
|
-
checkDebuggerGuard();
|
|
1917
|
-
console.error('Error scoring element:', e);
|
|
1918
|
-
}
|
|
1919
|
-
}
|
|
1920
|
-
if (iterationImprovedBest) {
|
|
1921
|
-
bestIterationCandidates = iterationCandidates.slice();
|
|
1922
|
-
}
|
|
1923
|
-
}
|
|
1924
|
-
}
|
|
1925
|
-
catch (e) {
|
|
1926
|
-
checkDebuggerGuard();
|
|
1927
|
-
console.error('Error fetching UI tree:', e);
|
|
1928
|
-
}
|
|
1929
|
-
if (shouldStop || Date.now() > deadline)
|
|
1930
|
-
break;
|
|
1931
|
-
await new Promise(r => setTimeout(r, 100));
|
|
1932
|
-
}
|
|
1933
|
-
if (!best)
|
|
1934
|
-
return { found: false, error: 'Element not found' };
|
|
1935
|
-
// If the best match is not interactable, try to resolve an actionable ancestor.
|
|
1936
|
-
try {
|
|
1937
|
-
const elements = (bestTree && Array.isArray(bestTree.elements)) ? bestTree.elements : [];
|
|
1938
|
-
const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution : null;
|
|
1939
|
-
let chosen = best;
|
|
1940
|
-
const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null;
|
|
1941
|
-
// Strategy 1: if parentId references an index, climb that chain
|
|
1942
|
-
let resolvedAncestor = null;
|
|
1943
|
-
if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
|
|
1944
|
-
let cur = chosen;
|
|
1945
|
-
let safety = 0;
|
|
1946
|
-
while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
|
|
1947
|
-
let pid = cur.el.parentId;
|
|
1948
|
-
let idx = null;
|
|
1949
|
-
if (typeof pid === 'number')
|
|
1950
|
-
idx = pid;
|
|
1951
|
-
else if (typeof pid === 'string' && /^\d+$/.test(pid))
|
|
1952
|
-
idx = Number(pid);
|
|
1953
|
-
// If parentId is not an index, try to find by matching resourceId or id field
|
|
1954
|
-
if (idx !== null && elements[idx]) {
|
|
1955
|
-
cur = { el: elements[idx], idx };
|
|
1956
|
-
if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) {
|
|
1957
|
-
resolvedAncestor = cur;
|
|
1958
|
-
break;
|
|
1959
|
-
}
|
|
1960
|
-
}
|
|
1961
|
-
else if (typeof pid === 'string') {
|
|
1962
|
-
// fallback: search elements for matching resourceId or id
|
|
1963
|
-
const foundIndex = elements.findIndex((el) => (el.resourceId === pid || el.id === pid));
|
|
1964
|
-
const found = foundIndex >= 0 ? elements[foundIndex] : null;
|
|
1965
|
-
if (found) {
|
|
1966
|
-
cur = { el: found, idx: foundIndex };
|
|
1967
|
-
if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) {
|
|
1968
|
-
resolvedAncestor = cur;
|
|
1969
|
-
break;
|
|
1970
|
-
}
|
|
1971
|
-
// otherwise continue climbing if this found element has its own parentId
|
|
1972
|
-
}
|
|
1973
|
-
else {
|
|
1974
|
-
break;
|
|
1975
|
-
}
|
|
1976
|
-
}
|
|
1977
|
-
else {
|
|
1978
|
-
break;
|
|
1979
|
-
}
|
|
1980
|
-
safety++;
|
|
1981
|
-
}
|
|
1982
|
-
}
|
|
1983
|
-
// Strategy 2: fallback - find a clickable element whose bounds fully contain the child's bounds
|
|
1984
|
-
if (!resolvedAncestor && childBounds) {
|
|
1985
|
-
const [cl, ct, cr, cb] = childBounds;
|
|
1986
|
-
// find candidates that are clickable and contain the child bounds
|
|
1987
|
-
const candidates = elements
|
|
1988
|
-
.map((el, idx) => ({ el, idx }))
|
|
1989
|
-
.filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds.length >= 4);
|
|
1990
|
-
let bestCandidate = null;
|
|
1991
|
-
let bestCandidateArea = Infinity;
|
|
1992
|
-
for (const c of candidates) {
|
|
1993
|
-
const bounds = c.el.bounds;
|
|
1994
|
-
const [pl, pt, pr, pb] = bounds;
|
|
1995
|
-
if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
|
|
1996
|
-
const area = (pr - pl) * (pb - pt);
|
|
1997
|
-
if (area < bestCandidateArea) {
|
|
1998
|
-
bestCandidateArea = area;
|
|
1999
|
-
bestCandidate = c;
|
|
2000
|
-
}
|
|
2001
|
-
}
|
|
2002
|
-
}
|
|
2003
|
-
if (bestCandidate)
|
|
2004
|
-
resolvedAncestor = bestCandidate;
|
|
2005
|
-
}
|
|
2006
|
-
if (resolvedAncestor) {
|
|
2007
|
-
best = {
|
|
2008
|
-
el: resolvedAncestor.el,
|
|
2009
|
-
idx: resolvedAncestor.idx,
|
|
2010
|
-
score: Math.min(1, best.score + 0.02),
|
|
2011
|
-
reason: 'clickable_parent_preferred',
|
|
2012
|
-
interactable: true
|
|
2013
|
-
};
|
|
2014
|
-
}
|
|
2015
|
-
if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
|
|
2016
|
-
const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen);
|
|
2017
|
-
if (nearbyActionable) {
|
|
2018
|
-
best = {
|
|
2019
|
-
el: nearbyActionable.el,
|
|
2020
|
-
idx: nearbyActionable.idx,
|
|
2021
|
-
score: Math.min(1, best.score + 0.02),
|
|
2022
|
-
reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
|
|
2023
|
-
interactable: true
|
|
2024
|
-
};
|
|
2025
|
-
}
|
|
2026
|
-
}
|
|
2027
|
-
}
|
|
2028
|
-
catch (e) {
|
|
2029
|
-
checkDebuggerGuard();
|
|
2030
|
-
console.error('Error resolving ancestor:', e);
|
|
2031
|
-
}
|
|
2032
|
-
if (!best)
|
|
2033
|
-
return { found: false, error: 'Element not found' };
|
|
2034
|
-
const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null;
|
|
2035
|
-
const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null;
|
|
2036
|
-
const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el));
|
|
2037
|
-
const alternateCandidates = uniqueRanked
|
|
2038
|
-
.filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
|
|
2039
|
-
.slice(0, 3)
|
|
2040
|
-
.map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate));
|
|
2041
|
-
const outEl = {
|
|
2042
|
-
text: best.el.text ?? null,
|
|
2043
|
-
resourceId: best.el.resourceId ?? null,
|
|
2044
|
-
contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
|
|
2045
|
-
class: best.el.type ?? best.el.class ?? null,
|
|
2046
|
-
bounds: boundsObj,
|
|
2047
|
-
clickable: !!best.el.clickable,
|
|
2048
|
-
enabled: !!best.el.enabled,
|
|
2049
|
-
stable_id: best.el.stable_id ?? null,
|
|
2050
|
-
role: best.el.role ?? null,
|
|
2051
|
-
test_tag: best.el.test_tag ?? null,
|
|
2052
|
-
selector: best.el.selector ?? null,
|
|
2053
|
-
semantic: best.el.semantic ?? null,
|
|
2054
|
-
tapCoordinates,
|
|
2055
|
-
telemetry: {
|
|
2056
|
-
matchedIndex: best.idx ?? null,
|
|
2057
|
-
matchedInteractable: !!best.interactable,
|
|
2058
|
-
sliderLike: best.reason === 'slider_track_preferred'
|
|
2059
|
-
}
|
|
2060
|
-
};
|
|
2061
|
-
if (best.reason === 'slider_track_preferred') {
|
|
2062
|
-
const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left);
|
|
2063
|
-
const interactionHint = {
|
|
2064
|
-
kind: 'slider',
|
|
2065
|
-
axis: isVertical ? 'vertical' : 'horizontal',
|
|
2066
|
-
trackBounds: boundsObj
|
|
2067
|
-
};
|
|
2068
|
-
outEl.interactionHint = interactionHint;
|
|
2069
|
-
}
|
|
2070
|
-
const scoreVal = Math.min(1, Number(best.score.toFixed(3)));
|
|
2071
|
-
const resolution = {
|
|
2072
|
-
confidence: scoreVal,
|
|
2073
|
-
reason: best.reason,
|
|
2074
|
-
fallback_available: alternateCandidates.length > 0,
|
|
2075
|
-
matched_count: uniqueRanked.length,
|
|
2076
|
-
alternates: alternateCandidates
|
|
2077
|
-
};
|
|
2078
|
-
return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution };
|
|
2079
|
-
}
|
|
2080
|
-
static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false, resolveTapTarget = false }) {
|
|
2081
|
-
const readinessTracker = currentReadinessWait();
|
|
2082
|
-
const overallStart = Date.now();
|
|
2083
|
-
const requestedIndex = typeof match?.index === 'number' ? match.index : null;
|
|
2084
|
-
const requested = {
|
|
2085
|
-
selector: selector ?? {},
|
|
2086
|
-
condition,
|
|
2087
|
-
match: requestedIndex === null ? null : { index: requestedIndex }
|
|
2088
|
-
};
|
|
2089
|
-
// Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
|
|
2090
|
-
const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
|
|
2091
|
-
const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
|
|
2092
|
-
const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
|
|
2093
|
-
if (!hasText && !hasResId && !hasAccId) {
|
|
2094
|
-
return {
|
|
2095
|
-
status: 'timeout',
|
|
2096
|
-
error: {
|
|
2097
|
-
code: 'INVALID_SELECTOR',
|
|
2098
|
-
message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
|
|
2099
|
-
},
|
|
2100
|
-
metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
|
|
2101
|
-
requested,
|
|
2102
|
-
observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
|
|
2103
|
-
};
|
|
2104
|
-
}
|
|
2105
|
-
// Validate condition
|
|
2106
|
-
if (!['exists', 'not_exists', 'visible', 'clickable'].includes(condition)) {
|
|
2107
|
-
return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } };
|
|
2108
|
-
}
|
|
2109
|
-
// Platform check
|
|
2110
|
-
if (platform && !['android', 'ios'].includes(platform)) {
|
|
2111
|
-
return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } };
|
|
2112
|
-
}
|
|
2113
|
-
const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000));
|
|
2114
|
-
const maxAttempts = (retry && retry.max_attempts) ? Math.max(1, retry.max_attempts) : 1;
|
|
2115
|
-
const backoff = (retry && retry.backoff_ms) ? Math.max(0, retry.backoff_ms) : 0;
|
|
2116
|
-
let attempts = 0;
|
|
2117
|
-
let totalPollCount = 0;
|
|
2118
|
-
let lastMatchedCount = 0;
|
|
2119
|
-
let lastMatchedElement = null;
|
|
2120
|
-
let lastConditionSatisfied = false;
|
|
2121
|
-
let matchedAt = null;
|
|
2122
|
-
let stableMatchCount = 0;
|
|
2123
|
-
const stableObservationCount = singleObservation ? 1 : 2;
|
|
2124
|
-
const snapshotStaleThresholdMs = 500;
|
|
2125
|
-
// Precompute normalized selector values and helpers (constant across polls)
|
|
2126
|
-
const normalize = ToolsInteract._normalize;
|
|
2127
|
-
const containsFlag = !!selector?.contains;
|
|
2128
|
-
const selText = normalize(selector?.text);
|
|
2129
|
-
const selRid = normalize(selector?.resource_id);
|
|
2130
|
-
const selAid = normalize(selector?.accessibility_id);
|
|
2131
|
-
try {
|
|
2132
|
-
while (attempts < maxAttempts) {
|
|
2133
|
-
attempts++;
|
|
2134
|
-
const attemptStart = Date.now();
|
|
2135
|
-
const deadline = attemptStart + (timeout_ms || 0);
|
|
2136
|
-
while (Date.now() <= deadline || (singleAttempt && totalPollCount === 0)) {
|
|
2137
|
-
remainingBudget();
|
|
2138
|
-
totalPollCount++;
|
|
2139
|
-
try {
|
|
2140
|
-
const acquisitionStarted = Date.now();
|
|
2141
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
2142
|
-
observeReadinessWait(tree, acquisitionStarted);
|
|
2143
|
-
// A failed acquisition cannot establish that the requested tap target is absent.
|
|
2144
|
-
if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error)))
|
|
2145
|
-
throw new Error('ACTION_TIMEOUT');
|
|
2146
|
-
const elements = (tree && Array.isArray(tree.elements)) ? tree.elements : [];
|
|
2147
|
-
const matches = [];
|
|
2148
|
-
for (let i = 0; i < elements.length; i++) {
|
|
2149
|
-
const el = elements[i];
|
|
2150
|
-
let ok = true;
|
|
2151
|
-
// text
|
|
2152
|
-
if (selector.text !== undefined && selector.text !== null) {
|
|
2153
|
-
const val = normalize(el.text || el.label || el.value || '');
|
|
2154
|
-
if (containsFlag) {
|
|
2155
|
-
if (!val.includes(selText))
|
|
2156
|
-
ok = false;
|
|
2157
|
-
}
|
|
2158
|
-
else {
|
|
2159
|
-
if (val !== selText)
|
|
2160
|
-
ok = false;
|
|
2161
|
-
}
|
|
2162
|
-
}
|
|
2163
|
-
// resource_id
|
|
2164
|
-
if (ok && selector.resource_id !== undefined && selector.resource_id !== null) {
|
|
2165
|
-
const rid = normalize(el.resourceId || el.resourceID || el.id || '');
|
|
2166
|
-
if (containsFlag) {
|
|
2167
|
-
if (!rid.includes(selRid))
|
|
2168
|
-
ok = false;
|
|
2169
|
-
}
|
|
2170
|
-
else {
|
|
2171
|
-
if (rid !== selRid)
|
|
2172
|
-
ok = false;
|
|
2173
|
-
}
|
|
2174
|
-
}
|
|
2175
|
-
// accessibility_id
|
|
2176
|
-
if (ok && selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
|
|
2177
|
-
const aid = normalize(el.contentDescription || el.contentDesc || el.accessibilityLabel || el.label || '');
|
|
2178
|
-
if (containsFlag) {
|
|
2179
|
-
if (!aid.includes(selAid))
|
|
2180
|
-
ok = false;
|
|
2181
|
-
}
|
|
2182
|
-
else {
|
|
2183
|
-
if (aid !== selAid)
|
|
2184
|
-
ok = false;
|
|
2185
|
-
}
|
|
2186
|
-
}
|
|
2187
|
-
if (ok)
|
|
2188
|
-
matches.push({ el, idx: i });
|
|
2189
|
-
}
|
|
2190
|
-
// Evaluate condition
|
|
2191
|
-
const matchedCount = matches.length;
|
|
2192
|
-
if (rejectAmbiguous && matchedCount > 1 && match?.index === undefined)
|
|
2193
|
-
return { status: 'timeout', error: { code: 'AMBIGUOUS_TARGET' } };
|
|
2194
|
-
const pickIndex = (typeof match?.index === 'number') ? match.index : undefined;
|
|
2195
|
-
let chosen = null;
|
|
2196
|
-
if (matches.length > 0) {
|
|
2197
|
-
if (pickIndex !== undefined) {
|
|
2198
|
-
// If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
|
|
2199
|
-
if (pickIndex >= 0 && pickIndex < matches.length)
|
|
2200
|
-
chosen = matches[pickIndex];
|
|
2201
|
-
else
|
|
2202
|
-
chosen = null;
|
|
2203
|
-
}
|
|
2204
|
-
else {
|
|
2205
|
-
chosen = matches[0];
|
|
2206
|
-
}
|
|
2207
|
-
}
|
|
2208
|
-
else {
|
|
2209
|
-
chosen = null;
|
|
2210
|
-
}
|
|
2211
|
-
let conditionMet = false;
|
|
2212
|
-
let matchedElement = chosen;
|
|
2213
|
-
if (condition === 'exists') {
|
|
2214
|
-
// when an index is specified, existence requires that specific index be present
|
|
2215
|
-
conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1);
|
|
2216
|
-
}
|
|
2217
|
-
else if (condition === 'not_exists') {
|
|
2218
|
-
// when an index is specified, not_exists is true if that index is absent
|
|
2219
|
-
conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0);
|
|
2220
|
-
}
|
|
2221
|
-
else if (condition === 'visible') {
|
|
2222
|
-
if (chosen) {
|
|
2223
|
-
const b = chosen.el.bounds;
|
|
2224
|
-
const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
|
|
2225
|
-
conditionMet = visibleFlag;
|
|
2226
|
-
}
|
|
2227
|
-
else
|
|
2228
|
-
conditionMet = false;
|
|
2229
|
-
}
|
|
2230
|
-
else if (condition === 'clickable') {
|
|
2231
|
-
matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
|
|
2232
|
-
? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen) || chosen) : null;
|
|
2233
|
-
if (matchedElement) {
|
|
2234
|
-
const b = matchedElement.el.bounds;
|
|
2235
|
-
const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
|
|
2236
|
-
const enabled = !!matchedElement.el.enabled;
|
|
2237
|
-
const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable;
|
|
2238
|
-
conditionMet = visibleFlag && enabled && clickable;
|
|
2239
|
-
}
|
|
2240
|
-
else
|
|
2241
|
-
conditionMet = false;
|
|
2242
|
-
}
|
|
2243
|
-
// Waiting for a label to exist/be visible is distinct from resolving its
|
|
2244
|
-
// executable control. Reuse the clickable-wait ancestor policy without
|
|
2245
|
-
// promoting hidden or disabled labels into an unrelated enabled target.
|
|
2246
|
-
if (resolveTapTarget && conditionMet && matchedElement &&
|
|
2247
|
-
ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
|
|
2248
|
-
matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement;
|
|
2249
|
-
}
|
|
2250
|
-
const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
|
|
2251
|
-
const resolvedDeviceId = tree?.device?.id || deviceId;
|
|
2252
|
-
lastMatchedCount = matchedCount;
|
|
2253
|
-
lastConditionSatisfied = conditionMet;
|
|
2254
|
-
lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null;
|
|
2255
|
-
const now = Date.now();
|
|
2256
|
-
const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null;
|
|
2257
|
-
const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs;
|
|
2258
|
-
if (conditionMet && snapshotFresh) {
|
|
2259
|
-
if (matchedAt === null)
|
|
2260
|
-
matchedAt = now;
|
|
2261
|
-
stableMatchCount++;
|
|
2262
|
-
if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
|
|
2263
|
-
const latency_ms = now - overallStart;
|
|
2264
|
-
const outEl = lastMatchedElement;
|
|
2265
|
-
return {
|
|
2266
|
-
status: 'success',
|
|
2267
|
-
...(singleObservation ? { _tree: tree } : {}),
|
|
2268
|
-
matched: matchedCount,
|
|
2269
|
-
element: outEl,
|
|
2270
|
-
metrics: { latency_ms, poll_count: totalPollCount, attempts },
|
|
2271
|
-
requested,
|
|
2272
|
-
observed: {
|
|
2273
|
-
matched_count: matchedCount,
|
|
2274
|
-
condition_satisfied: true,
|
|
2275
|
-
selected_index: outEl?.index ?? null,
|
|
2276
|
-
last_matched_element: outEl
|
|
2277
|
-
}
|
|
2278
|
-
};
|
|
2279
|
-
}
|
|
2280
|
-
}
|
|
2281
|
-
else {
|
|
2282
|
-
stableMatchCount = 0;
|
|
2283
|
-
matchedAt = null;
|
|
2284
|
-
}
|
|
2285
|
-
}
|
|
2286
|
-
catch (e) {
|
|
2287
|
-
checkDebuggerGuard();
|
|
2288
|
-
// Deadline expiry does not invalidate the last successfully observed state.
|
|
2289
|
-
if (e instanceof Error && e.message === 'ACTION_TIMEOUT')
|
|
2290
|
-
throw e;
|
|
2291
|
-
if (readinessTracker && !String(e).includes('READINESS_'))
|
|
2292
|
-
readinessTracker.unavailable();
|
|
2293
|
-
if (readinessTracker && isDeviceUnavailable(e))
|
|
2294
|
-
throw e;
|
|
2295
|
-
// Non-fatal per-poll error; record and continue
|
|
2296
|
-
if (!readinessTracker)
|
|
2297
|
-
console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e));
|
|
2298
|
-
}
|
|
2299
|
-
if (singleAttempt)
|
|
2300
|
-
break;
|
|
2301
|
-
// Sleep until next poll
|
|
2302
|
-
await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)));
|
|
2303
|
-
}
|
|
2304
|
-
// Attempt timed out; if more attempts allowed, backoff then retry
|
|
2305
|
-
if (attempts < maxAttempts) {
|
|
2306
|
-
if (backoff > 0)
|
|
2307
|
-
await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)));
|
|
2308
|
-
continue;
|
|
2309
|
-
}
|
|
2310
|
-
// Final failure for this call
|
|
2311
|
-
const elapsed = Date.now() - overallStart;
|
|
2312
|
-
const observed = {
|
|
2313
|
-
matched_count: lastMatchedCount,
|
|
2314
|
-
condition_satisfied: lastConditionSatisfied,
|
|
2315
|
-
selected_index: lastMatchedElement?.index ?? null,
|
|
2316
|
-
last_matched_element: lastMatchedElement
|
|
2317
|
-
};
|
|
2318
|
-
const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
|
|
2319
|
-
? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
|
|
2320
|
-
: ` observed ${lastMatchedCount} match(es)`;
|
|
2321
|
-
return {
|
|
2322
|
-
status: 'timeout',
|
|
2323
|
-
error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
|
|
2324
|
-
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
2325
|
-
requested,
|
|
2326
|
-
observed
|
|
2327
|
-
};
|
|
2328
|
-
}
|
|
2329
|
-
}
|
|
2330
|
-
catch (err) {
|
|
2331
|
-
checkDebuggerGuard();
|
|
2332
|
-
if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT')
|
|
2333
|
-
throw err;
|
|
2334
|
-
if (readinessTracker && isDeviceUnavailable(err))
|
|
2335
|
-
throw err;
|
|
2336
|
-
const elapsed = Date.now() - overallStart;
|
|
2337
|
-
return {
|
|
2338
|
-
status: 'timeout',
|
|
2339
|
-
error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
|
|
2340
|
-
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
2341
|
-
requested,
|
|
2342
|
-
observed: {
|
|
2343
|
-
matched_count: lastMatchedCount,
|
|
2344
|
-
condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
|
|
2345
|
-
selected_index: lastMatchedElement?.index ?? null,
|
|
2346
|
-
last_matched_element: lastMatchedElement
|
|
2347
|
-
}
|
|
2348
|
-
};
|
|
2349
|
-
}
|
|
170
|
+
static async findElementHandler(input) {
|
|
171
|
+
return findElement(input, target => ToolsObserve.getUITreeHandler(target));
|
|
2350
172
|
}
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
return [];
|
|
2355
|
-
return logsArr.map((l) => {
|
|
2356
|
-
if (typeof l === 'string')
|
|
2357
|
-
return l;
|
|
2358
|
-
if (l && (l.message || l.msg))
|
|
2359
|
-
return l.message || l.msg;
|
|
2360
|
-
try {
|
|
2361
|
-
return JSON.stringify(l);
|
|
2362
|
-
}
|
|
2363
|
-
catch {
|
|
2364
|
-
return String(l);
|
|
2365
|
-
}
|
|
2366
|
-
});
|
|
173
|
+
static async waitForUIHandler(input) {
|
|
174
|
+
return waitForUI(input, { observe: target => ToolsObserve.getUITreeHandler(target),
|
|
175
|
+
resolve: (platform, deviceId, element, index) => ToolsInteract._buildResolvedElement(platform, deviceId, element, index) });
|
|
2367
176
|
}
|
|
2368
|
-
static async waitForScreenChangeHandler(
|
|
2369
|
-
|
|
2370
|
-
let lastFingerprint = null;
|
|
2371
|
-
let lastActivity = null;
|
|
2372
|
-
while (Date.now() - start < timeoutMs) {
|
|
2373
|
-
try {
|
|
2374
|
-
const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
2375
|
-
const fp = res?.fingerprint ?? null;
|
|
2376
|
-
lastActivity = res?.activity ?? lastActivity;
|
|
2377
|
-
if (fp === null || fp === undefined) {
|
|
2378
|
-
lastFingerprint = null;
|
|
2379
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
|
|
2380
|
-
continue;
|
|
2381
|
-
}
|
|
2382
|
-
lastFingerprint = fp;
|
|
2383
|
-
if (fp !== previousFingerprint) {
|
|
2384
|
-
// Stability confirmation
|
|
2385
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
|
|
2386
|
-
try {
|
|
2387
|
-
const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
2388
|
-
const confirmFp = confirmRes?.fingerprint ?? null;
|
|
2389
|
-
lastActivity = confirmRes?.activity ?? lastActivity;
|
|
2390
|
-
if (confirmFp === fp) {
|
|
2391
|
-
return {
|
|
2392
|
-
success: true,
|
|
2393
|
-
previousFingerprint,
|
|
2394
|
-
newFingerprint: fp,
|
|
2395
|
-
elapsedMs: Date.now() - start,
|
|
2396
|
-
observed_screen: {
|
|
2397
|
-
fingerprint: fp,
|
|
2398
|
-
activity: lastActivity
|
|
2399
|
-
}
|
|
2400
|
-
};
|
|
2401
|
-
}
|
|
2402
|
-
lastFingerprint = confirmFp;
|
|
2403
|
-
continue;
|
|
2404
|
-
}
|
|
2405
|
-
catch (e) {
|
|
2406
|
-
checkDebuggerGuard();
|
|
2407
|
-
console.error('Error confirming fingerprint:', e);
|
|
2408
|
-
continue;
|
|
2409
|
-
}
|
|
2410
|
-
}
|
|
2411
|
-
}
|
|
2412
|
-
catch (e) {
|
|
2413
|
-
checkDebuggerGuard();
|
|
2414
|
-
console.error('Error getting screen fingerprint:', e);
|
|
2415
|
-
}
|
|
2416
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
|
|
2417
|
-
}
|
|
2418
|
-
return {
|
|
2419
|
-
success: false,
|
|
2420
|
-
reason: 'timeout',
|
|
2421
|
-
previousFingerprint,
|
|
2422
|
-
lastFingerprint,
|
|
2423
|
-
elapsedMs: Date.now() - start,
|
|
2424
|
-
observed_screen: {
|
|
2425
|
-
fingerprint: lastFingerprint,
|
|
2426
|
-
activity: lastActivity
|
|
2427
|
-
}
|
|
2428
|
-
};
|
|
177
|
+
static async waitForScreenChangeHandler(input) {
|
|
178
|
+
return waitForScreenChange(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target) });
|
|
2429
179
|
}
|
|
2430
|
-
static async waitForUIChangeHandler(
|
|
2431
|
-
|
|
2432
|
-
const start = Date.now();
|
|
2433
|
-
const pollIntervalMs = 300;
|
|
2434
|
-
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300);
|
|
2435
|
-
let baseline = null;
|
|
2436
|
-
let baselineScope = null;
|
|
2437
|
-
let lastObservedRevision = null;
|
|
2438
|
-
let lastLoadingState = null;
|
|
2439
|
-
let lastSnapshotFreshnessMs = null;
|
|
2440
|
-
let candidateSignatures = null;
|
|
2441
|
-
let candidateObservedChange = null;
|
|
2442
|
-
let candidateSinceMs = null;
|
|
2443
|
-
let lastChangeSummary = null;
|
|
2444
|
-
let lastScopeResolution = {
|
|
2445
|
-
scope: scope === 'subtree' ? 'subtree' : 'screen',
|
|
2446
|
-
target: target && typeof target === 'string' ? target : null,
|
|
2447
|
-
resolved: scope !== 'subtree',
|
|
2448
|
-
resolvedIndex: null,
|
|
2449
|
-
resolvedStableId: null,
|
|
2450
|
-
reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
|
|
2451
|
-
};
|
|
2452
|
-
while (Date.now() - start < timeout_ms) {
|
|
2453
|
-
try {
|
|
2454
|
-
if (readinessTracker)
|
|
2455
|
-
remainingBudget();
|
|
2456
|
-
const acquisitionStarted = Date.now();
|
|
2457
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
2458
|
-
observeReadinessWait(tree, acquisitionStarted);
|
|
2459
|
-
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target);
|
|
2460
|
-
if (scopedTree.error) {
|
|
2461
|
-
lastScopeResolution = scopedTree.resolution;
|
|
2462
|
-
return {
|
|
2463
|
-
success: false,
|
|
2464
|
-
observed_change: null,
|
|
2465
|
-
snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
|
|
2466
|
-
snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
|
|
2467
|
-
timeout: true,
|
|
2468
|
-
elapsed_ms: Date.now() - start,
|
|
2469
|
-
expected_change,
|
|
2470
|
-
loading_state: tree?.loading_state ?? lastLoadingState ?? null,
|
|
2471
|
-
scope: scopedTree.resolution.scope,
|
|
2472
|
-
target: scopedTree.resolution.target,
|
|
2473
|
-
stability_state: 'transient',
|
|
2474
|
-
change_summary: lastChangeSummary,
|
|
2475
|
-
reason: scopedTree.error.message,
|
|
2476
|
-
error: scopedTree.error
|
|
2477
|
-
};
|
|
2478
|
-
}
|
|
2479
|
-
const scopedElements = scopedTree.elements;
|
|
2480
|
-
const scopedSignatureTree = {
|
|
2481
|
-
...tree,
|
|
2482
|
-
elements: scopedElements
|
|
2483
|
-
};
|
|
2484
|
-
const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree);
|
|
2485
|
-
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
|
|
2486
|
-
lastLoadingState = tree?.loading_state ?? lastLoadingState;
|
|
2487
|
-
lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs;
|
|
2488
|
-
lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary;
|
|
2489
|
-
lastScopeResolution = scopedTree.resolution;
|
|
2490
|
-
baselineScope = baselineScope ?? scopedTree;
|
|
2491
|
-
if (!baseline) {
|
|
2492
|
-
baseline = signatures;
|
|
2493
|
-
baselineScope = scopedTree;
|
|
2494
|
-
}
|
|
2495
|
-
else {
|
|
2496
|
-
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
|
|
2497
|
-
if (observedChange) {
|
|
2498
|
-
if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
|
|
2499
|
-
candidateSignatures = signatures;
|
|
2500
|
-
candidateObservedChange = observedChange;
|
|
2501
|
-
candidateSinceMs = Date.now();
|
|
2502
|
-
}
|
|
2503
|
-
const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs;
|
|
2504
|
-
if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
|
|
2505
|
-
return {
|
|
2506
|
-
success: true,
|
|
2507
|
-
observed_change: candidateObservedChange ?? observedChange,
|
|
2508
|
-
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2509
|
-
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2510
|
-
timeout: false,
|
|
2511
|
-
elapsed_ms: Date.now() - start,
|
|
2512
|
-
expected_change,
|
|
2513
|
-
loading_state: lastLoadingState ?? null,
|
|
2514
|
-
scope: lastScopeResolution.scope,
|
|
2515
|
-
target: lastScopeResolution.target,
|
|
2516
|
-
stability_state: 'stable',
|
|
2517
|
-
change_summary: lastChangeSummary,
|
|
2518
|
-
reason: 'UI change observed'
|
|
2519
|
-
};
|
|
2520
|
-
}
|
|
2521
|
-
}
|
|
2522
|
-
else {
|
|
2523
|
-
candidateSignatures = null;
|
|
2524
|
-
candidateObservedChange = null;
|
|
2525
|
-
candidateSinceMs = null;
|
|
2526
|
-
}
|
|
2527
|
-
}
|
|
2528
|
-
}
|
|
2529
|
-
catch (error) {
|
|
2530
|
-
checkDebuggerGuard();
|
|
2531
|
-
if (readinessTracker && !String(error).includes('READINESS_'))
|
|
2532
|
-
readinessTracker.unavailable();
|
|
2533
|
-
if (readinessTracker && isDeviceUnavailable(error))
|
|
2534
|
-
throw error;
|
|
2535
|
-
if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) {
|
|
2536
|
-
readinessTracker.deadline();
|
|
2537
|
-
break;
|
|
2538
|
-
}
|
|
2539
|
-
// Default waits retain best-effort polling.
|
|
2540
|
-
}
|
|
2541
|
-
await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)));
|
|
2542
|
-
}
|
|
2543
|
-
return {
|
|
2544
|
-
success: false,
|
|
2545
|
-
observed_change: null,
|
|
2546
|
-
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2547
|
-
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2548
|
-
timeout: true,
|
|
2549
|
-
elapsed_ms: Date.now() - start,
|
|
2550
|
-
expected_change,
|
|
2551
|
-
loading_state: lastLoadingState ?? null,
|
|
2552
|
-
scope: lastScopeResolution.scope,
|
|
2553
|
-
target: lastScopeResolution.target,
|
|
2554
|
-
stability_state: 'transient',
|
|
2555
|
-
change_summary: lastChangeSummary,
|
|
2556
|
-
reason: 'timeout'
|
|
2557
|
-
};
|
|
180
|
+
static async waitForUIChangeHandler(input) {
|
|
181
|
+
return waitForUIChange(input, { observe: target => ToolsObserve.getUITreeHandler(target), scope: (tree, scope, target) => ToolsInteract._resolveUiChangeScope(tree, scope, target) });
|
|
2558
182
|
}
|
|
2559
|
-
static async expectScreenHandler(
|
|
2560
|
-
|
|
2561
|
-
const observedScreen = {
|
|
2562
|
-
fingerprint: observedFingerprint?.fingerprint ?? null,
|
|
2563
|
-
screen: observedFingerprint?.activity ?? null
|
|
2564
|
-
};
|
|
2565
|
-
let observedScreenLabel = observedScreen.screen;
|
|
2566
|
-
if (!fingerprint && screen && platform !== 'ios') {
|
|
2567
|
-
try {
|
|
2568
|
-
const current = await ToolsObserve.getCurrentScreenHandler({ deviceId });
|
|
2569
|
-
observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel;
|
|
2570
|
-
}
|
|
2571
|
-
catch {
|
|
2572
|
-
// Keep fingerprint-derived activity when current-screen lookup is unavailable.
|
|
2573
|
-
}
|
|
2574
|
-
}
|
|
2575
|
-
const expectedScreen = {
|
|
2576
|
-
fingerprint: fingerprint ?? null,
|
|
2577
|
-
screen: screen ?? null
|
|
2578
|
-
};
|
|
2579
|
-
let success = false;
|
|
2580
|
-
let basis = 'none';
|
|
2581
|
-
let reason = 'No fingerprint or screen expectation provided';
|
|
2582
|
-
if (fingerprint) {
|
|
2583
|
-
basis = 'fingerprint';
|
|
2584
|
-
success = observedScreen.fingerprint === fingerprint;
|
|
2585
|
-
reason = success
|
|
2586
|
-
? `observed fingerprint matches expected fingerprint ${fingerprint}`
|
|
2587
|
-
: `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`;
|
|
2588
|
-
}
|
|
2589
|
-
else if (screen) {
|
|
2590
|
-
basis = 'screen';
|
|
2591
|
-
const candidates = new Set();
|
|
2592
|
-
if (observedScreen.screen)
|
|
2593
|
-
candidates.add(observedScreen.screen);
|
|
2594
|
-
if (observedScreenLabel)
|
|
2595
|
-
candidates.add(observedScreenLabel);
|
|
2596
|
-
success = candidates.has(screen);
|
|
2597
|
-
reason = success
|
|
2598
|
-
? `observed screen matches expected screen ${screen}`
|
|
2599
|
-
: `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`;
|
|
2600
|
-
}
|
|
2601
|
-
return {
|
|
2602
|
-
success,
|
|
2603
|
-
observed_screen: {
|
|
2604
|
-
fingerprint: observedScreen.fingerprint,
|
|
2605
|
-
screen: observedScreenLabel
|
|
2606
|
-
},
|
|
2607
|
-
expected_screen: expectedScreen,
|
|
2608
|
-
confidence: success ? 1 : 0,
|
|
2609
|
-
comparison: {
|
|
2610
|
-
basis,
|
|
2611
|
-
matched: success,
|
|
2612
|
-
reason
|
|
2613
|
-
},
|
|
2614
|
-
trace: buildObservationTrace({
|
|
2615
|
-
actionType: 'expect_screen',
|
|
2616
|
-
stage: 'verify',
|
|
2617
|
-
success,
|
|
2618
|
-
attempts: 1,
|
|
2619
|
-
metadata: {
|
|
2620
|
-
expected_screen: expectedScreen,
|
|
2621
|
-
observed_screen: {
|
|
2622
|
-
fingerprint: observedScreen.fingerprint,
|
|
2623
|
-
screen: observedScreenLabel
|
|
2624
|
-
},
|
|
2625
|
-
comparison: {
|
|
2626
|
-
basis,
|
|
2627
|
-
matched: success
|
|
2628
|
-
}
|
|
2629
|
-
}
|
|
2630
|
-
})
|
|
2631
|
-
};
|
|
183
|
+
static async expectScreenHandler(input) {
|
|
184
|
+
return expectScreen(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), current: target => ToolsObserve.getCurrentScreenHandler(target) });
|
|
2632
185
|
}
|
|
2633
|
-
static async expectElementVisibleHandler(
|
|
2634
|
-
|
|
2635
|
-
selector,
|
|
2636
|
-
condition: 'visible',
|
|
2637
|
-
timeout_ms,
|
|
2638
|
-
poll_interval_ms,
|
|
2639
|
-
platform,
|
|
2640
|
-
deviceId
|
|
2641
|
-
});
|
|
2642
|
-
if (result?.status === 'success' && result?.element) {
|
|
2643
|
-
return {
|
|
2644
|
-
success: true,
|
|
2645
|
-
selector,
|
|
2646
|
-
element_id: result.element.elementId ?? element_id ?? null,
|
|
2647
|
-
expected_condition: 'visible',
|
|
2648
|
-
element: {
|
|
2649
|
-
elementId: result.element.elementId ?? null,
|
|
2650
|
-
text: result.element.text ?? null,
|
|
2651
|
-
resource_id: result.element.resource_id ?? null,
|
|
2652
|
-
accessibility_id: result.element.accessibility_id ?? null,
|
|
2653
|
-
class: result.element.class ?? null,
|
|
2654
|
-
bounds: result.element.bounds ?? null,
|
|
2655
|
-
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
2656
|
-
state: result.element.state ?? null,
|
|
2657
|
-
stable_id: result.element.stable_id ?? null,
|
|
2658
|
-
role: result.element.role ?? null,
|
|
2659
|
-
test_tag: result.element.test_tag ?? null,
|
|
2660
|
-
selector: result.element.selector ?? null,
|
|
2661
|
-
semantic: result.element.semantic ?? null
|
|
2662
|
-
},
|
|
2663
|
-
observed: {
|
|
2664
|
-
status: result.status,
|
|
2665
|
-
matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
|
|
2666
|
-
condition_satisfied: true,
|
|
2667
|
-
selected_index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
2668
|
-
last_matched_element: {
|
|
2669
|
-
elementId: result.element.elementId ?? null,
|
|
2670
|
-
text: result.element.text ?? null,
|
|
2671
|
-
resource_id: result.element.resource_id ?? null,
|
|
2672
|
-
accessibility_id: result.element.accessibility_id ?? null,
|
|
2673
|
-
class: result.element.class ?? null,
|
|
2674
|
-
bounds: result.element.bounds ?? null,
|
|
2675
|
-
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
2676
|
-
state: result.element.state ?? null,
|
|
2677
|
-
stable_id: result.element.stable_id ?? null,
|
|
2678
|
-
role: result.element.role ?? null,
|
|
2679
|
-
test_tag: result.element.test_tag ?? null,
|
|
2680
|
-
selector: result.element.selector ?? null,
|
|
2681
|
-
semantic: result.element.semantic ?? null
|
|
2682
|
-
}
|
|
2683
|
-
},
|
|
2684
|
-
reason: 'selector is visible',
|
|
2685
|
-
trace: buildObservationTrace({
|
|
2686
|
-
actionType: 'expect_element_visible',
|
|
2687
|
-
stage: 'verify',
|
|
2688
|
-
success: true,
|
|
2689
|
-
attempts: 1,
|
|
2690
|
-
metadata: {
|
|
2691
|
-
selector,
|
|
2692
|
-
element_id: result.element.elementId ?? element_id ?? null,
|
|
2693
|
-
status: result.status
|
|
2694
|
-
}
|
|
2695
|
-
})
|
|
2696
|
-
};
|
|
2697
|
-
}
|
|
2698
|
-
const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT';
|
|
2699
|
-
return {
|
|
2700
|
-
success: false,
|
|
2701
|
-
selector,
|
|
2702
|
-
element_id: element_id ?? null,
|
|
2703
|
-
expected_condition: 'visible',
|
|
2704
|
-
observed: {
|
|
2705
|
-
status: result?.status,
|
|
2706
|
-
matched_count: result?.observed?.matched_count,
|
|
2707
|
-
condition_satisfied: result?.observed?.condition_satisfied ?? false,
|
|
2708
|
-
selected_index: result?.observed?.selected_index ?? null,
|
|
2709
|
-
last_matched_element: result?.observed?.last_matched_element ?? null
|
|
2710
|
-
},
|
|
2711
|
-
reason: result?.error?.message ?? 'selector is not visible',
|
|
2712
|
-
failure_code: errorCode,
|
|
2713
|
-
retryable: errorCode === 'TIMEOUT',
|
|
2714
|
-
trace: buildObservationTrace({
|
|
2715
|
-
actionType: 'expect_element_visible',
|
|
2716
|
-
stage: 'verify',
|
|
2717
|
-
success: false,
|
|
2718
|
-
attempts: 1,
|
|
2719
|
-
metadata: {
|
|
2720
|
-
selector,
|
|
2721
|
-
element_id: element_id ?? null,
|
|
2722
|
-
status: result?.status ?? null,
|
|
2723
|
-
reason: result?.error?.message ?? 'selector is not visible'
|
|
2724
|
-
}
|
|
2725
|
-
})
|
|
2726
|
-
};
|
|
186
|
+
static async expectElementVisibleHandler(input) {
|
|
187
|
+
return expectVisible(input, request => ToolsInteract.waitForUIHandler(request));
|
|
2727
188
|
}
|
|
2728
|
-
static async expectStateHandler(
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
2733
|
-
const start = Date.now();
|
|
2734
|
-
const deadline = start + (exact_budget ? stabilization_window_ms : Math.max(500, stabilization_window_ms));
|
|
2735
|
-
const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2));
|
|
2736
|
-
const pollDelay = exact_budget ? Math.max(50, Math.min(poll_interval_ms || 150, 1000)) : Math.max(100, Math.min(poll_interval_ms || 150, 200));
|
|
2737
|
-
const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800));
|
|
2738
|
-
let attempts = 0;
|
|
2739
|
-
let stableCount = 0;
|
|
2740
|
-
let lastReason = 'element not found';
|
|
2741
|
-
let lastFailureCode = 'ELEMENT_NOT_FOUND';
|
|
2742
|
-
let lastObservedElement = null;
|
|
2743
|
-
let lastObservedValue = null;
|
|
2744
|
-
let lastRawValue = null;
|
|
2745
|
-
let lastResolvedElementId = element_id ?? null;
|
|
2746
|
-
const traceSteps = [];
|
|
2747
|
-
let traceAttemptIndex = 0;
|
|
2748
|
-
let resolveRecorded = false;
|
|
2749
|
-
const recordTraceStep = (stage, result, metadata) => {
|
|
2750
|
-
traceSteps.push(createTraceStep({
|
|
2751
|
-
stage,
|
|
2752
|
-
timestamp: Date.now(),
|
|
2753
|
-
result,
|
|
2754
|
-
attemptIndex: traceAttemptIndex++,
|
|
2755
|
-
metadata
|
|
2756
|
-
}));
|
|
2757
|
-
};
|
|
2758
|
-
const buildStateTrace = (outcome) => ({
|
|
2759
|
-
action_id: nextActionId('expect_state', Date.now()),
|
|
2760
|
-
steps: traceSteps,
|
|
2761
|
-
final_outcome: outcome,
|
|
2762
|
-
attempts
|
|
2763
|
-
});
|
|
2764
|
-
while (Date.now() <= deadline) {
|
|
2765
|
-
attempts++;
|
|
2766
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
2767
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
2768
|
-
const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
|
|
2769
|
-
const treeDeviceId = tree?.device?.id || deviceId;
|
|
2770
|
-
const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null;
|
|
2771
|
-
let matched = null;
|
|
2772
|
-
if (element_id) {
|
|
2773
|
-
const resolved = ToolsInteract._resolvedUiElements.get(element_id);
|
|
2774
|
-
if (resolved) {
|
|
2775
|
-
const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
|
|
2776
|
-
if (current)
|
|
2777
|
-
matched = { el: current.el, idx: current.index };
|
|
2778
|
-
}
|
|
2779
|
-
}
|
|
2780
|
-
if (!matched && selector) {
|
|
2781
|
-
matched = ToolsInteract._findFirstMatchingElement(elements, selector);
|
|
2782
|
-
}
|
|
2783
|
-
if (!matched) {
|
|
2784
|
-
lastReason = 'element not found';
|
|
2785
|
-
lastFailureCode = 'ELEMENT_NOT_FOUND';
|
|
2786
|
-
stableCount = 0;
|
|
2787
|
-
recordTraceStep('resolve', 'retry', {
|
|
2788
|
-
selector: selector ?? null,
|
|
2789
|
-
element_id: lastResolvedElementId,
|
|
2790
|
-
matched: false,
|
|
2791
|
-
reason: lastReason,
|
|
2792
|
-
attempt: attempts
|
|
2793
|
-
});
|
|
2794
|
-
recordTraceStep('stabilize', 'retry', {
|
|
2795
|
-
stabilization_attempts: attempts,
|
|
2796
|
-
stable_observation_count: stableCount,
|
|
2797
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
2798
|
-
reason: lastReason
|
|
2799
|
-
});
|
|
2800
|
-
await sleep(pollDelay);
|
|
2801
|
-
continue;
|
|
2802
|
-
}
|
|
2803
|
-
const resolvedElement = ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx), matched.el, matched.idx);
|
|
2804
|
-
lastResolvedElementId = resolvedElement.elementId;
|
|
2805
|
-
lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null };
|
|
2806
|
-
if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
|
|
2807
|
-
lastReason = 'stale snapshot';
|
|
2808
|
-
lastFailureCode = 'UNKNOWN';
|
|
2809
|
-
stableCount = 0;
|
|
2810
|
-
recordTraceStep('resolve', 'retry', {
|
|
2811
|
-
selector: selector ?? null,
|
|
2812
|
-
element_id: lastResolvedElementId,
|
|
2813
|
-
matched: true,
|
|
2814
|
-
reason: lastReason,
|
|
2815
|
-
attempt: attempts
|
|
2816
|
-
});
|
|
2817
|
-
recordTraceStep('stabilize', 'retry', {
|
|
2818
|
-
stabilization_attempts: attempts,
|
|
2819
|
-
stable_observation_count: stableCount,
|
|
2820
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
2821
|
-
reason: lastReason
|
|
2822
|
-
});
|
|
2823
|
-
await sleep(pollDelay);
|
|
2824
|
-
continue;
|
|
2825
|
-
}
|
|
2826
|
-
if (!resolveRecorded) {
|
|
2827
|
-
recordTraceStep('resolve', 'success', {
|
|
2828
|
-
selector: selector ?? null,
|
|
2829
|
-
element_id: lastResolvedElementId,
|
|
2830
|
-
matched: true,
|
|
2831
|
-
reason: 'element resolved'
|
|
2832
|
-
});
|
|
2833
|
-
resolveRecorded = true;
|
|
2834
|
-
}
|
|
2835
|
-
const observedState = matched.el.state ?? null;
|
|
2836
|
-
const actual = observedState?.[property] ?? null;
|
|
2837
|
-
let success = false;
|
|
2838
|
-
let reason = '';
|
|
2839
|
-
let rawValue = null;
|
|
2840
|
-
let observedValue = actual;
|
|
2841
|
-
switch (property) {
|
|
2842
|
-
case 'checked':
|
|
2843
|
-
case 'focused':
|
|
2844
|
-
case 'expanded':
|
|
2845
|
-
case 'enabled': {
|
|
2846
|
-
const expectedBool = compareBoolean(expected);
|
|
2847
|
-
const actualBool = compareBoolean(actual);
|
|
2848
|
-
if (expectedBool === null) {
|
|
2849
|
-
reason = `expected ${property} must be boolean`;
|
|
2850
|
-
}
|
|
2851
|
-
else if (actualBool === null) {
|
|
2852
|
-
reason = `${property} state unavailable`;
|
|
2853
|
-
}
|
|
2854
|
-
else {
|
|
2855
|
-
rawValue = actualBool;
|
|
2856
|
-
success = actualBool === expectedBool;
|
|
2857
|
-
reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`;
|
|
2858
|
-
}
|
|
2859
|
-
observedValue = actualBool;
|
|
2860
|
-
break;
|
|
2861
|
-
}
|
|
2862
|
-
case 'value':
|
|
2863
|
-
case 'raw_value': {
|
|
2864
|
-
const expectedNumber = compareNumber(expected);
|
|
2865
|
-
const actualNumber = compareNumber(actual);
|
|
2866
|
-
if (expectedNumber !== null && actualNumber !== null) {
|
|
2867
|
-
success = actualNumber === expectedNumber;
|
|
2868
|
-
rawValue = actualNumber;
|
|
2869
|
-
observedValue = actualNumber;
|
|
2870
|
-
reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`;
|
|
2871
|
-
break;
|
|
2872
|
-
}
|
|
2873
|
-
const expectedString = typeof expected === 'string' ? expected : null;
|
|
2874
|
-
const actualString = compareString(actual);
|
|
2875
|
-
if (expectedString !== null && actualString !== null) {
|
|
2876
|
-
success = actualString === expectedString;
|
|
2877
|
-
rawValue = actualString;
|
|
2878
|
-
observedValue = actualString;
|
|
2879
|
-
reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`;
|
|
2880
|
-
}
|
|
2881
|
-
else {
|
|
2882
|
-
reason = 'value state unavailable';
|
|
2883
|
-
}
|
|
2884
|
-
break;
|
|
2885
|
-
}
|
|
2886
|
-
case 'selected': {
|
|
2887
|
-
const expectedBool = typeof expected === 'boolean' ? expected : null;
|
|
2888
|
-
const expectedString = typeof expected === 'string'
|
|
2889
|
-
? expected
|
|
2890
|
-
: expected && typeof expected === 'object'
|
|
2891
|
-
? String(expected.id ?? expected.label ?? '')
|
|
2892
|
-
: null;
|
|
2893
|
-
if (!observedState || observedState.selected === undefined || observedState.selected === null) {
|
|
2894
|
-
reason = 'selected state unavailable';
|
|
2895
|
-
break;
|
|
2896
|
-
}
|
|
2897
|
-
if (expectedBool !== null) {
|
|
2898
|
-
const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null;
|
|
2899
|
-
if (actualBool === null) {
|
|
2900
|
-
reason = 'selected state is not boolean';
|
|
2901
|
-
break;
|
|
2902
|
-
}
|
|
2903
|
-
rawValue = actualBool;
|
|
2904
|
-
observedValue = actualBool;
|
|
2905
|
-
success = actualBool === expectedBool;
|
|
2906
|
-
reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`;
|
|
2907
|
-
break;
|
|
2908
|
-
}
|
|
2909
|
-
const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
|
|
2910
|
-
? String(observedState.selected.id ?? observedState.selected.label ?? '')
|
|
2911
|
-
: String(observedState.selected);
|
|
2912
|
-
const actualString = actualSelected.trim();
|
|
2913
|
-
if (!expectedString) {
|
|
2914
|
-
reason = 'expected selected must be boolean, string, or object with id/label';
|
|
2915
|
-
break;
|
|
2916
|
-
}
|
|
2917
|
-
rawValue = actualString;
|
|
2918
|
-
observedValue = actualString;
|
|
2919
|
-
success = actualString === expectedString;
|
|
2920
|
-
reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`;
|
|
2921
|
-
break;
|
|
2922
|
-
}
|
|
2923
|
-
case 'text_value': {
|
|
2924
|
-
const expectedString = typeof expected === 'string' ? expected : null;
|
|
2925
|
-
const actualString = compareString(actual);
|
|
2926
|
-
if (!expectedString) {
|
|
2927
|
-
reason = 'expected text_value must be string';
|
|
2928
|
-
}
|
|
2929
|
-
else if (!actualString) {
|
|
2930
|
-
reason = 'text_value state unavailable';
|
|
2931
|
-
}
|
|
2932
|
-
else {
|
|
2933
|
-
success = actualString === expectedString;
|
|
2934
|
-
rawValue = actualString;
|
|
2935
|
-
observedValue = actualString;
|
|
2936
|
-
reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`;
|
|
2937
|
-
}
|
|
2938
|
-
break;
|
|
2939
|
-
}
|
|
2940
|
-
default: {
|
|
2941
|
-
if (actual !== null && actual !== undefined) {
|
|
2942
|
-
success = actual === expected;
|
|
2943
|
-
observedValue = actual;
|
|
2944
|
-
rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null;
|
|
2945
|
-
reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`;
|
|
2946
|
-
}
|
|
2947
|
-
else {
|
|
2948
|
-
reason = `unsupported or unavailable state property: ${property}`;
|
|
2949
|
-
}
|
|
2950
|
-
}
|
|
2951
|
-
}
|
|
2952
|
-
if (success) {
|
|
2953
|
-
stableCount++;
|
|
2954
|
-
recordTraceStep('stabilize', 'success', {
|
|
2955
|
-
stabilization_attempts: attempts,
|
|
2956
|
-
stable_observation_count: stableCount,
|
|
2957
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
2958
|
-
reason
|
|
2959
|
-
});
|
|
2960
|
-
recordTraceStep('verify', 'success', {
|
|
2961
|
-
property,
|
|
2962
|
-
expected,
|
|
2963
|
-
observed_value: observedValue,
|
|
2964
|
-
reason
|
|
2965
|
-
});
|
|
2966
|
-
if (stableCount >= stableTarget) {
|
|
2967
|
-
return {
|
|
2968
|
-
success: true,
|
|
2969
|
-
selector,
|
|
2970
|
-
element_id: lastResolvedElementId,
|
|
2971
|
-
expected_state: { property, expected },
|
|
2972
|
-
element: lastObservedElement,
|
|
2973
|
-
observed_state: {
|
|
2974
|
-
property,
|
|
2975
|
-
value: observedValue,
|
|
2976
|
-
...(rawValue !== null ? { raw_value: rawValue } : {})
|
|
2977
|
-
},
|
|
2978
|
-
reason,
|
|
2979
|
-
stabilization_attempts: attempts,
|
|
2980
|
-
stabilization_window_ms: Date.now() - start,
|
|
2981
|
-
stable_observation_count: stableCount,
|
|
2982
|
-
snapshot_freshness_ms: treeAgeMs ?? undefined,
|
|
2983
|
-
trace: buildStateTrace('success')
|
|
2984
|
-
};
|
|
2985
|
-
}
|
|
2986
|
-
}
|
|
2987
|
-
else {
|
|
2988
|
-
stableCount = 0;
|
|
2989
|
-
lastReason = reason || lastReason;
|
|
2990
|
-
lastFailureCode = 'UNKNOWN';
|
|
2991
|
-
recordTraceStep('stabilize', 'retry', {
|
|
2992
|
-
stabilization_attempts: attempts,
|
|
2993
|
-
stable_observation_count: stableCount,
|
|
2994
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
2995
|
-
reason: lastReason
|
|
2996
|
-
});
|
|
2997
|
-
recordTraceStep('verify', 'retry', {
|
|
2998
|
-
property,
|
|
2999
|
-
expected,
|
|
3000
|
-
observed_value: observedValue,
|
|
3001
|
-
reason: lastReason
|
|
3002
|
-
});
|
|
3003
|
-
}
|
|
3004
|
-
if (!success) {
|
|
3005
|
-
lastObservedValue = observedValue;
|
|
3006
|
-
lastRawValue = rawValue;
|
|
3007
|
-
}
|
|
3008
|
-
await sleep(pollDelay);
|
|
3009
|
-
}
|
|
3010
|
-
return {
|
|
3011
|
-
success: false,
|
|
3012
|
-
selector,
|
|
3013
|
-
element_id: lastResolvedElementId,
|
|
3014
|
-
expected_state: { property, expected },
|
|
3015
|
-
element: lastObservedElement,
|
|
3016
|
-
observed_state: {
|
|
3017
|
-
property,
|
|
3018
|
-
value: lastObservedValue,
|
|
3019
|
-
...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
|
|
3020
|
-
},
|
|
3021
|
-
reason: lastReason,
|
|
3022
|
-
failure_code: lastFailureCode,
|
|
3023
|
-
retryable: true,
|
|
3024
|
-
trace: buildStateTrace('failure')
|
|
3025
|
-
};
|
|
189
|
+
static async expectStateHandler(input) {
|
|
190
|
+
return expectState(input, { observe: target => ToolsObserve.getUITreeHandler(target), getHandle: id => ToolsInteract._resolvedUiElements.get(id),
|
|
191
|
+
currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
|
|
192
|
+
target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx) });
|
|
3026
193
|
}
|
|
3027
|
-
static async waitForUICore(
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
// Clamp polling interval to 250-500ms for consistent behavior
|
|
3032
|
-
const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500));
|
|
3033
|
-
// Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
|
|
3034
|
-
let initialFingerprint = null;
|
|
3035
|
-
let baselineLastLine = null;
|
|
3036
|
-
try {
|
|
3037
|
-
const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
3038
|
-
const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
|
|
3039
|
-
const withTimeout = (p, ms) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))]);
|
|
3040
|
-
const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)]);
|
|
3041
|
-
if (fpRes && typeof fpRes === 'object')
|
|
3042
|
-
initialFingerprint = fpRes.fingerprint ?? null;
|
|
3043
|
-
if (gl) {
|
|
3044
|
-
const logsArr = Array.isArray(gl.logs) ? gl.logs : [];
|
|
3045
|
-
// Normalize to last message string for baseline comparison
|
|
3046
|
-
const msgs = ToolsInteract._logsToMessages(logsArr);
|
|
3047
|
-
baselineLastLine = msgs.length ? msgs[msgs.length - 1] : null;
|
|
3048
|
-
}
|
|
3049
|
-
}
|
|
3050
|
-
catch (err) {
|
|
3051
|
-
checkDebuggerGuard();
|
|
3052
|
-
try {
|
|
3053
|
-
console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err));
|
|
3054
|
-
}
|
|
3055
|
-
catch { }
|
|
3056
|
-
}
|
|
3057
|
-
// Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
|
|
3058
|
-
let lastChangeAt = Date.now();
|
|
3059
|
-
let prevFingerprint = initialFingerprint;
|
|
3060
|
-
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
3061
|
-
// Optional initial observation delay requested by caller
|
|
3062
|
-
if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
|
|
3063
|
-
try {
|
|
3064
|
-
console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`);
|
|
3065
|
-
}
|
|
3066
|
-
catch { }
|
|
3067
|
-
await sleep(observationDelayMs);
|
|
3068
|
-
}
|
|
3069
|
-
// Telemetry
|
|
3070
|
-
let pollCount = 0;
|
|
3071
|
-
let matchedAt = null;
|
|
3072
|
-
let lastObservedState = null;
|
|
3073
|
-
let stableDuration = 0;
|
|
3074
|
-
let matchSource = null;
|
|
3075
|
-
while (Date.now() <= deadline) {
|
|
3076
|
-
pollCount++;
|
|
3077
|
-
const now = Date.now();
|
|
3078
|
-
// Evaluate condition per type
|
|
3079
|
-
if (type === 'ui') {
|
|
3080
|
-
try {
|
|
3081
|
-
// Prefer using the public findElementHandler which tests can override. This avoids relying
|
|
3082
|
-
// on resolveObserve/getUITree for unit tests which may not have devices available.
|
|
3083
|
-
try {
|
|
3084
|
-
const findRes = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId });
|
|
3085
|
-
const isPresent = !!(findRes && findRes.found);
|
|
3086
|
-
const conditionTrue = (match === 'present') ? isPresent : !isPresent;
|
|
3087
|
-
if (conditionTrue) {
|
|
3088
|
-
if (matchedAt === null)
|
|
3089
|
-
matchedAt = Date.now();
|
|
3090
|
-
stableDuration = Date.now() - matchedAt;
|
|
3091
|
-
lastObservedState = true;
|
|
3092
|
-
if (stableDuration >= stability_ms) {
|
|
3093
|
-
matchSource = 'ui-find';
|
|
3094
|
-
const element = isPresent ? findRes.element : null;
|
|
3095
|
-
const now2 = Date.now();
|
|
3096
|
-
return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null };
|
|
3097
|
-
}
|
|
3098
|
-
}
|
|
3099
|
-
else {
|
|
3100
|
-
matchedAt = null;
|
|
3101
|
-
stableDuration = 0;
|
|
3102
|
-
lastObservedState = false;
|
|
3103
|
-
}
|
|
3104
|
-
}
|
|
3105
|
-
catch (err) {
|
|
3106
|
-
checkDebuggerGuard();
|
|
3107
|
-
console.error('waitForUI(ui) find error:', err);
|
|
3108
|
-
}
|
|
3109
|
-
}
|
|
3110
|
-
catch (err) {
|
|
3111
|
-
checkDebuggerGuard();
|
|
3112
|
-
console.error('waitForUI(ui) outer error:', err);
|
|
3113
|
-
}
|
|
3114
|
-
}
|
|
3115
|
-
else if (type === 'log') {
|
|
3116
|
-
try {
|
|
3117
|
-
// Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
|
|
3118
|
-
const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 });
|
|
3119
|
-
const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : [];
|
|
3120
|
-
for (const ent of entries) {
|
|
3121
|
-
const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : '';
|
|
3122
|
-
if (q && String(msg).includes(q)) {
|
|
3123
|
-
const now2 = Date.now();
|
|
3124
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true };
|
|
3125
|
-
}
|
|
3126
|
-
}
|
|
3127
|
-
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
|
|
3128
|
-
const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : [];
|
|
3129
|
-
// Normalize to messages for comparison
|
|
3130
|
-
const msgs = ToolsInteract._logsToMessages(logsArr);
|
|
3131
|
-
let startIndex = 0;
|
|
3132
|
-
if (baselineLastLine) {
|
|
3133
|
-
const idx = msgs.lastIndexOf(baselineLastLine);
|
|
3134
|
-
startIndex = idx >= 0 ? idx + 1 : 0;
|
|
3135
|
-
}
|
|
3136
|
-
for (let i = startIndex; i < msgs.length; i++) {
|
|
3137
|
-
const line = msgs[i];
|
|
3138
|
-
if (q && String(line).includes(q)) {
|
|
3139
|
-
const now2 = Date.now();
|
|
3140
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true };
|
|
3141
|
-
}
|
|
3142
|
-
}
|
|
3143
|
-
}
|
|
3144
|
-
catch (err) {
|
|
3145
|
-
checkDebuggerGuard();
|
|
3146
|
-
console.error('waitForUI(log) error:', err);
|
|
3147
|
-
}
|
|
3148
|
-
}
|
|
3149
|
-
else if (type === 'screen') {
|
|
3150
|
-
try {
|
|
3151
|
-
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
3152
|
-
const fp = fpRes?.fingerprint ?? null;
|
|
3153
|
-
if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
|
|
3154
|
-
// when screen changed, require stability_ms where fingerprint remains the same
|
|
3155
|
-
if (matchedAt === null)
|
|
3156
|
-
matchedAt = now;
|
|
3157
|
-
const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }))?.fingerprint ?? null;
|
|
3158
|
-
if (confirmFp === fp) {
|
|
3159
|
-
stableDuration = Date.now() - matchedAt;
|
|
3160
|
-
lastObservedState = true;
|
|
3161
|
-
if (stableDuration >= stability_ms) {
|
|
3162
|
-
const now2 = Date.now();
|
|
3163
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null };
|
|
3164
|
-
}
|
|
3165
|
-
}
|
|
3166
|
-
else {
|
|
3167
|
-
matchedAt = null;
|
|
3168
|
-
stableDuration = 0;
|
|
3169
|
-
lastObservedState = false;
|
|
3170
|
-
}
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3173
|
-
catch (err) {
|
|
3174
|
-
checkDebuggerGuard();
|
|
3175
|
-
console.error('waitForUI(screen) error:', err);
|
|
3176
|
-
}
|
|
3177
|
-
}
|
|
3178
|
-
else if (type === 'idle') {
|
|
3179
|
-
try {
|
|
3180
|
-
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
3181
|
-
const fp = fpRes?.fingerprint ?? null;
|
|
3182
|
-
if (fp !== prevFingerprint) {
|
|
3183
|
-
prevFingerprint = fp;
|
|
3184
|
-
lastChangeAt = Date.now();
|
|
3185
|
-
matchedAt = null;
|
|
3186
|
-
stableDuration = 0;
|
|
3187
|
-
lastObservedState = false;
|
|
3188
|
-
}
|
|
3189
|
-
else {
|
|
3190
|
-
const idleMs = Date.now() - lastChangeAt;
|
|
3191
|
-
lastObservedState = true;
|
|
3192
|
-
if (idleMs >= stability_ms) {
|
|
3193
|
-
const now2 = Date.now();
|
|
3194
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null };
|
|
3195
|
-
}
|
|
3196
|
-
}
|
|
3197
|
-
}
|
|
3198
|
-
catch (err) {
|
|
3199
|
-
checkDebuggerGuard();
|
|
3200
|
-
console.error('waitForUI(idle) error:', err);
|
|
3201
|
-
}
|
|
3202
|
-
}
|
|
3203
|
-
// Respect poll interval and avoid tight loop
|
|
3204
|
-
await sleep(pollInterval);
|
|
3205
|
-
}
|
|
3206
|
-
// On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
|
|
3207
|
-
let snapshot = null;
|
|
3208
|
-
if (includeSnapshotOnFailure) {
|
|
3209
|
-
try {
|
|
3210
|
-
// Use dynamic import to avoid circular-initialization issues where the ToolsObserve
|
|
3211
|
-
// binding captured earlier may not reflect test-time overrides. Importing at call
|
|
3212
|
-
// time ensures the latest exported ToolsObserve object is used.
|
|
3213
|
-
const Obs = await import('../observe/index.js');
|
|
3214
|
-
snapshot = await Obs.ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId });
|
|
3215
|
-
}
|
|
3216
|
-
catch (err) {
|
|
3217
|
-
checkDebuggerGuard();
|
|
3218
|
-
snapshot = { error: err instanceof Error ? err.message : String(err) };
|
|
3219
|
-
}
|
|
3220
|
-
}
|
|
3221
|
-
const elapsed = Date.now() - start;
|
|
3222
|
-
return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null };
|
|
194
|
+
static async waitForUICore(input) {
|
|
195
|
+
return waitForUICore(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), logs: target => ToolsObserve.getLogsHandler(target),
|
|
196
|
+
stream: target => ToolsObserve.readLogStreamHandler(target), find: input => ToolsInteract.findElementHandler(input),
|
|
197
|
+
snapshot: async (input) => { const { ToolsObserve } = await import('../observe/index.js'); return ToolsObserve.captureDebugSnapshotHandler(input); } });
|
|
3223
198
|
}
|
|
3224
199
|
}
|