mobile-debug-mcp 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/mcp-builder/SKILL.md +44 -64
- package/.agents/skills/mcp-builder/references/build-flags.md +44 -43
- package/.agents/skills/mcp-builder/references/diagnostics-schema.md +4 -1
- package/.agents/skills/mcp-builder/references/toolchain-details.md +49 -62
- package/.agents/skills/spec-review/references/spec-review-checklist.md +7 -0
- package/.agents/skills/test-authoring/SKILL.md +7 -6
- package/.agents/skills/test-authoring/references/repo-test-layout.md +3 -1
- package/.agents/skills/test-authoring/references/test-authoring-workflow.md +16 -7
- package/.agents/skills/test-authoring/references/test-quality-checklist.md +2 -1
- package/.github/copilot-instructions.md +14 -36
- package/.github/workflows/ci.yml +23 -2
- package/AGENTS.md +12 -3
- package/CONTRIBUTING.md +79 -0
- package/README.md +158 -124
- package/agent-plugin/.claude-plugin/plugin.json +2 -2
- package/agent-plugin/.codex-plugin/plugin.json +1 -1
- package/agent-plugin/.cursor-plugin/plugin.json +2 -2
- package/agent-plugin/.mcp.json +1 -1
- package/agent-plugin/README.md +20 -6
- package/agent-plugin/mcp.json +1 -1
- package/agent-plugin/plugin.json +1 -1
- package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +113 -4
- package/dist/contracts/execution-controls.js +24 -0
- package/dist/contracts/selector.js +8 -0
- package/dist/contracts/target.js +15 -0
- package/dist/contracts/ui.js +1 -0
- package/dist/debugger/android.js +79 -0
- package/dist/debugger/diagnostics.js +54 -0
- package/dist/debugger/guard.js +30 -0
- package/dist/debugger/protocol.js +33 -0
- package/dist/debugger/sessions.js +304 -0
- package/dist/debugger/worker.js +176 -0
- package/dist/interact/adjustment/adjustment-context.js +41 -0
- package/dist/interact/adjustment/adjustment-gestures.js +55 -0
- package/dist/interact/adjustment/adjustment-plan.js +18 -0
- package/dist/interact/adjustment/adjustment-resolution.js +127 -0
- package/dist/interact/adjustment/adjustment-result.js +19 -0
- package/dist/interact/adjustment/adjustment-semantic.js +64 -0
- package/dist/interact/adjustment/adjustment-verification.js +35 -0
- package/dist/interact/adjustment/adjustment.js +52 -0
- package/dist/interact/adjustment/control-geometry.js +87 -0
- package/dist/interact/adjustment/control-values.js +136 -0
- package/dist/interact/adjustment/slider-labels.js +84 -0
- package/dist/interact/assertions/screen-assertion.js +37 -0
- package/dist/interact/assertions/state-assertion-context.js +35 -0
- package/dist/interact/assertions/state-assertion.js +76 -0
- package/dist/interact/assertions/state-comparison.js +141 -0
- package/dist/interact/assertions/visibility-assertion.js +34 -0
- package/dist/interact/conditional-action.js +12 -7
- package/dist/interact/index.js +65 -3074
- package/dist/interact/input-service.js +1 -0
- package/dist/interact/interaction-trace.js +21 -0
- package/dist/interact/model.js +1 -0
- package/dist/interact/observation.js +1 -0
- package/dist/interact/targeting/candidate-ancestor.js +58 -0
- package/dist/interact/targeting/candidate-result.js +23 -0
- package/dist/interact/targeting/candidate-scoring.js +50 -0
- package/dist/interact/targeting/element-helpers.js +216 -0
- package/dist/interact/targeting/find-element.js +65 -0
- package/dist/interact/targeting/nearby-control.js +56 -0
- package/dist/interact/targeting/resolved-elements.js +21 -0
- package/dist/interact/targeting/tap-target.js +17 -0
- package/dist/interact/targeting/target-scope.js +104 -0
- package/dist/interact/waits/legacy-log-wait.js +38 -0
- package/dist/interact/waits/legacy-wait-observations.js +103 -0
- package/dist/interact/waits/legacy-wait-state.js +34 -0
- package/dist/interact/waits/legacy-wait.js +48 -0
- package/dist/interact/waits/screen-wait.js +49 -0
- package/dist/interact/waits/ui-change-helpers.js +157 -0
- package/dist/interact/waits/ui-change-state.js +18 -0
- package/dist/interact/waits/ui-change-wait.js +77 -0
- package/dist/interact/waits/ui-wait-condition.js +56 -0
- package/dist/interact/waits/ui-wait-state.js +38 -0
- package/dist/interact/waits/ui-wait.js +106 -0
- package/dist/manage/android.js +55 -51
- package/dist/manage/index.js +38 -75
- package/dist/manage/ios/build-types.js +1 -0
- package/dist/manage/ios/build.js +210 -0
- package/dist/manage/ios/project-discovery.js +29 -0
- package/dist/manage/ios/scheme.js +18 -0
- package/dist/manage/ios.js +42 -256
- package/dist/observe/android/context.js +1 -0
- package/dist/observe/android/current-screen.js +55 -0
- package/dist/observe/android/fingerprint.js +13 -0
- package/dist/observe/android/log-stream.js +132 -0
- package/dist/observe/android/logs.js +107 -0
- package/dist/observe/android/screenshot.js +70 -0
- package/dist/observe/android/ui-tree.js +107 -0
- package/dist/observe/android.js +16 -520
- package/dist/observe/evidence.js +122 -76
- package/dist/observe/index.js +16 -263
- package/dist/observe/ios/context.js +16 -0
- package/dist/observe/ios/fingerprint.js +12 -0
- package/dist/observe/ios/log-stream.js +105 -0
- package/dist/observe/ios/logs.js +168 -0
- package/dist/observe/ios/screenshot.js +22 -0
- package/dist/observe/ios/ui-elements.js +241 -0
- package/dist/observe/ios/ui-tree.js +137 -0
- package/dist/observe/ios.js +17 -715
- package/dist/observe/logs/model.js +1 -0
- package/dist/{network → observe/network}/index.js +2 -2
- package/dist/observe/readiness-service.js +13 -2
- package/dist/observe/readiness.js +44 -18
- package/dist/observe/screenshot-encoding.js +41 -0
- package/dist/observe/snapshot-collection.js +79 -0
- package/dist/observe/snapshot-logs.js +30 -0
- package/dist/observe/snapshot-metadata.js +43 -31
- package/dist/observe/snapshot-semantic.js +115 -0
- package/dist/observe/system-trace-android.js +135 -0
- package/dist/observe/system-trace.js +294 -0
- package/dist/runtime/action-results.js +270 -0
- package/dist/runtime/clock.js +4 -0
- package/dist/server/common.js +1 -256
- package/dist/server/definitions/assert.js +162 -0
- package/dist/server/definitions/diagnostic.js +118 -0
- package/dist/server/definitions/input.js +274 -0
- package/dist/server/definitions/manage.js +194 -0
- package/dist/server/definitions/observe.js +149 -0
- package/dist/server/definitions/target.js +138 -0
- package/dist/server/definitions/wait.js +105 -0
- package/dist/server/handlers/action-execution.js +163 -0
- package/dist/server/handlers/correlation.js +23 -0
- package/dist/server/handlers/debugger-handlers.js +48 -0
- package/dist/server/handlers/debugger-response.js +9 -0
- package/dist/server/handlers/diagnostic.js +69 -0
- package/dist/server/handlers/handler-types.js +1 -0
- package/dist/server/handlers/input.js +154 -0
- package/dist/server/handlers/interact.js +113 -0
- package/dist/server/handlers/journey-assertions.js +35 -0
- package/dist/server/handlers/journey.js +147 -0
- package/dist/server/handlers/manage.js +119 -0
- package/dist/server/handlers/observe.js +95 -0
- package/dist/server/handlers/payload.js +4 -0
- package/dist/server/handlers/tool-handlers.js +160 -0
- package/dist/server/schemas/condition-schema.js +2 -0
- package/dist/server/schemas/debugger-schema.js +47 -0
- package/dist/server/{journey-schema.js → schemas/journey-schema.js} +8 -22
- package/dist/server/schemas/system-trace-schema.js +17 -0
- package/dist/server/tool-definitions.js +62 -1134
- package/dist/server-core.js +4 -4
- package/dist/server.js +34 -0
- package/dist/system/android-status.js +101 -0
- package/dist/system/android.js +37 -27
- package/dist/system/index.js +13 -438
- package/dist/system/ios-status.js +132 -0
- package/dist/system/ios.js +11 -8
- package/dist/system/model.js +1 -0
- package/dist/system/probes.js +105 -0
- package/dist/system/status-summary.js +43 -0
- package/dist/system/unsupported-status.js +52 -0
- package/dist/utils/android/commands.js +89 -0
- package/dist/utils/android/gradle.js +139 -0
- package/dist/utils/android/logs.js +103 -0
- package/dist/utils/android/ui-tree.js +233 -0
- package/dist/utils/android/utils.js +6 -564
- package/dist/utils/cli/idb/idb-helper.js +4 -6
- package/dist/utils/cli/idb/install-idb.js +13 -9
- package/dist/utils/cli/ios/preflight-ios.js +42 -38
- package/dist/utils/exec.js +2 -0
- package/dist/utils/ios/utils.js +45 -46
- package/dist/utils/java.js +75 -57
- package/dist/utils/operation-budget.js +2 -0
- package/dist/utils/resolve-device.js +6 -2
- package/dist/utils/ui/index.js +26 -26
- package/docs/CHANGELOG.md +48 -0
- package/docs/README.md +38 -0
- package/docs/benchmarks/README.md +18 -0
- package/docs/benchmarks/interaction-latency/README.md +14 -0
- package/docs/benchmarks/interaction-latency/final.md +4 -1
- package/docs/benchmarks/interaction-latency/ios-validation.md +2 -0
- package/docs/benchmarks/ui-tree-performance/README.md +24 -16
- package/docs/benchmarks/ui-tree-performance/comparison.md +11 -10
- package/docs/code-structure.md +118 -0
- package/docs/specs/001-state-verification/spec.md +16 -14
- package/docs/specs/002-richer-element-identity/spec.md +12 -10
- package/docs/specs/003-wait-and-synchronization-reliability/spec.md +12 -10
- package/docs/specs/004-action-verification-routing/spec.md +3 -3
- package/docs/specs/005-unified-action-execution-and-verification-model/spec.md +2 -0
- package/docs/specs/006-runtime-action-instrumentation-and-binding-layer/spec.md +2 -0
- package/docs/specs/007-actionability-resolution-and-executable-target-selection/spec.md +3 -1
- package/docs/specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md +3 -4
- package/docs/specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md +2 -0
- package/docs/specs/010-verification-stabilization-and-temporal-convergence/spec.md +2 -2
- package/docs/specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md +2 -2
- package/docs/specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md +2 -0
- package/docs/specs/012-action-trace-and-xecution-observability/spec.md +2 -3
- package/docs/specs/013-wait-and-synchronization-reliability/spec.md +19 -19
- package/docs/specs/014-actionability-resolution/spec.md +20 -18
- package/docs/specs/015-adjustable-control-support/spec.md +17 -15
- package/docs/specs/016-environment-tools/spec.md +20 -20
- package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +2 -0
- package/docs/specs/019-ui-tree-observation-performance/spec.md +3 -1
- package/docs/specs/020-conditional-actions/intent.md +5 -3
- package/docs/specs/020-conditional-actions/plan.md +8 -9
- package/docs/specs/020-conditional-actions/spec.md +4 -2
- package/docs/specs/021-app-readiness-signals/intent.md +4 -2
- package/docs/specs/021-app-readiness-signals/plan.md +7 -6
- package/docs/specs/021-app-readiness-signals/spec.md +3 -1
- package/docs/specs/022-correlated-evidence/intent.md +4 -4
- package/docs/specs/022-correlated-evidence/plan.md +10 -11
- package/docs/specs/022-correlated-evidence/spec.md +3 -1
- package/docs/specs/023-android-system-traces/device-evidence.json +38 -0
- package/docs/specs/023-android-system-traces/emulator-recheck.json +38 -0
- package/docs/specs/023-android-system-traces/intent.md +41 -0
- package/docs/specs/023-android-system-traces/plan.md +115 -0
- package/docs/specs/023-android-system-traces/spec.md +122 -0
- package/docs/specs/024-android-heap-dump-capture/intent.md +41 -0
- package/docs/specs/025-android-runtime-debugger/intent.md +41 -0
- package/docs/specs/025-android-runtime-debugger/plan.md +106 -0
- package/docs/specs/025-android-runtime-debugger/spec.md +183 -0
- package/docs/specs/026-code-structure-and-quality-gates/health-plan.md +88 -0
- package/docs/specs/026-code-structure-and-quality-gates/intent.md +47 -0
- package/docs/specs/026-code-structure-and-quality-gates/plan.md +191 -0
- package/docs/specs/026-code-structure-and-quality-gates/spec.md +138 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-final.json +200 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification-thin-facade.json +154 -0
- package/docs/specs/026-code-structure-and-quality-gates/verification.json +136 -0
- package/docs/specs/027-debugger-failure-diagnostics/intent.md +43 -0
- package/docs/specs/027-debugger-failure-diagnostics/plan.md +86 -0
- package/docs/specs/027-debugger-failure-diagnostics/spec.md +133 -0
- package/docs/specs/028-compact-debugger-responses/intent.md +41 -0
- package/docs/specs/028-compact-debugger-responses/plan.md +112 -0
- package/docs/specs/028-compact-debugger-responses/spec.md +108 -0
- package/docs/specs/029-kotlin-locals-validation/report.md +119 -0
- package/docs/specs/030-android-coroutine-validation/evidence.json +475 -0
- package/docs/specs/030-android-coroutine-validation/intent.md +47 -0
- package/docs/specs/030-android-coroutine-validation/plan.md +147 -0
- package/docs/specs/030-android-coroutine-validation/report.md +240 -0
- package/docs/specs/030-android-coroutine-validation/spec.md +107 -0
- package/docs/tools/TOOLS.md +31 -16
- package/docs/tools/{evidence.md → correlated-evidence.md} +14 -3
- package/docs/tools/debugger.md +180 -0
- package/docs/tools/interact.md +42 -20
- package/docs/tools/manage.md +25 -44
- package/docs/tools/observe.md +64 -25
- package/docs/tools/system-traces.md +134 -0
- package/docs/tools/system.md +12 -5
- package/package.json +11 -8
- package/scripts/architecture-graph.mjs +69 -0
- package/scripts/check-architecture.mjs +47 -0
- package/scripts/code-quality-baseline.json +1 -0
- package/scripts/code-quality.mjs +161 -0
- package/src/contracts/execution-controls.ts +26 -0
- package/src/contracts/selector.ts +17 -0
- package/src/contracts/target.ts +19 -0
- package/src/contracts/ui.ts +94 -0
- package/src/debugger/android.ts +66 -0
- package/src/debugger/diagnostics.ts +43 -0
- package/src/debugger/guard.ts +31 -0
- package/src/debugger/java/DebuggerWorker.java +166 -0
- package/src/debugger/protocol.ts +37 -0
- package/src/debugger/sessions.ts +327 -0
- package/src/debugger/worker.ts +135 -0
- package/src/interact/adjustment/adjustment-context.ts +83 -0
- package/src/interact/adjustment/adjustment-gestures.ts +48 -0
- package/src/interact/adjustment/adjustment-plan.ts +23 -0
- package/src/interact/adjustment/adjustment-resolution.ts +104 -0
- package/src/interact/adjustment/adjustment-result.ts +34 -0
- package/src/interact/adjustment/adjustment-semantic.ts +53 -0
- package/src/interact/adjustment/adjustment-verification.ts +60 -0
- package/src/interact/adjustment/adjustment.ts +49 -0
- package/src/interact/adjustment/control-geometry.ts +120 -0
- package/src/interact/adjustment/control-values.ts +141 -0
- package/src/interact/adjustment/slider-labels.ts +84 -0
- package/src/interact/assertions/screen-assertion.ts +39 -0
- package/src/interact/assertions/state-assertion-context.ts +48 -0
- package/src/interact/assertions/state-assertion.ts +64 -0
- package/src/interact/assertions/state-comparison.ts +168 -0
- package/src/interact/assertions/visibility-assertion.ts +41 -0
- package/src/interact/conditional-action.ts +18 -12
- package/src/interact/index.ts +76 -3491
- package/src/interact/input-service.ts +11 -0
- package/src/interact/interaction-trace.ts +36 -0
- package/src/interact/model.ts +109 -0
- package/src/interact/observation.ts +7 -0
- package/src/interact/targeting/candidate-ancestor.ts +49 -0
- package/src/interact/targeting/candidate-result.ts +24 -0
- package/src/interact/targeting/candidate-scoring.ts +42 -0
- package/src/interact/targeting/element-helpers.ts +243 -0
- package/src/interact/targeting/find-element.ts +47 -0
- package/src/interact/targeting/nearby-control.ts +46 -0
- package/src/interact/targeting/resolved-elements.ts +20 -0
- package/src/interact/targeting/tap-target.ts +22 -0
- package/src/interact/targeting/target-scope.ts +121 -0
- package/src/interact/waits/legacy-log-wait.ts +30 -0
- package/src/interact/waits/legacy-wait-observations.ts +55 -0
- package/src/interact/waits/legacy-wait-state.ts +48 -0
- package/src/interact/waits/legacy-wait.ts +34 -0
- package/src/interact/waits/screen-wait.ts +37 -0
- package/src/interact/waits/ui-change-helpers.ts +184 -0
- package/src/interact/waits/ui-change-state.ts +26 -0
- package/src/interact/waits/ui-change-wait.ts +59 -0
- package/src/interact/waits/ui-wait-condition.ts +52 -0
- package/src/interact/waits/ui-wait-state.ts +55 -0
- package/src/interact/waits/ui-wait.ts +84 -0
- package/src/manage/android.ts +56 -50
- package/src/manage/index.ts +63 -93
- package/src/manage/ios/build-types.ts +14 -0
- package/src/manage/ios/build.ts +230 -0
- package/src/manage/ios/project-discovery.ts +26 -0
- package/src/manage/ios/scheme.ts +16 -0
- package/src/manage/ios.ts +65 -298
- package/src/observe/android/context.ts +1 -0
- package/src/observe/android/current-screen.ts +63 -0
- package/src/observe/android/fingerprint.ts +13 -0
- package/src/observe/android/log-stream.ts +127 -0
- package/src/observe/android/logs.ts +110 -0
- package/src/observe/android/screenshot.ts +79 -0
- package/src/observe/android/ui-tree.ts +113 -0
- package/src/observe/android.ts +17 -531
- package/src/observe/evidence.ts +130 -71
- package/src/observe/index.ts +24 -286
- package/src/observe/ios/context.ts +22 -0
- package/src/observe/ios/fingerprint.ts +12 -0
- package/src/observe/ios/log-stream.ts +103 -0
- package/src/observe/ios/logs.ts +179 -0
- package/src/observe/ios/screenshot.ts +26 -0
- package/src/observe/ios/ui-elements.ts +266 -0
- package/src/observe/ios/ui-tree.ts +143 -0
- package/src/observe/ios.ts +18 -749
- package/src/observe/logs/model.ts +11 -0
- package/src/{network → observe/network}/index.ts +2 -2
- package/src/observe/readiness-service.ts +10 -3
- package/src/observe/readiness.ts +49 -22
- package/src/observe/screenshot-encoding.ts +32 -0
- package/src/observe/snapshot-collection.ts +69 -0
- package/src/observe/snapshot-logs.ts +26 -0
- package/src/observe/snapshot-metadata.ts +51 -31
- package/src/observe/snapshot-semantic.ts +153 -0
- package/src/observe/system-trace-android.ts +133 -0
- package/src/observe/system-trace.ts +227 -0
- package/src/runtime/action-results.ts +376 -0
- package/src/runtime/clock.ts +9 -0
- package/src/server/common.ts +9 -359
- package/src/server/definitions/assert.ts +164 -0
- package/src/server/definitions/diagnostic.ts +120 -0
- package/src/server/definitions/input.ts +276 -0
- package/src/server/definitions/manage.ts +196 -0
- package/src/server/definitions/observe.ts +151 -0
- package/src/server/definitions/target.ts +140 -0
- package/src/server/definitions/wait.ts +107 -0
- package/src/server/handlers/action-execution.ts +164 -0
- package/src/server/handlers/correlation.ts +20 -0
- package/src/server/handlers/debugger-handlers.ts +39 -0
- package/src/server/handlers/debugger-response.ts +7 -0
- package/src/server/handlers/diagnostic.ts +79 -0
- package/src/server/handlers/handler-types.ts +7 -0
- package/src/server/handlers/input.ts +173 -0
- package/src/server/handlers/interact.ts +135 -0
- package/src/server/handlers/journey-assertions.ts +39 -0
- package/src/server/handlers/journey.ts +132 -0
- package/src/server/handlers/manage.ts +141 -0
- package/src/server/handlers/observe.ts +116 -0
- package/src/server/handlers/payload.ts +36 -0
- package/src/server/handlers/tool-handlers.ts +159 -0
- package/src/server/schemas/condition-schema.ts +2 -0
- package/src/server/schemas/debugger-schema.ts +43 -0
- package/src/server/{journey-schema.ts → schemas/journey-schema.ts} +8 -22
- package/src/server/schemas/system-trace-schema.ts +17 -0
- package/src/server/tool-definitions.ts +69 -1140
- package/src/server-core.ts +4 -4
- package/src/server.ts +25 -1
- package/src/system/android-status.ts +124 -0
- package/src/system/android.ts +33 -21
- package/src/system/index.ts +21 -523
- package/src/system/ios-status.ts +147 -0
- package/src/system/ios.ts +14 -10
- package/src/system/model.ts +41 -0
- package/src/system/probes.ts +114 -0
- package/src/system/status-summary.ts +47 -0
- package/src/system/unsupported-status.ts +55 -0
- package/src/types.ts +16 -106
- package/src/utils/android/commands.ts +81 -0
- package/src/utils/android/gradle.ts +132 -0
- package/src/utils/android/logs.ts +103 -0
- package/src/utils/android/ui-tree.ts +239 -0
- package/src/utils/android/utils.ts +16 -556
- package/src/utils/cli/idb/check-idb.ts +4 -4
- package/src/utils/cli/idb/idb-helper.ts +6 -8
- package/src/utils/cli/idb/install-idb.ts +15 -9
- package/src/utils/cli/ios/preflight-ios.ts +39 -31
- package/src/utils/cli/ios/run-ios-ui-tree-tap.ts +4 -4
- package/src/utils/exec.ts +2 -0
- package/src/utils/ios/utils.ts +57 -61
- package/src/utils/java.ts +71 -51
- package/src/utils/operation-budget.ts +2 -0
- package/src/utils/resolve-device.ts +11 -6
- package/src/utils/ui/index.ts +28 -28
- package/test/device/automated/debugger/coroutine-lifecycle.ts +27 -0
- package/test/device/automated/debugger/coroutine-support.ts +140 -0
- package/test/device/automated/debugger/coroutine.android.test.ts +169 -0
- package/test/device/automated/debugger/kotlin.android.test.ts +127 -0
- package/test/device/automated/debugger/runtime.android.test.ts +219 -0
- package/test/device/automated/interact/interaction.android.test.ts +56 -0
- package/test/device/automated/observe/system-trace.smoke.ts +104 -0
- package/test/device/manual/interact/modul8_selector_tap.manual.ts +1 -1
- package/test/fixtures/contracts/tool-definitions.json +3536 -0
- package/test/fixtures/coroutine-app/MainActivity.kt +100 -0
- package/test/fixtures/coroutine-app/README.md +48 -0
- package/test/fixtures/coroutine-app/build.ts +35 -0
- package/test/fixtures/debugger-app/MainActivity.java +67 -0
- package/test/fixtures/debugger-app/MainActivity.kt +32 -0
- package/test/fixtures/debugger-app/NoLocals.java +7 -0
- package/test/fixtures/debugger-app/README.md +65 -0
- package/test/fixtures/debugger-app/build.ts +39 -0
- package/test/fixtures/latency-journey-app/README.md +2 -0
- package/test/fixtures/readiness-app/README.md +2 -0
- package/test/helpers/perfetto.ts +59 -0
- package/test/unit/debugger/coroutine-lifecycle.test.ts +43 -0
- package/test/unit/debugger/diagnostic-sessions.test.ts +67 -0
- package/test/unit/debugger/diagnostics.test.ts +80 -0
- package/test/unit/debugger/guards.test.ts +50 -0
- package/test/unit/debugger/native.test.ts +33 -0
- package/test/unit/debugger/preparation.test.ts +32 -0
- package/test/unit/debugger/protocol.test.ts +15 -0
- package/test/unit/debugger/sessions.test.ts +83 -0
- package/test/unit/interact/adjustment_service.test.ts +42 -0
- package/test/unit/interact/adjustment_verification.test.ts +35 -0
- package/test/unit/interact/clock_dependencies.test.ts +63 -0
- package/test/unit/interact/element_helpers.test.ts +19 -0
- package/test/unit/interact/legacy_wait_service.test.ts +17 -0
- package/test/unit/interact/state_comparison.test.ts +20 -0
- package/test/unit/interact/target_scope.test.ts +35 -0
- package/test/unit/network/get_network_activity.test.ts +1 -1
- package/test/unit/observe/readiness.test.ts +3 -1
- package/test/unit/observe/screenshot-encoding.test.ts +30 -0
- package/test/unit/observe/snapshot-logs.test.ts +14 -0
- package/test/unit/observe/system-trace-android.test.ts +52 -0
- package/test/unit/observe/system-trace.test.ts +166 -0
- package/test/unit/quality/architecture.test.ts +60 -0
- package/test/unit/quality/code-quality.test.ts +58 -0
- package/test/unit/quality/interaction_boundaries.test.ts +24 -0
- package/test/unit/server/conditional_actions.test.ts +1 -1
- package/test/unit/server/conditional_schema.test.ts +2 -2
- package/test/unit/server/debugger-response.test.ts +63 -0
- package/test/unit/server/system-trace.test.ts +64 -0
- package/test/unit/server/tool_definitions_parity.test.ts +8 -0
- package/dist/server/tool-handlers.js +0 -893
- package/docs/ROADMAP.md +0 -646
- package/docs/benchmarks/ui-tree-performance/bridge-probe.json +0 -29
- package/docs/benchmarks/ui-tree-performance/initial-baseline.json +0 -59560
- package/docs/specs/baseline-spec-v0.md +0 -312
- package/docs/specs/mcp-tooling-spec-v1.md +0 -446
- package/src/server/tool-handlers.ts +0 -923
- /package/dist/{server/condition-schema.js → contracts/condition.js} +0 -0
- /package/dist/server/{readiness-schema.js → schemas/readiness-schema.js} +0 -0
- /package/src/{server/condition-schema.ts → contracts/condition.ts} +0 -0
- /package/src/server/{readiness-schema.ts → schemas/readiness-schema.ts} +0 -0
package/src/interact/index.ts
CHANGED
|
@@ -1,18 +1,34 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { UiTree } from './observation.js'
|
|
2
|
+
import { resolveTap } from './targeting/tap-target.js'
|
|
3
|
+
import { waitForUICore } from './waits/legacy-wait.js'
|
|
4
|
+
import type { LegacyWaitInput } from './waits/legacy-wait-state.js'
|
|
5
|
+
import { expectState } from './assertions/state-assertion.js'
|
|
6
|
+
import type { StateInput } from './assertions/state-assertion-context.js'
|
|
7
|
+
import { waitForUIChange } from './waits/ui-change-wait.js'
|
|
8
|
+
import type { ChangeInput } from './waits/ui-change-state.js'
|
|
9
|
+
import { waitForScreenChange, type ScreenWaitInput } from './waits/screen-wait.js'
|
|
10
|
+
import { expectScreen, type ScreenExpectation } from './assertions/screen-assertion.js'
|
|
11
|
+
import { expectVisible, type VisibilityInput } from './assertions/visibility-assertion.js'
|
|
12
|
+
import { findElement, type FindInput } from './targeting/find-element.js'
|
|
13
|
+
import { waitForUI } from './waits/ui-wait.js'
|
|
14
|
+
import type { UIWaitInput } from './waits/ui-wait-state.js'
|
|
15
|
+
import { adjustControl } from './adjustment/adjustment.js'
|
|
16
|
+
import type { AdjustmentInput } from './adjustment/adjustment-context.js'
|
|
17
|
+
import type { InputService } from './input-service.js'
|
|
18
|
+
import { ResolvedElements } from './targeting/resolved-elements.js'
|
|
19
|
+
import { findScopedElement, resolveUiChangeScope, type ScopeTree } from './targeting/target-scope.js'
|
|
20
|
+
import type { ScreenFingerprintResponse, UiElement, ResolvedUiElementContext } from './model.js'
|
|
21
|
+
import { _normalizeBounds, _computeElementId, _resolvedTargetFromElement } from './targeting/element-helpers.js'
|
|
22
|
+
import { _collectSubtreeIndices } from './waits/ui-change-helpers.js'
|
|
3
23
|
import { isConditionBlocked } from './conditional-action.js'
|
|
4
|
-
import { remainingBudget } from '../utils/operation-budget.js'
|
|
5
|
-
import { createHash } from 'crypto'
|
|
6
24
|
import { AndroidInteract } from './android.js';
|
|
7
25
|
import { iOSInteract } from './ios.js';
|
|
8
26
|
export { AndroidInteract, iOSInteract };
|
|
9
27
|
|
|
10
28
|
import { resolveTargetDevice } from '../utils/resolve-device.js'
|
|
11
29
|
import { ToolsObserve } from '../observe/index.js'
|
|
12
|
-
import {
|
|
13
|
-
import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js'
|
|
30
|
+
import { buildActionExecutionResult } from '../runtime/action-results.js'
|
|
14
31
|
import type {
|
|
15
|
-
ActionTrace,
|
|
16
32
|
ActionFailureCode,
|
|
17
33
|
ActionTargetResolved,
|
|
18
34
|
AdjustControlResponse,
|
|
@@ -21,318 +37,17 @@ import type {
|
|
|
21
37
|
ExpectStateResponse,
|
|
22
38
|
ExpectScreenResponse,
|
|
23
39
|
WaitForUIChangeResponse,
|
|
24
|
-
UIElementSemanticMetadata,
|
|
25
|
-
UIElementState,
|
|
26
|
-
TraceStep,
|
|
27
40
|
TapElementResponse
|
|
28
41
|
} from '../types.js'
|
|
29
42
|
|
|
30
|
-
interface ScreenFingerprintResponse { fingerprint: string | null }
|
|
31
|
-
|
|
32
|
-
interface UiElement {
|
|
33
|
-
text?: string | null
|
|
34
|
-
label?: string | null
|
|
35
|
-
value?: string | null
|
|
36
|
-
contentDescription?: string | null
|
|
37
|
-
contentDesc?: string | null
|
|
38
|
-
accessibilityLabel?: string | null
|
|
39
|
-
resourceId?: string | null
|
|
40
|
-
resourceID?: string | null
|
|
41
|
-
id?: string | null
|
|
42
|
-
type?: string | null
|
|
43
|
-
class?: string | null
|
|
44
|
-
bounds?: number[] | null
|
|
45
|
-
clickable?: boolean
|
|
46
|
-
enabled?: boolean
|
|
47
|
-
focusable?: boolean
|
|
48
|
-
visible?: boolean
|
|
49
|
-
parentId?: number | string | null
|
|
50
|
-
children?: number[]
|
|
51
|
-
_index?: number
|
|
52
|
-
_interactable?: boolean
|
|
53
|
-
_sliderLike?: boolean
|
|
54
|
-
state?: UIElementState | null
|
|
55
|
-
stable_id?: string | null
|
|
56
|
-
role?: string | null
|
|
57
|
-
test_tag?: string | null
|
|
58
|
-
selector?: { value: string | null, confidence: { score: number, reason: string } | null } | null
|
|
59
|
-
semantic?: UIElementSemanticMetadata | null
|
|
60
|
-
actions?: string[] | null
|
|
61
|
-
accessibilityActions?: string[] | null
|
|
62
|
-
runtimeActions?: string[] | null
|
|
63
|
-
platformActions?: string[] | null
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
interface ResolvedUiElementContext {
|
|
67
|
-
elementId: string
|
|
68
|
-
platform: 'android' | 'ios'
|
|
69
|
-
deviceId?: string
|
|
70
|
-
bounds: [number, number, number, number] | null
|
|
71
|
-
index: number
|
|
72
|
-
stable_id?: string | null
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
interface UiResolution {
|
|
76
|
-
width?: number
|
|
77
|
-
height?: number
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
interface UiChangeSignatureSet {
|
|
81
|
-
hierarchy: string | null
|
|
82
|
-
text: string | null
|
|
83
|
-
state: string | null
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
interface UiChangeScopeResolution {
|
|
87
|
-
scope: 'screen' | 'subtree'
|
|
88
|
-
target: string | null
|
|
89
|
-
resolved: boolean
|
|
90
|
-
resolvedIndex: number | null
|
|
91
|
-
resolvedStableId: string | null
|
|
92
|
-
reason: string
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
interface UiChangeScopeResult {
|
|
96
|
-
elements: UiElement[]
|
|
97
|
-
resolution: UiChangeScopeResolution
|
|
98
|
-
error?: {
|
|
99
|
-
code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
|
|
100
|
-
message: string
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
interface RankedResolutionCandidate {
|
|
105
|
-
el: UiElement
|
|
106
|
-
idx: number
|
|
107
|
-
score: number
|
|
108
|
-
reason: string
|
|
109
|
-
interactable: boolean
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
interface InferredLabeledSliderState {
|
|
113
|
-
current: number
|
|
114
|
-
min: number
|
|
115
|
-
max: number
|
|
116
|
-
value_range: { min: number; max: number }
|
|
117
|
-
touch_bounds: [number, number, number, number]
|
|
118
|
-
source: 'nearby_labels'
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
function buildObservationTrace({
|
|
122
|
-
actionType,
|
|
123
|
-
stage,
|
|
124
|
-
success,
|
|
125
|
-
attempts,
|
|
126
|
-
metadata
|
|
127
|
-
}: {
|
|
128
|
-
actionType: string
|
|
129
|
-
stage: 'verify' | 'stabilize' | 'resolve'
|
|
130
|
-
success: boolean
|
|
131
|
-
attempts: number
|
|
132
|
-
metadata?: Record<string, unknown>
|
|
133
|
-
}): ActionTrace {
|
|
134
|
-
const now = Date.now()
|
|
135
|
-
const actionId = nextActionId(actionType, now)
|
|
136
|
-
const steps: TraceStep[] = [
|
|
137
|
-
createTraceStep({
|
|
138
|
-
stage,
|
|
139
|
-
timestamp: now,
|
|
140
|
-
result: success ? 'success' : 'failure',
|
|
141
|
-
attemptIndex: 0,
|
|
142
|
-
metadata
|
|
143
|
-
})
|
|
144
|
-
]
|
|
145
|
-
|
|
146
|
-
return {
|
|
147
|
-
action_id: actionId,
|
|
148
|
-
steps,
|
|
149
|
-
final_outcome: success ? 'success' : 'failure',
|
|
150
|
-
attempts: Math.max(1, Math.floor(attempts || 1))
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
interface FindElementResolutionSummary {
|
|
155
|
-
confidence: number
|
|
156
|
-
reason: string
|
|
157
|
-
fallback_available: boolean
|
|
158
|
-
matched_count: number
|
|
159
|
-
alternates: Array<{
|
|
160
|
-
text: string | null
|
|
161
|
-
resource_id: string | null
|
|
162
|
-
accessibility_id: string | null
|
|
163
|
-
class: string | null
|
|
164
|
-
bounds: { left: number; top: number; right: number; bottom: number } | null
|
|
165
|
-
clickable: boolean
|
|
166
|
-
enabled: boolean
|
|
167
|
-
score: number
|
|
168
|
-
reason: string
|
|
169
|
-
}>
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
|
|
173
43
|
export class ToolsInteract {
|
|
174
44
|
private static readonly _maxResolvedUiElements = 256
|
|
175
|
-
private static readonly _uiChangeKinds: Array<'hierarchy_diff' | 'text_change' | 'state_change'> = ['hierarchy_diff', 'text_change', 'state_change']
|
|
176
|
-
private static readonly _sliderSearchLookahead = 8
|
|
177
|
-
private static readonly _sliderNegativeGapTolerancePx = 32
|
|
178
|
-
private static readonly _sliderPositiveGapLimitPx = 640
|
|
179
|
-
private static readonly _sliderTrackMinLengthPx = 220
|
|
180
|
-
private static readonly _sliderTrackMaxThicknessPx = 180
|
|
181
|
-
private static readonly _sliderTrackLengthRatio = 0.18
|
|
182
|
-
private static readonly _sliderTrackThicknessRatio = 0.08
|
|
183
|
-
private static readonly _sliderLabelWidthRatio = 1.5
|
|
184
|
-
private static _resolvedUiElements = new Map<string, ResolvedUiElementContext>()
|
|
185
|
-
|
|
186
|
-
private static _normalize(s: any): string {
|
|
187
|
-
if (s === null || s === undefined) return ''
|
|
188
|
-
try { return String(s).toLowerCase().trim() } catch { return '' }
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
private static _normalizeBounds(bounds: any): [number, number, number, number] | null {
|
|
192
|
-
if (!Array.isArray(bounds) || bounds.length < 4) return null
|
|
193
|
-
const normalized = bounds.slice(0, 4).map((value: any) => Number(value))
|
|
194
|
-
if (normalized.some((value: number) => Number.isNaN(value))) return null
|
|
195
|
-
return normalized as [number, number, number, number]
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
private static _hash(value: unknown): string {
|
|
199
|
-
return createHash('sha256').update(JSON.stringify(value)).digest('hex')
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
private static _matchesSelector(el: UiElement, selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }): boolean {
|
|
203
|
-
if (!selector) return false
|
|
204
|
-
const normalize = ToolsInteract._normalize
|
|
205
|
-
const containsFlag = !!selector.contains
|
|
206
|
-
const text = normalize(el.text ?? el.label ?? el.value ?? '')
|
|
207
|
-
const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
|
|
208
|
-
const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '')
|
|
209
|
-
|
|
210
|
-
if (selector.text !== undefined && selector.text !== null) {
|
|
211
|
-
const q = normalize(selector.text)
|
|
212
|
-
if (containsFlag ? !text.includes(q) : text !== q) return false
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
if (selector.resource_id !== undefined && selector.resource_id !== null) {
|
|
216
|
-
const q = normalize(selector.resource_id)
|
|
217
|
-
if (containsFlag ? !resourceId.includes(q) : resourceId !== q) return false
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
|
|
221
|
-
const q = normalize(selector.accessibility_id)
|
|
222
|
-
if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q) return false
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
return true
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
private static _findFirstMatchingElement(
|
|
229
|
-
elements: UiElement[],
|
|
230
|
-
selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }
|
|
231
|
-
): { el: UiElement, idx: number } | null {
|
|
232
|
-
if (!selector) return null
|
|
233
|
-
for (let i = 0; i < elements.length; i++) {
|
|
234
|
-
const el = elements[i]
|
|
235
|
-
if (!el) continue
|
|
236
|
-
if (ToolsInteract._matchesSelector(el, selector)) return { el, idx: i }
|
|
237
|
-
}
|
|
238
|
-
return null
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
private static _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
|
|
242
|
-
if (parentId === undefined || parentId === null) return null
|
|
243
|
-
|
|
244
|
-
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
245
|
-
return parentId
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
if (typeof parentId === 'string') {
|
|
249
|
-
const normalized = ToolsInteract._normalize(parentId)
|
|
250
|
-
if (!normalized) return null
|
|
251
|
-
|
|
252
|
-
if (/^\d+$/.test(normalized)) {
|
|
253
|
-
const index = Number(normalized)
|
|
254
|
-
if (index >= 0 && index < elements.length) return index
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
const foundIndex = elements.findIndex((el) => {
|
|
258
|
-
if (!el) return false
|
|
259
|
-
return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
260
|
-
ToolsInteract._normalize(el.stable_id ?? '') === normalized
|
|
261
|
-
})
|
|
262
|
-
|
|
263
|
-
return foundIndex >= 0 ? foundIndex : null
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
return null
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
private static _isVisibleElement(el: UiElement): boolean {
|
|
270
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
271
|
-
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
private static _isTapActionable(
|
|
275
|
-
el: UiElement,
|
|
276
|
-
storedStableId?: string | null,
|
|
277
|
-
platform?: 'android' | 'ios'
|
|
278
|
-
): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
|
|
279
|
-
if (!ToolsInteract._isVisibleElement(el)) {
|
|
280
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
if (el.enabled === false) {
|
|
284
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
const semanticTapActionable = !!el.semantic && (
|
|
288
|
-
el.semantic.is_clickable ||
|
|
289
|
-
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap'))
|
|
290
|
-
)
|
|
291
|
-
|
|
292
|
-
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
293
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
if (storedStableId) {
|
|
297
|
-
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
298
|
-
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
return { actionable: true }
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
private static _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
|
|
306
|
-
if (!ToolsInteract._isVisibleElement(el)) {
|
|
307
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
if (el.enabled === false) {
|
|
311
|
-
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
|
|
312
|
-
}
|
|
313
45
|
|
|
314
|
-
|
|
315
|
-
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
316
|
-
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
|
|
317
|
-
}
|
|
318
|
-
}
|
|
46
|
+
private static _resolvedUiElements = new ResolvedElements(ToolsInteract._maxResolvedUiElements)
|
|
319
47
|
|
|
320
|
-
|
|
321
|
-
}
|
|
48
|
+
private static _normalizeBounds = _normalizeBounds
|
|
322
49
|
|
|
323
|
-
private static _computeElementId
|
|
324
|
-
const identity = {
|
|
325
|
-
platform,
|
|
326
|
-
deviceId: deviceId || '',
|
|
327
|
-
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
328
|
-
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
329
|
-
accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
|
|
330
|
-
class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
331
|
-
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
332
|
-
index
|
|
333
|
-
}
|
|
334
|
-
return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`
|
|
335
|
-
}
|
|
50
|
+
private static _computeElementId = _computeElementId
|
|
336
51
|
|
|
337
52
|
private static _buildResolvedElement(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number) {
|
|
338
53
|
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
@@ -364,247 +79,18 @@ export class ToolsInteract {
|
|
|
364
79
|
}
|
|
365
80
|
}
|
|
366
81
|
|
|
367
|
-
private static _resolveUiChangeScope(
|
|
368
|
-
tree
|
|
369
|
-
scope: 'screen' | 'subtree' | undefined,
|
|
370
|
-
target: string | null | undefined
|
|
371
|
-
): UiChangeScopeResult {
|
|
372
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
373
|
-
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen'
|
|
374
|
-
|
|
375
|
-
if (normalizedScope === 'screen') {
|
|
376
|
-
return {
|
|
377
|
-
elements,
|
|
378
|
-
resolution: {
|
|
379
|
-
scope: 'screen',
|
|
380
|
-
target: null,
|
|
381
|
-
resolved: true,
|
|
382
|
-
resolvedIndex: null,
|
|
383
|
-
resolvedStableId: null,
|
|
384
|
-
reason: 'screen scope'
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null
|
|
390
|
-
if (!requestedTarget) {
|
|
391
|
-
return {
|
|
392
|
-
elements: [],
|
|
393
|
-
resolution: {
|
|
394
|
-
scope: 'subtree',
|
|
395
|
-
target: null,
|
|
396
|
-
resolved: false,
|
|
397
|
-
resolvedIndex: null,
|
|
398
|
-
resolvedStableId: null,
|
|
399
|
-
reason: 'subtree scope requires a target element id'
|
|
400
|
-
},
|
|
401
|
-
error: {
|
|
402
|
-
code: 'INVALID_SCOPE',
|
|
403
|
-
message: 'scope=subtree requires a target element_id'
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
const resolved = ToolsInteract._findScopedElement(tree, requestedTarget)
|
|
409
|
-
if (!resolved) {
|
|
410
|
-
return {
|
|
411
|
-
elements: [],
|
|
412
|
-
resolution: {
|
|
413
|
-
scope: 'subtree',
|
|
414
|
-
target: requestedTarget,
|
|
415
|
-
resolved: false,
|
|
416
|
-
resolvedIndex: null,
|
|
417
|
-
resolvedStableId: null,
|
|
418
|
-
reason: 'target element could not be resolved'
|
|
419
|
-
},
|
|
420
|
-
error: {
|
|
421
|
-
code: 'ELEMENT_NOT_FOUND',
|
|
422
|
-
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index)
|
|
428
|
-
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element): element is UiElement => !!element)
|
|
429
|
-
|
|
430
|
-
return {
|
|
431
|
-
elements: scopedElements,
|
|
432
|
-
resolution: {
|
|
433
|
-
scope: 'subtree',
|
|
434
|
-
target: requestedTarget,
|
|
435
|
-
resolved: true,
|
|
436
|
-
resolvedIndex: resolved.index,
|
|
437
|
-
resolvedStableId: resolved.stableId,
|
|
438
|
-
reason: resolved.reason
|
|
439
|
-
}
|
|
440
|
-
}
|
|
82
|
+
private static _resolveUiChangeScope(tree: ScopeTree | null | undefined, scope: 'screen' | 'subtree' | undefined, target: string | null | undefined) {
|
|
83
|
+
return resolveUiChangeScope(tree, scope, target, (snapshot, id) => ToolsInteract._findScopedElement(snapshot, id))
|
|
441
84
|
}
|
|
442
85
|
|
|
443
|
-
private static _findScopedElement(tree:
|
|
444
|
-
|
|
445
|
-
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android'
|
|
446
|
-
const deviceId = tree?.device?.id ?? undefined
|
|
447
|
-
const normalizedTarget = ToolsInteract._normalize(targetElementId)
|
|
448
|
-
|
|
449
|
-
for (let i = 0; i < elements.length; i++) {
|
|
450
|
-
const el = elements[i]
|
|
451
|
-
if (!el) continue
|
|
452
|
-
|
|
453
|
-
const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i)
|
|
454
|
-
if (computedElementId === targetElementId) {
|
|
455
|
-
return {
|
|
456
|
-
index: i,
|
|
457
|
-
stableId: el.stable_id ?? null,
|
|
458
|
-
reason: 'element_id_match'
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
for (let i = 0; i < elements.length; i++) {
|
|
464
|
-
const el = elements[i]
|
|
465
|
-
if (!el) continue
|
|
466
|
-
|
|
467
|
-
if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
|
|
468
|
-
return {
|
|
469
|
-
index: i,
|
|
470
|
-
stableId: el.stable_id,
|
|
471
|
-
reason: 'stable_id_match'
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId)
|
|
477
|
-
if (storedContext?.stable_id) {
|
|
478
|
-
const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id)
|
|
479
|
-
for (let i = 0; i < elements.length; i++) {
|
|
480
|
-
const el = elements[i]
|
|
481
|
-
if (!el?.stable_id) continue
|
|
482
|
-
if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
|
|
483
|
-
return {
|
|
484
|
-
index: i,
|
|
485
|
-
stableId: el.stable_id,
|
|
486
|
-
reason: 'stored_stable_id_match'
|
|
487
|
-
}
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
return null
|
|
86
|
+
private static _findScopedElement(tree: ScopeTree | null | undefined, id: string) {
|
|
87
|
+
return findScopedElement(tree, id, ToolsInteract._resolvedUiElements)
|
|
493
88
|
}
|
|
494
89
|
|
|
495
|
-
private static _collectSubtreeIndices
|
|
496
|
-
if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length) return []
|
|
497
|
-
|
|
498
|
-
const visited = new Set<number>()
|
|
499
|
-
const stack = [rootIndex]
|
|
500
|
-
const result: number[] = []
|
|
501
|
-
|
|
502
|
-
while (stack.length > 0) {
|
|
503
|
-
const index = stack.pop()
|
|
504
|
-
if (index === undefined || visited.has(index) || index < 0 || index >= elements.length) continue
|
|
505
|
-
visited.add(index)
|
|
506
|
-
result.push(index)
|
|
507
|
-
|
|
508
|
-
const element = elements[index]
|
|
509
|
-
if (!element) continue
|
|
510
|
-
|
|
511
|
-
const directChildren = new Set<number>()
|
|
512
|
-
if (Array.isArray(element.children)) {
|
|
513
|
-
for (const childIndex of element.children) {
|
|
514
|
-
if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
|
|
515
|
-
directChildren.add(childIndex)
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
for (let i = 0; i < elements.length; i++) {
|
|
521
|
-
if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
|
|
522
|
-
directChildren.add(i)
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
for (const childIndex of directChildren) {
|
|
527
|
-
if (!visited.has(childIndex)) stack.push(childIndex)
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
return result.sort((left, right) => left - right)
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
private static _changeIdentityForElement(el: UiElement, index: number): string {
|
|
535
|
-
const stableId = ToolsInteract._normalize(el.stable_id)
|
|
536
|
-
if (stableId) return `stable:${stableId}`
|
|
537
|
-
|
|
538
|
-
return `fallback:${ToolsInteract._hash({
|
|
539
|
-
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
540
|
-
contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
|
|
541
|
-
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
542
|
-
type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
543
|
-
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
544
|
-
index
|
|
545
|
-
})}`
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
private static _summarizeUiChangeDelta(initialElements: UiElement[], currentElements: UiElement[]) {
|
|
549
|
-
const buildMap = (elements: UiElement[]) => {
|
|
550
|
-
const map = new Map<string, string>()
|
|
551
|
-
for (let i = 0; i < elements.length; i++) {
|
|
552
|
-
const element = elements[i]
|
|
553
|
-
if (!element) continue
|
|
554
|
-
const key = ToolsInteract._changeIdentityForElement(element, i)
|
|
555
|
-
map.set(key, ToolsInteract._hash({
|
|
556
|
-
text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
|
|
557
|
-
contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
|
|
558
|
-
resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
|
|
559
|
-
type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
|
|
560
|
-
bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
|
|
561
|
-
state: element.state ?? null,
|
|
562
|
-
visible: !!element.visible,
|
|
563
|
-
enabled: !!element.enabled,
|
|
564
|
-
clickable: !!element.clickable
|
|
565
|
-
}))
|
|
566
|
-
}
|
|
567
|
-
return map
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
const initialMap = buildMap(initialElements)
|
|
571
|
-
const currentMap = buildMap(currentElements)
|
|
572
|
-
let added = 0
|
|
573
|
-
let removed = 0
|
|
574
|
-
let mutated = 0
|
|
575
|
-
|
|
576
|
-
for (const [key, value] of currentMap.entries()) {
|
|
577
|
-
if (!initialMap.has(key)) {
|
|
578
|
-
added++
|
|
579
|
-
} else if (initialMap.get(key) !== value) {
|
|
580
|
-
mutated++
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
for (const key of initialMap.keys()) {
|
|
585
|
-
if (!currentMap.has(key)) removed++
|
|
586
|
-
}
|
|
587
|
-
|
|
588
|
-
return {
|
|
589
|
-
total_elements: currentElements.length,
|
|
590
|
-
added_elements: added,
|
|
591
|
-
removed_elements: removed,
|
|
592
|
-
mutated_elements: mutated
|
|
593
|
-
}
|
|
594
|
-
}
|
|
90
|
+
private static _collectSubtreeIndices = _collectSubtreeIndices
|
|
595
91
|
|
|
596
92
|
private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
|
|
597
|
-
|
|
598
|
-
ToolsInteract._resolvedUiElements.delete(elementId)
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
ToolsInteract._resolvedUiElements.set(elementId, context)
|
|
602
|
-
|
|
603
|
-
while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
|
|
604
|
-
const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value
|
|
605
|
-
if (!oldestElementId) break
|
|
606
|
-
ToolsInteract._resolvedUiElements.delete(oldestElementId)
|
|
607
|
-
}
|
|
93
|
+
ToolsInteract._resolvedUiElements.remember(elementId, context)
|
|
608
94
|
}
|
|
609
95
|
|
|
610
96
|
private static async _captureFingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null> {
|
|
@@ -617,454 +103,7 @@ export class ToolsInteract {
|
|
|
617
103
|
}
|
|
618
104
|
}
|
|
619
105
|
|
|
620
|
-
private static
|
|
621
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
622
|
-
const textPayload: Array<{ text: string, contentDescription: string, resourceId: string }> = []
|
|
623
|
-
const statePayload: Array<{
|
|
624
|
-
checked: boolean | null
|
|
625
|
-
selected: boolean | string | { id: string; label?: string } | null
|
|
626
|
-
focused: boolean | null
|
|
627
|
-
expanded: boolean | null
|
|
628
|
-
enabled: boolean | null
|
|
629
|
-
text_value: string | null
|
|
630
|
-
value: number | string | null
|
|
631
|
-
raw_value: number | string | null
|
|
632
|
-
value_range: UIElementState['value_range']
|
|
633
|
-
}> = []
|
|
634
|
-
|
|
635
|
-
for (const el of elements) {
|
|
636
|
-
textPayload.push({
|
|
637
|
-
text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
|
|
638
|
-
contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
|
|
639
|
-
resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
|
|
640
|
-
})
|
|
641
|
-
|
|
642
|
-
statePayload.push({
|
|
643
|
-
checked: el?.state?.checked ?? null,
|
|
644
|
-
selected: el?.state?.selected ?? null,
|
|
645
|
-
focused: el?.state?.focused ?? null,
|
|
646
|
-
expanded: el?.state?.expanded ?? null,
|
|
647
|
-
enabled: el?.state?.enabled ?? null,
|
|
648
|
-
text_value: el?.state?.text_value ?? null,
|
|
649
|
-
value: el?.state?.value ?? null,
|
|
650
|
-
raw_value: el?.state?.raw_value ?? null,
|
|
651
|
-
value_range: el?.state?.value_range ?? null
|
|
652
|
-
})
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
return {
|
|
656
|
-
hierarchy: computeSnapshotSignature(tree),
|
|
657
|
-
text: ToolsInteract._hash({
|
|
658
|
-
screen: ToolsInteract._normalize(tree?.screen),
|
|
659
|
-
elements: textPayload
|
|
660
|
-
}),
|
|
661
|
-
state: ToolsInteract._hash({
|
|
662
|
-
screen: ToolsInteract._normalize(tree?.screen),
|
|
663
|
-
elements: statePayload
|
|
664
|
-
})
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
private static _matchesUiChange(expected: 'hierarchy_diff' | 'text_change' | 'state_change' | undefined, initial: UiChangeSignatureSet, current: UiChangeSignatureSet): 'hierarchy_diff' | 'text_change' | 'state_change' | null {
|
|
669
|
-
const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds
|
|
670
|
-
|
|
671
|
-
for (const changeKind of candidates) {
|
|
672
|
-
if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy) return changeKind
|
|
673
|
-
if (changeKind === 'text_change' && initial.text !== current.text) return changeKind
|
|
674
|
-
if (changeKind === 'state_change' && initial.state !== current.state) return changeKind
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
return null
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
private static _uiChangeSignaturesEqual(left: UiChangeSignatureSet, right: UiChangeSignatureSet): boolean {
|
|
681
|
-
return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
private static _resolvedTargetFromElement(
|
|
685
|
-
elementId: string,
|
|
686
|
-
element: UiElement,
|
|
687
|
-
index: number
|
|
688
|
-
): ActionTargetResolved {
|
|
689
|
-
return {
|
|
690
|
-
elementId,
|
|
691
|
-
text: element.text ?? null,
|
|
692
|
-
resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
|
|
693
|
-
accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
|
|
694
|
-
class: element.type ?? element.class ?? null,
|
|
695
|
-
bounds: ToolsInteract._normalizeBounds(element.bounds),
|
|
696
|
-
index,
|
|
697
|
-
state: element.state ?? null,
|
|
698
|
-
stable_id: element.stable_id ?? null,
|
|
699
|
-
role: element.role ?? null,
|
|
700
|
-
test_tag: element.test_tag ?? null,
|
|
701
|
-
selector: element.selector ?? null,
|
|
702
|
-
semantic: element.semantic ?? null
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
private static _summarizeResolutionCandidate(candidate: RankedResolutionCandidate): FindElementResolutionSummary['alternates'][number] {
|
|
707
|
-
const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds)
|
|
708
|
-
return {
|
|
709
|
-
text: candidate.el.text ?? null,
|
|
710
|
-
resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
|
|
711
|
-
accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
|
|
712
|
-
class: candidate.el.type ?? candidate.el.class ?? null,
|
|
713
|
-
bounds: bounds
|
|
714
|
-
? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
|
|
715
|
-
: null,
|
|
716
|
-
clickable: !!candidate.el.clickable,
|
|
717
|
-
enabled: !!candidate.el.enabled,
|
|
718
|
-
score: candidate.score,
|
|
719
|
-
reason: candidate.reason
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
private static _isAdjustableControl(el: UiElement | null): boolean {
|
|
724
|
-
if (!el) return false
|
|
725
|
-
const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
|
|
726
|
-
const role = ToolsInteract._normalize(el.role ?? '')
|
|
727
|
-
return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
private static _isSemanticActionable(el: UiElement | null): boolean {
|
|
731
|
-
if (!el?.semantic) return false
|
|
732
|
-
if (el.semantic.adjustable) return true
|
|
733
|
-
return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
|
|
737
|
-
if (!el?.state) return null
|
|
738
|
-
const stateValue = el.state[property as keyof UIElementState]
|
|
739
|
-
if (typeof stateValue === 'number' && Number.isFinite(stateValue)) return stateValue
|
|
740
|
-
const parsedStateValue = ToolsInteract._parseNumericControlValue(stateValue)
|
|
741
|
-
if (parsedStateValue !== null) return parsedStateValue
|
|
742
|
-
if (property === 'value' || property === 'raw_value') {
|
|
743
|
-
const fallback = el.state.raw_value ?? el.state.value
|
|
744
|
-
if (typeof fallback === 'number' && Number.isFinite(fallback)) return fallback
|
|
745
|
-
return ToolsInteract._parseNumericControlValue(fallback)
|
|
746
|
-
}
|
|
747
|
-
return null
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
private static _parseNumericControlValue(value: unknown): number | null {
|
|
751
|
-
if (typeof value === 'number' && Number.isFinite(value)) return value
|
|
752
|
-
if (typeof value !== 'string') return null
|
|
753
|
-
const trimmed = value.trim()
|
|
754
|
-
if (!trimmed) return null
|
|
755
|
-
|
|
756
|
-
const percent = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s*%$/)
|
|
757
|
-
if (percent) return Number(percent[1])
|
|
758
|
-
|
|
759
|
-
const fraction = trimmed.match(/^([-+]?\d+(?:\.\d+)?)\s+of\s+[-+]?\d+(?:\.\d+)?$/i)
|
|
760
|
-
if (fraction) return Number(fraction[1])
|
|
761
|
-
|
|
762
|
-
const plain = Number(trimmed)
|
|
763
|
-
if (Number.isFinite(plain)) return plain
|
|
764
|
-
|
|
765
|
-
const trailing = trimmed.match(/,\s*([-+]?\d+(?:\.\d+)?)$/)
|
|
766
|
-
if (trailing) return Number(trailing[1])
|
|
767
|
-
|
|
768
|
-
return null
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
private static _readNumericControlRange(el: UiElement | null): { min: number; max: number } | null {
|
|
772
|
-
const min = typeof el?.state?.value_range?.min === 'number' ? el.state.value_range.min : null
|
|
773
|
-
const max = typeof el?.state?.value_range?.max === 'number' ? el.state.value_range.max : null
|
|
774
|
-
if (min !== null && max !== null && Number.isFinite(min) && Number.isFinite(max) && max > min) {
|
|
775
|
-
return { min, max }
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
const value = el?.state?.value ?? el?.state?.raw_value
|
|
779
|
-
if (typeof value === 'string') {
|
|
780
|
-
const fraction = value.trim().match(/^[-+]?\d+(?:\.\d+)?\s+of\s+([-+]?\d+(?:\.\d+)?)$/i)
|
|
781
|
-
if (fraction) {
|
|
782
|
-
const parsedMax = Number(fraction[1])
|
|
783
|
-
if (Number.isFinite(parsedMax) && parsedMax > 0) return { min: 0, max: parsedMax }
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
return null
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
private static _readControlStep(el: UiElement | null): number | null {
|
|
791
|
-
const candidates = [
|
|
792
|
-
el?.state?.step,
|
|
793
|
-
el?.state?.value_range?.step
|
|
794
|
-
]
|
|
795
|
-
for (const candidate of candidates) {
|
|
796
|
-
if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0) return candidate
|
|
797
|
-
}
|
|
798
|
-
return null
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
private static _quantizeTargetValue(targetValue: number, min: number, max: number, step: number | null): number {
|
|
802
|
-
if (!step) return targetValue
|
|
803
|
-
const quantized = min + (Math.round((targetValue - min) / step) * step)
|
|
804
|
-
const bounded = Math.max(min, Math.min(max, quantized))
|
|
805
|
-
const precision = ToolsInteract._decimalPrecisionForStep(step)
|
|
806
|
-
return Number(bounded.toFixed(precision))
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
private static _decimalPrecisionForStep(step: number): number {
|
|
810
|
-
const normalized = String(step).toLowerCase()
|
|
811
|
-
const [coefficient, exponentPart] = normalized.split('e')
|
|
812
|
-
const coefficientDecimals = coefficient.includes('.')
|
|
813
|
-
? coefficient.split('.')[1]?.length ?? 0
|
|
814
|
-
: 0
|
|
815
|
-
const exponent = exponentPart !== undefined ? Number(exponentPart) : 0
|
|
816
|
-
const precision = exponent < 0
|
|
817
|
-
? coefficientDecimals + Math.abs(exponent)
|
|
818
|
-
: Math.max(0, coefficientDecimals - exponent)
|
|
819
|
-
return Math.max(0, Math.min(12, precision))
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
private static _defaultAdjustTolerance(range: { min: number; max: number } | null, step: number | null): number {
|
|
823
|
-
if (step) return 0
|
|
824
|
-
if (!range) return 0
|
|
825
|
-
return Math.max(0, 0.01 * Math.abs(range.max - range.min))
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
private static _runtimeActionSet(el: UiElement | null): Set<string> {
|
|
829
|
-
const values = [
|
|
830
|
-
...(Array.isArray(el?.actions) ? el!.actions! : []),
|
|
831
|
-
...(Array.isArray(el?.accessibilityActions) ? el!.accessibilityActions! : []),
|
|
832
|
-
...(Array.isArray(el?.runtimeActions) ? el!.runtimeActions! : []),
|
|
833
|
-
...(Array.isArray(el?.platformActions) ? el!.platformActions! : [])
|
|
834
|
-
]
|
|
835
|
-
return new Set(values.map((value) => ToolsInteract._normalize(value)).filter(Boolean))
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
private static _hasDirectAdjustmentAction(el: UiElement | null): boolean {
|
|
839
|
-
const actions = ToolsInteract._runtimeActionSet(el)
|
|
840
|
-
return [
|
|
841
|
-
'action_set_progress',
|
|
842
|
-
'set_progress',
|
|
843
|
-
'setprogress',
|
|
844
|
-
'semanticsactions.setprogress',
|
|
845
|
-
'set_value',
|
|
846
|
-
'setvalue',
|
|
847
|
-
'direct_set_value'
|
|
848
|
-
].some((action) => actions.has(action))
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
private static _adjustmentDirections(el: UiElement | null): { increment: boolean; decrement: boolean } {
|
|
852
|
-
const actions = ToolsInteract._runtimeActionSet(el)
|
|
853
|
-
return {
|
|
854
|
-
increment: ['increment', 'accessibility_increment', 'action_increment'].some((action) => actions.has(action)),
|
|
855
|
-
decrement: ['decrement', 'accessibility_decrement', 'action_decrement'].some((action) => actions.has(action))
|
|
856
|
-
}
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
private static _textFromElement(el: UiElement | null): string {
|
|
860
|
-
return String(el?.text ?? el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? el?.value ?? '').trim()
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
private static _parseLeadingNumber(text: string): number | null {
|
|
864
|
-
const match = text.trim().match(/^([-+]?\d+(?:\.\d+)?)(?:\s*[a-zA-Z%]+)?$/)
|
|
865
|
-
if (!match) return null
|
|
866
|
-
const value = Number(match[1])
|
|
867
|
-
return Number.isFinite(value) ? value : null
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
private static _inferLabeledSliderState(elements: UiElement[], track: UiElement | null): InferredLabeledSliderState | null {
|
|
871
|
-
const trackBounds = ToolsInteract._normalizeBounds(track?.bounds)
|
|
872
|
-
if (!track || !trackBounds) return null
|
|
873
|
-
const type = ToolsInteract._normalize(track.type ?? track.class ?? '')
|
|
874
|
-
const role = ToolsInteract._normalize(track.role ?? '')
|
|
875
|
-
const width = trackBounds[2] - trackBounds[0]
|
|
876
|
-
const height = trackBounds[3] - trackBounds[1]
|
|
877
|
-
const wideTrackLike = width >= 240 && height <= 220
|
|
878
|
-
const knownTrackLike = /slider|seekbar|range/.test(type) || /slider|range/.test(role)
|
|
879
|
-
if (!wideTrackLike && !knownTrackLike) return null
|
|
880
|
-
|
|
881
|
-
const trackTop = trackBounds[1]
|
|
882
|
-
const trackBottom = trackBounds[3]
|
|
883
|
-
const trackCenterX = (trackBounds[0] + trackBounds[2]) / 2
|
|
884
|
-
const labels = elements
|
|
885
|
-
.map((el, index) => {
|
|
886
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
887
|
-
const text = ToolsInteract._textFromElement(el)
|
|
888
|
-
const value = ToolsInteract._parseLeadingNumber(text)
|
|
889
|
-
return bounds && text && value !== null
|
|
890
|
-
? {
|
|
891
|
-
index,
|
|
892
|
-
el,
|
|
893
|
-
text,
|
|
894
|
-
value,
|
|
895
|
-
bounds,
|
|
896
|
-
centerX: (bounds[0] + bounds[2]) / 2,
|
|
897
|
-
centerY: (bounds[1] + bounds[3]) / 2,
|
|
898
|
-
hasUnit: /[a-zA-Z%]/.test(text)
|
|
899
|
-
}
|
|
900
|
-
: null
|
|
901
|
-
})
|
|
902
|
-
.filter((label): label is NonNullable<typeof label> => !!label)
|
|
903
|
-
|
|
904
|
-
const rangeLabels = labels
|
|
905
|
-
.filter((label) => {
|
|
906
|
-
const verticallyNearTrack = label.centerY >= trackTop - 520 && label.centerY <= trackBottom + 260
|
|
907
|
-
const horizontallyNearTrack = label.centerX >= trackBounds[0] - 120 && label.centerX <= trackBounds[2] + 120
|
|
908
|
-
return verticallyNearTrack && horizontallyNearTrack && label.hasUnit
|
|
909
|
-
})
|
|
910
|
-
.sort((a, b) => a.centerX - b.centerX)
|
|
911
|
-
|
|
912
|
-
if (rangeLabels.length < 2) return null
|
|
913
|
-
const minLabel = rangeLabels[0]
|
|
914
|
-
const maxLabel = rangeLabels[rangeLabels.length - 1]
|
|
915
|
-
if (!minLabel || !maxLabel || maxLabel.value <= minLabel.value) return null
|
|
916
|
-
|
|
917
|
-
const currentCandidates = labels
|
|
918
|
-
.filter((label) => {
|
|
919
|
-
if (label.index === minLabel.index || label.index === maxLabel.index) return false
|
|
920
|
-
if (label.value < minLabel.value || label.value > maxLabel.value) return false
|
|
921
|
-
if (label.centerY < trackTop - 950 || label.centerY > minLabel.centerY - 20) return false
|
|
922
|
-
return label.centerX >= trackCenterX - 80
|
|
923
|
-
})
|
|
924
|
-
.sort((a, b) => {
|
|
925
|
-
const yDistance = Math.abs((trackTop - 420) - a.centerY) - Math.abs((trackTop - 420) - b.centerY)
|
|
926
|
-
if (yDistance !== 0) return yDistance
|
|
927
|
-
return b.centerX - a.centerX
|
|
928
|
-
})
|
|
929
|
-
|
|
930
|
-
const currentLabel = currentCandidates[0]
|
|
931
|
-
if (!currentLabel) return null
|
|
932
|
-
|
|
933
|
-
const candidateBelowRangeLabels = minLabel.centerY < trackTop && maxLabel.centerY < trackTop
|
|
934
|
-
const labelDerivedBounds: [number, number, number, number] = [
|
|
935
|
-
Math.min(minLabel.bounds[0], trackBounds[0]),
|
|
936
|
-
Math.max(currentLabel.bounds[3] + 12, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 160),
|
|
937
|
-
Math.max(maxLabel.bounds[2], trackBounds[2]),
|
|
938
|
-
Math.max(currentLabel.bounds[3] + 80, Math.min(minLabel.bounds[1], maxLabel.bounds[1]) - 32)
|
|
939
|
-
]
|
|
940
|
-
|
|
941
|
-
return {
|
|
942
|
-
current: currentLabel.value,
|
|
943
|
-
min: minLabel.value,
|
|
944
|
-
max: maxLabel.value,
|
|
945
|
-
value_range: { min: minLabel.value, max: maxLabel.value },
|
|
946
|
-
touch_bounds: candidateBelowRangeLabels ? labelDerivedBounds : trackBounds,
|
|
947
|
-
source: 'nearby_labels'
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
|
|
951
|
-
private static _buildControlPoint(bounds: [number, number, number, number], ratio: number, axis: 'horizontal' | 'vertical') {
|
|
952
|
-
const clampedRatio = Math.max(0, Math.min(1, ratio))
|
|
953
|
-
const [left, top, right, bottom] = bounds
|
|
954
|
-
const width = Math.max(1, right - left)
|
|
955
|
-
const height = Math.max(1, bottom - top)
|
|
956
|
-
const insetX = Math.max(8, Math.floor(width * 0.08))
|
|
957
|
-
const insetY = Math.max(8, Math.floor(height * 0.08))
|
|
958
|
-
if (axis === 'vertical') {
|
|
959
|
-
const usableHeight = Math.max(1, height - (insetY * 2))
|
|
960
|
-
return {
|
|
961
|
-
x: Math.floor((left + right) / 2),
|
|
962
|
-
y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
|
|
963
|
-
}
|
|
964
|
-
}
|
|
965
|
-
const usableWidth = Math.max(1, width - (insetX * 2))
|
|
966
|
-
return {
|
|
967
|
-
x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
|
|
968
|
-
y: Math.floor((top + bottom) / 2)
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
private static _buildConservativeControlPoint(
|
|
973
|
-
bounds: [number, number, number, number],
|
|
974
|
-
targetValue: number,
|
|
975
|
-
currentValue: number | null,
|
|
976
|
-
min: number,
|
|
977
|
-
max: number,
|
|
978
|
-
axis: 'horizontal' | 'vertical'
|
|
979
|
-
) {
|
|
980
|
-
const range = Math.max(1, max - min)
|
|
981
|
-
const targetRatio = (targetValue - min) / range
|
|
982
|
-
const stepRatio = 1 / range
|
|
983
|
-
const centerBias = stepRatio / 2
|
|
984
|
-
const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue)
|
|
985
|
-
const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0])
|
|
986
|
-
const edgeWindow = Math.max(3, Math.floor(range * 0.1))
|
|
987
|
-
const isNearLowEdge = targetValue - min <= edgeWindow
|
|
988
|
-
const isNearHighEdge = max - targetValue <= edgeWindow
|
|
989
|
-
const directionBias = direction > 0
|
|
990
|
-
? -stepRatio * 0.15
|
|
991
|
-
: direction < 0
|
|
992
|
-
? stepRatio * 0.65
|
|
993
|
-
: 0
|
|
994
|
-
const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx))
|
|
995
|
-
const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin)
|
|
996
|
-
const edgeBias = isNearLowEdge
|
|
997
|
-
? endpointMargin
|
|
998
|
-
: isNearHighEdge
|
|
999
|
-
? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
|
|
1000
|
-
: 0
|
|
1001
|
-
const safeRatio = Math.min(
|
|
1002
|
-
1 - (endpointMargin * 0.25),
|
|
1003
|
-
Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias)
|
|
1004
|
-
)
|
|
1005
|
-
return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
|
|
1006
|
-
}
|
|
1007
|
-
|
|
1008
|
-
private static _buildAdjustmentProbePoints(
|
|
1009
|
-
bounds: [number, number, number, number],
|
|
1010
|
-
targetValue: number,
|
|
1011
|
-
currentValue: number | null,
|
|
1012
|
-
min: number,
|
|
1013
|
-
max: number,
|
|
1014
|
-
axis: 'horizontal' | 'vertical'
|
|
1015
|
-
) {
|
|
1016
|
-
const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
|
|
1017
|
-
const currentPoint = currentValue !== null
|
|
1018
|
-
? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
|
|
1019
|
-
: ToolsInteract._buildControlPoint(bounds, 0.5, axis)
|
|
1020
|
-
|
|
1021
|
-
const [left, top, right, bottom] = bounds
|
|
1022
|
-
const width = Math.max(1, right - left)
|
|
1023
|
-
const height = Math.max(1, bottom - top)
|
|
1024
|
-
const crossAxisBumps = axis === 'horizontal'
|
|
1025
|
-
? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
|
|
1026
|
-
: [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))]
|
|
1027
|
-
|
|
1028
|
-
const clampPoint = (point: { x: number, y: number }) => ({
|
|
1029
|
-
x: axis === 'horizontal'
|
|
1030
|
-
? Math.max(left, Math.min(right, point.x))
|
|
1031
|
-
: Math.max(left, Math.min(right + Math.max(width, height), point.x)),
|
|
1032
|
-
y: axis === 'vertical'
|
|
1033
|
-
? Math.max(top, Math.min(bottom, point.y))
|
|
1034
|
-
: Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
|
|
1035
|
-
})
|
|
1036
|
-
|
|
1037
|
-
const probes = [targetPoint, currentPoint]
|
|
1038
|
-
for (const bump of crossAxisBumps) {
|
|
1039
|
-
if (axis === 'horizontal') {
|
|
1040
|
-
probes.push(
|
|
1041
|
-
{ x: targetPoint.x, y: bottom + bump },
|
|
1042
|
-
{ x: currentPoint.x, y: bottom + bump }
|
|
1043
|
-
)
|
|
1044
|
-
} else {
|
|
1045
|
-
probes.push(
|
|
1046
|
-
{ x: right + bump, y: targetPoint.y },
|
|
1047
|
-
{ x: right + bump, y: currentPoint.y }
|
|
1048
|
-
)
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
return Array.from(
|
|
1053
|
-
new Map(
|
|
1054
|
-
probes
|
|
1055
|
-
.map(clampPoint)
|
|
1056
|
-
.map((point) => [`${point.x}:${point.y}`, point] as const)
|
|
1057
|
-
).values()
|
|
1058
|
-
)
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
|
-
private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
|
|
1062
|
-
const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
|
|
1063
|
-
const role = ToolsInteract._normalize(el.role ?? '')
|
|
1064
|
-
if (/vertical/.test(type) || /vertical/.test(role)) return 'vertical'
|
|
1065
|
-
if (/horizontal/.test(type) || /horizontal/.test(role)) return 'horizontal'
|
|
1066
|
-
return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal'
|
|
1067
|
-
}
|
|
106
|
+
private static _resolvedTargetFromElement = _resolvedTargetFromElement
|
|
1068
107
|
|
|
1069
108
|
private static _actionFailure(
|
|
1070
109
|
actionType: string,
|
|
@@ -1106,129 +145,11 @@ export class ToolsInteract {
|
|
|
1106
145
|
return null
|
|
1107
146
|
}
|
|
1108
147
|
|
|
1109
|
-
private static _resolveActionableAncestor(elements: UiElement[], chosen: { el: UiElement, idx: number } | null): { el: UiElement, idx: number } | null {
|
|
1110
|
-
if (!chosen) return null
|
|
1111
|
-
if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el)) return chosen
|
|
1112
|
-
|
|
1113
|
-
let current = chosen
|
|
1114
|
-
let safety = 0
|
|
1115
|
-
|
|
1116
|
-
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
1117
|
-
const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId)
|
|
1118
|
-
|
|
1119
|
-
if (parentIndex !== null && elements[parentIndex]) {
|
|
1120
|
-
current = { el: elements[parentIndex], idx: parentIndex }
|
|
1121
|
-
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
|
|
1122
|
-
} else {
|
|
1123
|
-
break
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
safety++
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
|
|
1130
|
-
if (!childBounds) return null
|
|
1131
|
-
const [cl, ct, cr, cb] = childBounds
|
|
1132
|
-
|
|
1133
|
-
let best: { el: UiElement, idx: number } | null = null
|
|
1134
|
-
let bestArea = Infinity
|
|
1135
|
-
|
|
1136
|
-
for (let i = 0; i < elements.length; i++) {
|
|
1137
|
-
const el = elements[i]
|
|
1138
|
-
if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el))) continue
|
|
1139
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
1140
|
-
if (!bounds) continue
|
|
1141
|
-
const [pl, pt, pr, pb] = bounds
|
|
1142
|
-
if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
|
|
1143
|
-
const area = (pr - pl) * (pb - pt)
|
|
1144
|
-
if (area < bestArea) {
|
|
1145
|
-
bestArea = area
|
|
1146
|
-
best = { el, idx: i }
|
|
1147
|
-
}
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
return best
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
private static _resolveNearbyActionableControl(
|
|
1155
|
-
elements: UiElement[],
|
|
1156
|
-
chosen: { el: UiElement, idx: number } | null,
|
|
1157
|
-
screen?: UiResolution | null
|
|
1158
|
-
): { el: UiElement, idx: number, sliderLike?: boolean } | null {
|
|
1159
|
-
if (!chosen) return null
|
|
1160
|
-
|
|
1161
|
-
const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
|
|
1162
|
-
if (!labelBounds) return null
|
|
1163
|
-
|
|
1164
|
-
const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds
|
|
1165
|
-
const labelWidth = labelRight - labelLeft
|
|
1166
|
-
const labelHeight = labelBottom - labelTop
|
|
1167
|
-
const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0
|
|
1168
|
-
const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0
|
|
1169
|
-
const minTrackLengthPx = Math.max(
|
|
1170
|
-
ToolsInteract._sliderTrackMinLengthPx,
|
|
1171
|
-
screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0,
|
|
1172
|
-
screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0
|
|
1173
|
-
)
|
|
1174
|
-
const maxTrackThicknessPx = Math.max(
|
|
1175
|
-
ToolsInteract._sliderTrackMaxThicknessPx,
|
|
1176
|
-
screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0,
|
|
1177
|
-
screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0
|
|
1178
|
-
)
|
|
1179
|
-
|
|
1180
|
-
let best: { el: UiElement, idx: number, sliderLike?: boolean } | null = null
|
|
1181
|
-
let bestScore = Infinity
|
|
1182
|
-
|
|
1183
|
-
for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
|
|
1184
|
-
const candidate = elements[i]
|
|
1185
|
-
if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false) continue
|
|
1186
|
-
|
|
1187
|
-
const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds)
|
|
1188
|
-
if (!candidateBounds) continue
|
|
1189
|
-
|
|
1190
|
-
const [left, top, right] = candidateBounds
|
|
1191
|
-
const width = right - left
|
|
1192
|
-
const height = candidateBounds[3] - top
|
|
1193
|
-
const verticalGap = top - labelBottom
|
|
1194
|
-
if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx) continue
|
|
1195
|
-
|
|
1196
|
-
const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left)
|
|
1197
|
-
if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx) continue
|
|
1198
|
-
|
|
1199
|
-
const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '')
|
|
1200
|
-
const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '')
|
|
1201
|
-
const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '')
|
|
1202
|
-
|
|
1203
|
-
let score = verticalGap
|
|
1204
|
-
const horizontalTrackLike =
|
|
1205
|
-
width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
|
|
1206
|
-
height <= maxTrackThicknessPx
|
|
1207
|
-
const verticalTrackLike =
|
|
1208
|
-
height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
|
|
1209
|
-
width <= maxTrackThicknessPx
|
|
1210
|
-
const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike
|
|
1211
|
-
if (!candidateText && !candidateContent) score -= 18
|
|
1212
|
-
if (trackLike) score -= 30
|
|
1213
|
-
if (/view|layout|group|frame/i.test(candidateClass)) score -= 10
|
|
1214
|
-
if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio) score -= 8
|
|
1215
|
-
if (candidateText || candidateContent) score += 20
|
|
1216
|
-
|
|
1217
|
-
if (score < bestScore) {
|
|
1218
|
-
bestScore = score
|
|
1219
|
-
best = { el: candidate, idx: i, sliderLike: trackLike }
|
|
1220
|
-
}
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
return best
|
|
1224
|
-
}
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
148
|
private static async getInteractionService(platform?: 'android' | 'ios', deviceId?: string) {
|
|
1228
149
|
const effectivePlatform = platform || 'android'
|
|
1229
150
|
const resolved = await resolveTargetDevice({ platform: effectivePlatform as 'android' | 'ios', deviceId })
|
|
1230
|
-
const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract()
|
|
1231
|
-
return { interact
|
|
151
|
+
const interact: InputService = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract()
|
|
152
|
+
return { interact, resolved, platform: effectivePlatform }
|
|
1232
153
|
}
|
|
1233
154
|
|
|
1234
155
|
static async tapHandler({ platform, x, y, deviceId, timeoutMs }: { platform?: 'android' | 'ios', x: number, y: number, deviceId?: string, timeoutMs?: number }) {
|
|
@@ -1236,7 +157,7 @@ export class ToolsInteract {
|
|
|
1236
157
|
return await interact.tap(x, y, resolved.id, timeoutMs)
|
|
1237
158
|
}
|
|
1238
159
|
|
|
1239
|
-
static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }: { elementId: string, verificationMode?: 'none' | 'light' | 'full', timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string, freshTree?:
|
|
160
|
+
static async tapElementHandler({ elementId, verificationMode = 'full', timeoutMs, platform, deviceId, freshTree }: { elementId: string, verificationMode?: 'none' | 'light' | 'full', timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string, freshTree?: UiTree }): Promise<TapElementResponse> {
|
|
1240
161
|
const actionType = 'tap_element'
|
|
1241
162
|
const selector = { elementId }
|
|
1242
163
|
const resolved = ToolsInteract._resolvedUiElements.get(elementId)
|
|
@@ -1246,37 +167,14 @@ export class ToolsInteract {
|
|
|
1246
167
|
|
|
1247
168
|
const fingerprintBefore = verificationMode === 'full' ? await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId) : null
|
|
1248
169
|
|
|
1249
|
-
const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId })
|
|
1250
|
-
const
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
|
|
1260
|
-
|
|
1261
|
-
const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform)
|
|
1262
|
-
if (!tapActionability.actionable) {
|
|
1263
|
-
return ToolsInteract._actionFailure(
|
|
1264
|
-
actionType,
|
|
1265
|
-
selector,
|
|
1266
|
-
resolvedTarget,
|
|
1267
|
-
tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
|
|
1268
|
-
true,
|
|
1269
|
-
fingerprintBefore
|
|
1270
|
-
)
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
|
|
1274
|
-
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
|
|
1275
|
-
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
|
-
const x = Math.floor((bounds[0] + bounds[2]) / 2)
|
|
1279
|
-
const y = Math.floor((bounds[1] + bounds[3]) / 2)
|
|
170
|
+
const tree = freshTree ?? await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId })
|
|
171
|
+
const tap = resolveTap(tree, resolved, {
|
|
172
|
+
current: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
|
|
173
|
+
target: (element, index) => ToolsInteract._resolvedTargetFromElement(resolved.elementId, element, index)
|
|
174
|
+
})
|
|
175
|
+
if ('failure' in tap) return ToolsInteract._actionFailure(actionType, selector, tap.target, tap.failure, true, fingerprintBefore)
|
|
176
|
+
const resolvedTarget = tap.target
|
|
177
|
+
const { x, y } = tap.point
|
|
1280
178
|
const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId, timeoutMs })
|
|
1281
179
|
|
|
1282
180
|
if (!tapResult.success) {
|
|
@@ -1297,916 +195,17 @@ export class ToolsInteract {
|
|
|
1297
195
|
})
|
|
1298
196
|
}
|
|
1299
197
|
|
|
1300
|
-
static async adjustControlHandler({
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
element_id?: string,
|
|
1312
|
-
property?: string,
|
|
1313
|
-
targetValue: number,
|
|
1314
|
-
tolerance?: number,
|
|
1315
|
-
maxAttempts?: number,
|
|
1316
|
-
platform?: 'android' | 'ios',
|
|
1317
|
-
deviceId?: string
|
|
1318
|
-
}): Promise<AdjustControlResponse> {
|
|
1319
|
-
const actionType = 'adjust_control'
|
|
1320
|
-
const targetSelector = selector ?? (element_id ? { elementId: element_id } : null)
|
|
1321
|
-
const requestedTolerance = typeof tolerance === 'number' && Number.isFinite(tolerance) ? Math.max(0, tolerance) : null
|
|
1322
|
-
const attemptsLimit = maxAttempts === undefined || maxAttempts === null
|
|
1323
|
-
? 5
|
|
1324
|
-
: Math.max(1, Math.floor(Number(maxAttempts) || 1))
|
|
1325
|
-
const sourcePlatform: 'android' | 'ios' = platform || 'android'
|
|
1326
|
-
let resolvedPlatform = sourcePlatform
|
|
1327
|
-
let resolvedDeviceId = deviceId
|
|
1328
|
-
const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null
|
|
1329
|
-
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1330
|
-
let semanticFallbackElement: FindElementResponse['element'] | null = null
|
|
1331
|
-
const traceSteps: TraceStep[] = []
|
|
1332
|
-
let traceAttemptIndex = 0
|
|
1333
|
-
|
|
1334
|
-
const recordTraceStep = (
|
|
1335
|
-
stage: TraceStep['stage'],
|
|
1336
|
-
result: TraceStep['result'],
|
|
1337
|
-
metadata?: Record<string, unknown>
|
|
1338
|
-
) => {
|
|
1339
|
-
traceSteps.push(createTraceStep({
|
|
1340
|
-
stage,
|
|
1341
|
-
timestamp: Date.now(),
|
|
1342
|
-
result,
|
|
1343
|
-
attemptIndex: traceAttemptIndex++,
|
|
1344
|
-
metadata
|
|
1345
|
-
}))
|
|
1346
|
-
}
|
|
1347
|
-
|
|
1348
|
-
const buildFailure = (
|
|
1349
|
-
failureCode: ActionFailureCode,
|
|
1350
|
-
reason: string,
|
|
1351
|
-
resolved: ActionTargetResolved | null,
|
|
1352
|
-
device: any,
|
|
1353
|
-
actualState: { property: string; value: number | null; raw_value?: number | null } | null,
|
|
1354
|
-
attempts: number,
|
|
1355
|
-
adjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture',
|
|
1356
|
-
retryable = false,
|
|
1357
|
-
uiFingerprintAfter: string | null = null
|
|
1358
|
-
): AdjustControlResponse => {
|
|
1359
|
-
if (!traceSteps.some((step) => step.stage === 'resolve')) {
|
|
1360
|
-
recordTraceStep('resolve', 'failure',
|
|
1361
|
-
{
|
|
1362
|
-
reason,
|
|
1363
|
-
failure_code: failureCode
|
|
1364
|
-
})
|
|
1365
|
-
}
|
|
1366
|
-
if (!traceSteps.some((step) => step.stage === 'recover')) {
|
|
1367
|
-
recordTraceStep('recover', retryable ? 'retry' : 'failure', {
|
|
1368
|
-
reason,
|
|
1369
|
-
failure_code: failureCode,
|
|
1370
|
-
retry_allowed: retryable,
|
|
1371
|
-
recovery_attempts: attempts,
|
|
1372
|
-
retry_depth: attempts
|
|
1373
|
-
})
|
|
1374
|
-
}
|
|
1375
|
-
const responseTolerance = requestedTolerance ?? 0
|
|
1376
|
-
const base = buildActionExecutionResult({
|
|
1377
|
-
actionType,
|
|
1378
|
-
sourceModule: 'interact',
|
|
1379
|
-
device,
|
|
1380
|
-
selector: targetSelector,
|
|
1381
|
-
resolved,
|
|
1382
|
-
success: false,
|
|
1383
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1384
|
-
uiFingerprintAfter,
|
|
1385
|
-
failure: { failureCode, retryable },
|
|
1386
|
-
details: {
|
|
1387
|
-
target_value: targetValue,
|
|
1388
|
-
tolerance: responseTolerance,
|
|
1389
|
-
property,
|
|
1390
|
-
attempts,
|
|
1391
|
-
adjustment_mode: adjustmentMode,
|
|
1392
|
-
actual_state: actualState,
|
|
1393
|
-
converged: false,
|
|
1394
|
-
within_tolerance: false,
|
|
1395
|
-
reason
|
|
1396
|
-
},
|
|
1397
|
-
traceSteps
|
|
1398
|
-
}) as AdjustControlResponse
|
|
1399
|
-
|
|
1400
|
-
return {
|
|
1401
|
-
...base,
|
|
1402
|
-
target_state: {
|
|
1403
|
-
property,
|
|
1404
|
-
target_value: targetValue,
|
|
1405
|
-
tolerance: responseTolerance
|
|
1406
|
-
},
|
|
1407
|
-
actual_state: actualState,
|
|
1408
|
-
within_tolerance: false,
|
|
1409
|
-
converged: false,
|
|
1410
|
-
attempts,
|
|
1411
|
-
adjustment_mode: adjustmentMode
|
|
1412
|
-
}
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
const resolveCurrentMatch = async (): Promise<{
|
|
1416
|
-
tree: any
|
|
1417
|
-
device: any
|
|
1418
|
-
match: { el: UiElement, idx: number } | null
|
|
1419
|
-
resolvedTarget: ActionTargetResolved | null
|
|
1420
|
-
} | null> => {
|
|
1421
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
|
|
1422
|
-
resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform
|
|
1423
|
-
resolvedDeviceId = tree?.device?.id || resolvedDeviceId
|
|
1424
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
1425
|
-
|
|
1426
|
-
if (element_id) {
|
|
1427
|
-
const stored = ToolsInteract._resolvedUiElements.get(element_id)
|
|
1428
|
-
if (!stored) {
|
|
1429
|
-
return null
|
|
1430
|
-
}
|
|
1431
|
-
const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored)
|
|
1432
|
-
if (!current) {
|
|
1433
|
-
return null
|
|
1434
|
-
}
|
|
1435
|
-
return {
|
|
1436
|
-
tree,
|
|
1437
|
-
device: tree?.device,
|
|
1438
|
-
match: { el: current.el, idx: current.index },
|
|
1439
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
1440
|
-
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index),
|
|
1441
|
-
current.el,
|
|
1442
|
-
current.index
|
|
1443
|
-
)
|
|
1444
|
-
}
|
|
1445
|
-
}
|
|
1446
|
-
|
|
1447
|
-
if (semanticFallbackElement) {
|
|
1448
|
-
const fallbackBounds = ToolsInteract._normalizeBounds(
|
|
1449
|
-
Array.isArray(semanticFallbackElement.bounds)
|
|
1450
|
-
? semanticFallbackElement.bounds
|
|
1451
|
-
: semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
|
|
1452
|
-
? [
|
|
1453
|
-
Number((semanticFallbackElement.bounds as any).left),
|
|
1454
|
-
Number((semanticFallbackElement.bounds as any).top),
|
|
1455
|
-
Number((semanticFallbackElement.bounds as any).right),
|
|
1456
|
-
Number((semanticFallbackElement.bounds as any).bottom)
|
|
1457
|
-
]
|
|
1458
|
-
: null
|
|
1459
|
-
)
|
|
1460
|
-
|
|
1461
|
-
let matchedIndex = -1
|
|
1462
|
-
if (fallbackBounds) {
|
|
1463
|
-
matchedIndex = elements.findIndex((el) => {
|
|
1464
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
1465
|
-
return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3]
|
|
1466
|
-
})
|
|
1467
|
-
}
|
|
1468
|
-
|
|
1469
|
-
if (matchedIndex === -1 && fallbackBounds) {
|
|
1470
|
-
const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2)
|
|
1471
|
-
const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2)
|
|
1472
|
-
let bestDistance = Infinity
|
|
1473
|
-
for (let i = 0; i < elements.length; i++) {
|
|
1474
|
-
const el = elements[i]
|
|
1475
|
-
if (!ToolsInteract._isAdjustableControl(el)) continue
|
|
1476
|
-
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
1477
|
-
if (!bounds) continue
|
|
1478
|
-
const centerX = Math.floor((bounds[0] + bounds[2]) / 2)
|
|
1479
|
-
const centerY = Math.floor((bounds[1] + bounds[3]) / 2)
|
|
1480
|
-
const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY)
|
|
1481
|
-
if (distance < bestDistance) {
|
|
1482
|
-
bestDistance = distance
|
|
1483
|
-
matchedIndex = i
|
|
1484
|
-
}
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
|
|
1488
|
-
if (matchedIndex >= 0 && elements[matchedIndex]) {
|
|
1489
|
-
const matched = { el: elements[matchedIndex], idx: matchedIndex }
|
|
1490
|
-
return {
|
|
1491
|
-
tree,
|
|
1492
|
-
device: tree?.device,
|
|
1493
|
-
match: matched,
|
|
1494
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
1495
|
-
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
|
|
1496
|
-
matched.el,
|
|
1497
|
-
matched.idx
|
|
1498
|
-
)
|
|
1499
|
-
}
|
|
1500
|
-
}
|
|
1501
|
-
}
|
|
1502
|
-
|
|
1503
|
-
if (selector) {
|
|
1504
|
-
const matched = ToolsInteract._findFirstMatchingElement(elements, selector)
|
|
1505
|
-
if (!matched) {
|
|
1506
|
-
return null
|
|
1507
|
-
}
|
|
1508
|
-
return {
|
|
1509
|
-
tree,
|
|
1510
|
-
device: tree?.device,
|
|
1511
|
-
match: matched,
|
|
1512
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
1513
|
-
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
|
|
1514
|
-
matched.el,
|
|
1515
|
-
matched.idx
|
|
1516
|
-
)
|
|
1517
|
-
}
|
|
1518
|
-
}
|
|
1519
|
-
|
|
1520
|
-
return null
|
|
1521
|
-
}
|
|
1522
|
-
|
|
1523
|
-
if ((!selector && !element_id) || (selector && element_id)) {
|
|
1524
|
-
return buildFailure('ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', null, undefined, null, 0, 'gesture', false)
|
|
1525
|
-
}
|
|
1526
|
-
|
|
1527
|
-
if (selector && !element_id) {
|
|
1528
|
-
const waitResult = await ToolsInteract.waitForUIHandler({
|
|
1529
|
-
selector,
|
|
1530
|
-
condition: 'clickable',
|
|
1531
|
-
timeout_ms: 5000,
|
|
1532
|
-
poll_interval_ms: 300,
|
|
1533
|
-
platform: resolvedPlatform,
|
|
1534
|
-
deviceId: resolvedDeviceId
|
|
1535
|
-
}) as any
|
|
1536
|
-
|
|
1537
|
-
if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
|
|
1538
|
-
const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
|
|
1539
|
-
if (!semanticQuery) {
|
|
1540
|
-
return buildFailure(
|
|
1541
|
-
waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
|
|
1542
|
-
waitResult?.error?.message ?? 'adjustable control not found',
|
|
1543
|
-
null,
|
|
1544
|
-
waitResult?.device,
|
|
1545
|
-
null,
|
|
1546
|
-
0,
|
|
1547
|
-
'gesture',
|
|
1548
|
-
waitResult?.error?.code === 'ELEMENT_NOT_FOUND'
|
|
1549
|
-
)
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
const fallback = await ToolsInteract.findElementHandler({
|
|
1553
|
-
query: semanticQuery,
|
|
1554
|
-
exact: false,
|
|
1555
|
-
timeoutMs: 3000,
|
|
1556
|
-
platform: resolvedPlatform,
|
|
1557
|
-
deviceId: resolvedDeviceId
|
|
1558
|
-
})
|
|
1559
|
-
|
|
1560
|
-
if (!fallback.found || !fallback.element) {
|
|
1561
|
-
return buildFailure(
|
|
1562
|
-
'ELEMENT_NOT_FOUND',
|
|
1563
|
-
waitResult?.error?.message ?? 'adjustable control not found',
|
|
1564
|
-
null,
|
|
1565
|
-
waitResult?.device,
|
|
1566
|
-
null,
|
|
1567
|
-
0,
|
|
1568
|
-
'gesture',
|
|
1569
|
-
true
|
|
1570
|
-
)
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
semanticFallbackElement = fallback.element
|
|
1574
|
-
} else {
|
|
1575
|
-
element_id = waitResult.element.elementId
|
|
1576
|
-
semanticFallbackElement = null
|
|
1577
|
-
}
|
|
1578
|
-
}
|
|
1579
|
-
|
|
1580
|
-
let lastObservedState: { property: string; value: number | null; raw_value?: number | null } | null = null
|
|
1581
|
-
let lastAdjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture'
|
|
1582
|
-
let resolvedTarget: ActionTargetResolved | null = null
|
|
1583
|
-
let currentDevice: any = undefined
|
|
1584
|
-
let attemptCount = 0
|
|
1585
|
-
let cachedResolvedMatch: { el: UiElement, idx: number } | null = null
|
|
1586
|
-
let coordinateTapsAwaitingFreshGesture = false
|
|
1587
|
-
|
|
1588
|
-
adjustmentAttempts:
|
|
1589
|
-
for (let attempt = 0; attempt < attemptsLimit && attemptCount < attemptsLimit; attempt++) {
|
|
1590
|
-
const resolved: {
|
|
1591
|
-
tree: any
|
|
1592
|
-
device: any
|
|
1593
|
-
match: { el: UiElement, idx: number } | null
|
|
1594
|
-
resolvedTarget: ActionTargetResolved | null
|
|
1595
|
-
} | null = cachedResolvedMatch
|
|
1596
|
-
? {
|
|
1597
|
-
tree: null,
|
|
1598
|
-
device: currentDevice,
|
|
1599
|
-
match: cachedResolvedMatch,
|
|
1600
|
-
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
1601
|
-
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx),
|
|
1602
|
-
cachedResolvedMatch.el,
|
|
1603
|
-
cachedResolvedMatch.idx
|
|
1604
|
-
)
|
|
1605
|
-
}
|
|
1606
|
-
: await resolveCurrentMatch()
|
|
1607
|
-
if (!resolved || !resolved.match || !resolved.resolvedTarget) {
|
|
1608
|
-
return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true)
|
|
1609
|
-
}
|
|
1610
|
-
|
|
1611
|
-
currentDevice = resolved.device
|
|
1612
|
-
resolvedTarget = resolved.resolvedTarget
|
|
1613
|
-
const currentEl: UiElement = resolved.match.el
|
|
1614
|
-
cachedResolvedMatch = resolved.match
|
|
1615
|
-
|
|
1616
|
-
const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id)
|
|
1617
|
-
if (!adjustableActionability.actionable) {
|
|
1618
|
-
return buildFailure(
|
|
1619
|
-
adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
|
|
1620
|
-
adjustableActionability.reason ?? 'adjustable control is not actionable',
|
|
1621
|
-
resolvedTarget,
|
|
1622
|
-
currentDevice,
|
|
1623
|
-
lastObservedState,
|
|
1624
|
-
attemptCount,
|
|
1625
|
-
lastAdjustmentMode,
|
|
1626
|
-
true
|
|
1627
|
-
)
|
|
1628
|
-
}
|
|
1629
|
-
|
|
1630
|
-
const elementsForInference = Array.isArray(resolved.tree?.elements)
|
|
1631
|
-
? resolved.tree.elements as UiElement[]
|
|
1632
|
-
: cachedResolvedMatch
|
|
1633
|
-
? [cachedResolvedMatch.el]
|
|
1634
|
-
: [currentEl]
|
|
1635
|
-
const inferredSliderState = ToolsInteract._inferLabeledSliderState(elementsForInference, currentEl)
|
|
1636
|
-
const nativeBounds = ToolsInteract._normalizeBounds(currentEl.bounds)
|
|
1637
|
-
const bounds = inferredSliderState?.touch_bounds ?? nativeBounds
|
|
1638
|
-
const valueRange = ToolsInteract._readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null
|
|
1639
|
-
const step = ToolsInteract._readControlStep(currentEl)
|
|
1640
|
-
const normalizedTargetValue = valueRange
|
|
1641
|
-
? ToolsInteract._quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step)
|
|
1642
|
-
: targetValue
|
|
1643
|
-
const effectiveTolerance = requestedTolerance ?? ToolsInteract._defaultAdjustTolerance(valueRange, step)
|
|
1644
|
-
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null
|
|
1645
|
-
const actualState = currentValue !== null
|
|
1646
|
-
? {
|
|
1647
|
-
property,
|
|
1648
|
-
value: currentValue,
|
|
1649
|
-
raw_value: typeof currentEl.state?.raw_value === 'number'
|
|
1650
|
-
? currentEl.state.raw_value
|
|
1651
|
-
: inferredSliderState?.current
|
|
1652
|
-
}
|
|
1653
|
-
: null
|
|
1654
|
-
|
|
1655
|
-
lastObservedState = actualState
|
|
1656
|
-
|
|
1657
|
-
if (!traceSteps.some((step) => step.stage === 'resolve')) {
|
|
1658
|
-
recordTraceStep('resolve', 'success', {
|
|
1659
|
-
resolved_target: resolvedTarget,
|
|
1660
|
-
current_value: currentValue,
|
|
1661
|
-
adjustment_mode: lastAdjustmentMode
|
|
1662
|
-
})
|
|
1663
|
-
}
|
|
1664
|
-
|
|
1665
|
-
if (property !== 'value' && property !== 'raw_value') {
|
|
1666
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1667
|
-
}
|
|
1668
|
-
|
|
1669
|
-
const isAdjustableControl = ToolsInteract._isAdjustableControl(currentEl) || !!inferredSliderState
|
|
1670
|
-
if (!isAdjustableControl) {
|
|
1671
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1672
|
-
}
|
|
1673
|
-
|
|
1674
|
-
if (currentValue === null) {
|
|
1675
|
-
return buildFailure('UNKNOWN', 'current numeric value is unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
if (valueRange && (targetValue < valueRange.min || targetValue > valueRange.max)) {
|
|
1679
|
-
return buildFailure('CONTROL_CONVERGENCE_FAILED', `targetValue ${targetValue} is outside the control range ${valueRange.min}..${valueRange.max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1680
|
-
}
|
|
1681
|
-
|
|
1682
|
-
if (currentValue !== null && Math.abs(currentValue - normalizedTargetValue) <= effectiveTolerance) {
|
|
1683
|
-
recordTraceStep('verify', 'success', {
|
|
1684
|
-
property,
|
|
1685
|
-
target_value: normalizedTargetValue,
|
|
1686
|
-
actual_state: actualState,
|
|
1687
|
-
reason: 'control already within tolerance'
|
|
1688
|
-
})
|
|
1689
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1690
|
-
const base = buildActionExecutionResult({
|
|
1691
|
-
actionType,
|
|
1692
|
-
sourceModule: 'interact',
|
|
1693
|
-
device: currentDevice,
|
|
1694
|
-
selector: targetSelector,
|
|
1695
|
-
resolved: resolvedTarget,
|
|
1696
|
-
success: true,
|
|
1697
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1698
|
-
uiFingerprintAfter,
|
|
1699
|
-
details: {
|
|
1700
|
-
target_value: normalizedTargetValue,
|
|
1701
|
-
tolerance: effectiveTolerance,
|
|
1702
|
-
property,
|
|
1703
|
-
attempts: attemptCount,
|
|
1704
|
-
adjustment_mode: 'semantic',
|
|
1705
|
-
actual_state: actualState,
|
|
1706
|
-
converged: true,
|
|
1707
|
-
within_tolerance: true,
|
|
1708
|
-
reason: 'control already within tolerance'
|
|
1709
|
-
},
|
|
1710
|
-
traceSteps
|
|
1711
|
-
}) as AdjustControlResponse
|
|
1712
|
-
|
|
1713
|
-
return {
|
|
1714
|
-
...base,
|
|
1715
|
-
target_state: {
|
|
1716
|
-
property,
|
|
1717
|
-
target_value: normalizedTargetValue,
|
|
1718
|
-
tolerance: effectiveTolerance
|
|
1719
|
-
},
|
|
1720
|
-
actual_state: actualState,
|
|
1721
|
-
within_tolerance: true,
|
|
1722
|
-
converged: true,
|
|
1723
|
-
attempts: attemptCount,
|
|
1724
|
-
adjustment_mode: 'semantic'
|
|
1725
|
-
}
|
|
1726
|
-
}
|
|
1727
|
-
|
|
1728
|
-
if (!bounds) {
|
|
1729
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1730
|
-
}
|
|
1731
|
-
|
|
1732
|
-
const runVerification = async (): Promise<{
|
|
1733
|
-
verification: any
|
|
1734
|
-
observedState: { property: string; value: number | null; raw_value?: number | null } | null
|
|
1735
|
-
withinTolerance: boolean
|
|
1736
|
-
}> => {
|
|
1737
|
-
if (inferredSliderState) {
|
|
1738
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
|
|
1739
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
1740
|
-
const refreshed = ToolsInteract._inferLabeledSliderState(elements, currentEl)
|
|
1741
|
-
const observedValue = refreshed?.current ?? null
|
|
1742
|
-
const observedState = observedValue !== null
|
|
1743
|
-
? { property, value: observedValue, raw_value: observedValue }
|
|
1744
|
-
: actualState
|
|
1745
|
-
|
|
1746
|
-
return {
|
|
1747
|
-
verification: {
|
|
1748
|
-
success: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance,
|
|
1749
|
-
observed_state: observedState,
|
|
1750
|
-
reason: observedValue !== null
|
|
1751
|
-
? 'visible labeled value read from nearby slider labels'
|
|
1752
|
-
: 'visible labeled value unavailable'
|
|
1753
|
-
},
|
|
1754
|
-
observedState,
|
|
1755
|
-
withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
|
|
1756
|
-
}
|
|
1757
|
-
}
|
|
1758
|
-
|
|
1759
|
-
const verification = await ToolsInteract.expectStateHandler({
|
|
1760
|
-
element_id: resolvedTarget?.elementId ?? element_id,
|
|
1761
|
-
selector: selector ?? undefined,
|
|
1762
|
-
property,
|
|
1763
|
-
expected: normalizedTargetValue,
|
|
1764
|
-
platform: resolvedPlatform,
|
|
1765
|
-
deviceId: resolvedDeviceId
|
|
1766
|
-
}) as any
|
|
1767
|
-
|
|
1768
|
-
const observedValue = ToolsInteract._parseNumericControlValue(verification?.observed_state?.value) ??
|
|
1769
|
-
ToolsInteract._parseNumericControlValue(verification?.observed_state?.raw_value)
|
|
1770
|
-
const observedState = observedValue !== null
|
|
1771
|
-
? {
|
|
1772
|
-
property,
|
|
1773
|
-
value: observedValue,
|
|
1774
|
-
raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
|
|
1775
|
-
}
|
|
1776
|
-
: actualState
|
|
1777
|
-
|
|
1778
|
-
return {
|
|
1779
|
-
verification,
|
|
1780
|
-
observedState,
|
|
1781
|
-
withinTolerance: observedValue !== null && Math.abs(observedValue - normalizedTargetValue) <= effectiveTolerance
|
|
1782
|
-
}
|
|
1783
|
-
}
|
|
1784
|
-
|
|
1785
|
-
let actionDevice: any = currentDevice
|
|
1786
|
-
let observedState: { property: string; value: number | null; raw_value?: number | null } | null = actualState
|
|
1787
|
-
let verification: any = null
|
|
1788
|
-
let verificationResult: any = { verification: null, observedState: actualState, withinTolerance: false }
|
|
1789
|
-
|
|
1790
|
-
const directActionSupported = ToolsInteract._hasDirectAdjustmentAction(currentEl)
|
|
1791
|
-
if (directActionSupported && attemptCount < attemptsLimit) {
|
|
1792
|
-
const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId)
|
|
1793
|
-
if (typeof interact.setAdjustableValue === 'function') {
|
|
1794
|
-
lastAdjustmentMode = 'semantic'
|
|
1795
|
-
recordTraceStep('execute', 'retry', {
|
|
1796
|
-
attempt: attemptCount + 1,
|
|
1797
|
-
mode: 'semantic',
|
|
1798
|
-
strategy: 'direct_set_value',
|
|
1799
|
-
target_value: normalizedTargetValue
|
|
1800
|
-
})
|
|
1801
|
-
const actionResult = await interact.setAdjustableValue({
|
|
1802
|
-
deviceId: actionResolved.id,
|
|
1803
|
-
target: resolvedTarget,
|
|
1804
|
-
property,
|
|
1805
|
-
value: normalizedTargetValue
|
|
1806
|
-
})
|
|
1807
|
-
attemptCount++
|
|
1808
|
-
actionDevice = actionResult?.device ?? actionDevice
|
|
1809
|
-
|
|
1810
|
-
if (actionResult?.success !== false) {
|
|
1811
|
-
verificationResult = await runVerification()
|
|
1812
|
-
observedState = verificationResult.observedState
|
|
1813
|
-
lastObservedState = observedState
|
|
1814
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1815
|
-
attempt: attemptCount,
|
|
1816
|
-
property,
|
|
1817
|
-
target_value: normalizedTargetValue,
|
|
1818
|
-
actual_state: observedState,
|
|
1819
|
-
reason: verificationResult.verification?.reason ?? 'direct adjustment did not converge yet'
|
|
1820
|
-
})
|
|
1821
|
-
|
|
1822
|
-
if (verificationResult.withinTolerance) {
|
|
1823
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1824
|
-
const base = buildActionExecutionResult({
|
|
1825
|
-
actionType,
|
|
1826
|
-
sourceModule: 'interact',
|
|
1827
|
-
device: actionDevice ?? currentDevice,
|
|
1828
|
-
selector: targetSelector,
|
|
1829
|
-
resolved: resolvedTarget,
|
|
1830
|
-
success: true,
|
|
1831
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1832
|
-
uiFingerprintAfter,
|
|
1833
|
-
details: {
|
|
1834
|
-
target_value: normalizedTargetValue,
|
|
1835
|
-
tolerance: effectiveTolerance,
|
|
1836
|
-
property,
|
|
1837
|
-
attempts: attemptCount,
|
|
1838
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1839
|
-
actual_state: observedState,
|
|
1840
|
-
converged: true,
|
|
1841
|
-
within_tolerance: true,
|
|
1842
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1843
|
-
},
|
|
1844
|
-
traceSteps
|
|
1845
|
-
}) as AdjustControlResponse
|
|
1846
|
-
|
|
1847
|
-
return {
|
|
1848
|
-
...base,
|
|
1849
|
-
target_state: {
|
|
1850
|
-
property,
|
|
1851
|
-
target_value: normalizedTargetValue,
|
|
1852
|
-
tolerance: effectiveTolerance
|
|
1853
|
-
},
|
|
1854
|
-
actual_state: observedState,
|
|
1855
|
-
within_tolerance: true,
|
|
1856
|
-
converged: true,
|
|
1857
|
-
attempts: attemptCount,
|
|
1858
|
-
adjustment_mode: lastAdjustmentMode
|
|
1859
|
-
}
|
|
1860
|
-
}
|
|
1861
|
-
|
|
1862
|
-
cachedResolvedMatch = null
|
|
1863
|
-
if (attemptCount < attemptsLimit) {
|
|
1864
|
-
continue
|
|
1865
|
-
}
|
|
1866
|
-
}
|
|
1867
|
-
}
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
const directions = ToolsInteract._adjustmentDirections(currentEl)
|
|
1871
|
-
if ((directions.increment || directions.decrement) && attemptCount < attemptsLimit) {
|
|
1872
|
-
const { interact, resolved: actionResolved } = await ToolsInteract.getInteractionService(resolvedPlatform, resolvedDeviceId)
|
|
1873
|
-
const adjustByDirection = typeof interact.adjustAccessibleValue === 'function'
|
|
1874
|
-
? interact.adjustAccessibleValue.bind(interact)
|
|
1875
|
-
: null
|
|
1876
|
-
while (adjustByDirection && attemptCount < attemptsLimit) {
|
|
1877
|
-
const direction = normalizedTargetValue > (observedState?.value ?? currentValue) ? 'increment' : 'decrement'
|
|
1878
|
-
if ((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement)) break
|
|
1879
|
-
lastAdjustmentMode = 'semantic'
|
|
1880
|
-
recordTraceStep('execute', 'retry', {
|
|
1881
|
-
attempt: attemptCount + 1,
|
|
1882
|
-
mode: 'semantic',
|
|
1883
|
-
strategy: 'accessibility_increment_decrement',
|
|
1884
|
-
direction
|
|
1885
|
-
})
|
|
1886
|
-
const actionResult = await adjustByDirection({
|
|
1887
|
-
deviceId: actionResolved.id,
|
|
1888
|
-
target: resolvedTarget,
|
|
1889
|
-
property,
|
|
1890
|
-
direction
|
|
1891
|
-
})
|
|
1892
|
-
attemptCount++
|
|
1893
|
-
actionDevice = actionResult?.device ?? actionDevice
|
|
1894
|
-
if (actionResult?.success === false) break
|
|
1895
|
-
|
|
1896
|
-
verificationResult = await runVerification()
|
|
1897
|
-
observedState = verificationResult.observedState
|
|
1898
|
-
lastObservedState = observedState
|
|
1899
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1900
|
-
attempt: attemptCount,
|
|
1901
|
-
property,
|
|
1902
|
-
target_value: normalizedTargetValue,
|
|
1903
|
-
actual_state: observedState,
|
|
1904
|
-
reason: verificationResult.verification?.reason ?? 'incremental adjustment did not converge yet'
|
|
1905
|
-
})
|
|
1906
|
-
|
|
1907
|
-
if (verificationResult.withinTolerance) {
|
|
1908
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1909
|
-
const base = buildActionExecutionResult({
|
|
1910
|
-
actionType,
|
|
1911
|
-
sourceModule: 'interact',
|
|
1912
|
-
device: actionDevice ?? currentDevice,
|
|
1913
|
-
selector: targetSelector,
|
|
1914
|
-
resolved: resolvedTarget,
|
|
1915
|
-
success: true,
|
|
1916
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
1917
|
-
uiFingerprintAfter,
|
|
1918
|
-
details: {
|
|
1919
|
-
target_value: normalizedTargetValue,
|
|
1920
|
-
tolerance: effectiveTolerance,
|
|
1921
|
-
property,
|
|
1922
|
-
attempts: attemptCount,
|
|
1923
|
-
adjustment_mode: lastAdjustmentMode,
|
|
1924
|
-
actual_state: observedState,
|
|
1925
|
-
converged: true,
|
|
1926
|
-
within_tolerance: true,
|
|
1927
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1928
|
-
},
|
|
1929
|
-
traceSteps
|
|
1930
|
-
}) as AdjustControlResponse
|
|
1931
|
-
|
|
1932
|
-
return {
|
|
1933
|
-
...base,
|
|
1934
|
-
target_state: {
|
|
1935
|
-
property,
|
|
1936
|
-
target_value: normalizedTargetValue,
|
|
1937
|
-
tolerance: effectiveTolerance
|
|
1938
|
-
},
|
|
1939
|
-
actual_state: observedState,
|
|
1940
|
-
within_tolerance: true,
|
|
1941
|
-
converged: true,
|
|
1942
|
-
attempts: attemptCount,
|
|
1943
|
-
adjustment_mode: lastAdjustmentMode
|
|
1944
|
-
}
|
|
1945
|
-
}
|
|
1946
|
-
|
|
1947
|
-
cachedResolvedMatch = null
|
|
1948
|
-
if (attemptCount < attemptsLimit) {
|
|
1949
|
-
continue adjustmentAttempts
|
|
1950
|
-
}
|
|
1951
|
-
}
|
|
1952
|
-
}
|
|
1953
|
-
|
|
1954
|
-
if (!bounds) {
|
|
1955
|
-
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1956
|
-
}
|
|
1957
|
-
|
|
1958
|
-
if (!valueRange) {
|
|
1959
|
-
return buildFailure('UNKNOWN', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1960
|
-
}
|
|
1961
|
-
|
|
1962
|
-
const min = valueRange.min
|
|
1963
|
-
const max = valueRange.max
|
|
1964
|
-
const axis = ToolsInteract._controlAxis(currentEl, bounds)
|
|
1965
|
-
const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, normalizedTargetValue, currentValue, min, max, axis)
|
|
1966
|
-
const currentPoint = ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
|
|
1967
|
-
if (!coordinateTapsAwaitingFreshGesture) {
|
|
1968
|
-
const probePoints = inferredSliderState
|
|
1969
|
-
? [targetPoint]
|
|
1970
|
-
: ToolsInteract._buildAdjustmentProbePoints(bounds, normalizedTargetValue, currentValue, min, max, axis)
|
|
1971
|
-
|
|
1972
|
-
for (let i = 0; i < probePoints.length && attemptCount < attemptsLimit; i++) {
|
|
1973
|
-
const probePoint = probePoints[i]
|
|
1974
|
-
lastAdjustmentMode = 'coordinate'
|
|
1975
|
-
recordTraceStep('execute', 'retry', {
|
|
1976
|
-
attempt: attemptCount + 1,
|
|
1977
|
-
mode: 'coordinate',
|
|
1978
|
-
point: probePoint
|
|
1979
|
-
})
|
|
1980
|
-
const actionResult = await ToolsInteract.tapHandler({
|
|
1981
|
-
platform: resolvedPlatform,
|
|
1982
|
-
x: probePoint.x,
|
|
1983
|
-
y: probePoint.y,
|
|
1984
|
-
deviceId: resolvedDeviceId
|
|
1985
|
-
})
|
|
1986
|
-
attemptCount++
|
|
1987
|
-
actionDevice = actionResult.device ?? actionDevice
|
|
1988
|
-
|
|
1989
|
-
if (!actionResult.success) {
|
|
1990
|
-
recordTraceStep('execute', 'retry', {
|
|
1991
|
-
attempt: attemptCount,
|
|
1992
|
-
mode: 'coordinate',
|
|
1993
|
-
point: probePoint,
|
|
1994
|
-
success: false
|
|
1995
|
-
})
|
|
1996
|
-
continue
|
|
1997
|
-
}
|
|
1998
|
-
|
|
1999
|
-
verificationResult = await runVerification()
|
|
2000
|
-
observedState = verificationResult.observedState
|
|
2001
|
-
lastObservedState = observedState
|
|
2002
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
2003
|
-
attempt: attemptCount,
|
|
2004
|
-
property,
|
|
2005
|
-
target_value: normalizedTargetValue,
|
|
2006
|
-
actual_state: observedState,
|
|
2007
|
-
reason: verificationResult.verification?.reason ?? 'control did not converge yet'
|
|
2008
|
-
})
|
|
2009
|
-
|
|
2010
|
-
if (verificationResult.withinTolerance) {
|
|
2011
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
2012
|
-
const base = buildActionExecutionResult({
|
|
2013
|
-
actionType,
|
|
2014
|
-
sourceModule: 'interact',
|
|
2015
|
-
device: actionDevice ?? currentDevice,
|
|
2016
|
-
selector: targetSelector,
|
|
2017
|
-
resolved: resolvedTarget,
|
|
2018
|
-
success: true,
|
|
2019
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
2020
|
-
uiFingerprintAfter,
|
|
2021
|
-
details: {
|
|
2022
|
-
target_value: normalizedTargetValue,
|
|
2023
|
-
tolerance: effectiveTolerance,
|
|
2024
|
-
property,
|
|
2025
|
-
attempts: attemptCount,
|
|
2026
|
-
adjustment_mode: lastAdjustmentMode,
|
|
2027
|
-
actual_state: observedState,
|
|
2028
|
-
converged: true,
|
|
2029
|
-
within_tolerance: true,
|
|
2030
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
2031
|
-
},
|
|
2032
|
-
traceSteps
|
|
2033
|
-
}) as AdjustControlResponse
|
|
2034
|
-
|
|
2035
|
-
return {
|
|
2036
|
-
...base,
|
|
2037
|
-
target_state: {
|
|
2038
|
-
property,
|
|
2039
|
-
target_value: normalizedTargetValue,
|
|
2040
|
-
tolerance: effectiveTolerance
|
|
2041
|
-
},
|
|
2042
|
-
actual_state: observedState,
|
|
2043
|
-
within_tolerance: true,
|
|
2044
|
-
converged: true,
|
|
2045
|
-
attempts: attemptCount,
|
|
2046
|
-
adjustment_mode: lastAdjustmentMode
|
|
2047
|
-
}
|
|
2048
|
-
}
|
|
2049
|
-
|
|
2050
|
-
coordinateTapsAwaitingFreshGesture = true
|
|
2051
|
-
cachedResolvedMatch = null
|
|
2052
|
-
if (attemptCount < attemptsLimit) {
|
|
2053
|
-
continue adjustmentAttempts
|
|
2054
|
-
}
|
|
2055
|
-
}
|
|
2056
|
-
}
|
|
2057
|
-
|
|
2058
|
-
if (attemptCount < attemptsLimit) {
|
|
2059
|
-
coordinateTapsAwaitingFreshGesture = false
|
|
2060
|
-
lastAdjustmentMode = 'gesture'
|
|
2061
|
-
recordTraceStep('execute', 'retry', {
|
|
2062
|
-
attempt: attemptCount + 1,
|
|
2063
|
-
mode: 'gesture',
|
|
2064
|
-
start: currentPoint,
|
|
2065
|
-
end: targetPoint
|
|
2066
|
-
})
|
|
2067
|
-
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
2068
|
-
platform: resolvedPlatform,
|
|
2069
|
-
x1: currentPoint.x,
|
|
2070
|
-
y1: currentPoint.y,
|
|
2071
|
-
x2: targetPoint.x,
|
|
2072
|
-
y2: targetPoint.y,
|
|
2073
|
-
duration: inferredSliderState ? 520 : 220,
|
|
2074
|
-
deviceId: resolvedDeviceId
|
|
2075
|
-
})
|
|
2076
|
-
attemptCount++
|
|
2077
|
-
if (!fallbackActionResult.success) {
|
|
2078
|
-
recordTraceStep('execute', 'failure', {
|
|
2079
|
-
attempt: attemptCount,
|
|
2080
|
-
mode: 'gesture',
|
|
2081
|
-
start: currentPoint,
|
|
2082
|
-
end: targetPoint,
|
|
2083
|
-
success: false
|
|
2084
|
-
})
|
|
2085
|
-
return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
|
|
2086
|
-
}
|
|
2087
|
-
|
|
2088
|
-
actionDevice = fallbackActionResult.device ?? actionDevice
|
|
2089
|
-
verificationResult = await runVerification()
|
|
2090
|
-
observedState = verificationResult.observedState
|
|
2091
|
-
lastObservedState = observedState
|
|
2092
|
-
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
2093
|
-
attempt: attemptCount,
|
|
2094
|
-
property,
|
|
2095
|
-
target_value: normalizedTargetValue,
|
|
2096
|
-
actual_state: observedState,
|
|
2097
|
-
reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
|
|
2098
|
-
})
|
|
2099
|
-
|
|
2100
|
-
if (verificationResult.withinTolerance) {
|
|
2101
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
2102
|
-
const base = buildActionExecutionResult({
|
|
2103
|
-
actionType,
|
|
2104
|
-
sourceModule: 'interact',
|
|
2105
|
-
device: actionDevice ?? currentDevice,
|
|
2106
|
-
selector: targetSelector,
|
|
2107
|
-
resolved: resolvedTarget,
|
|
2108
|
-
success: true,
|
|
2109
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
2110
|
-
uiFingerprintAfter,
|
|
2111
|
-
details: {
|
|
2112
|
-
target_value: normalizedTargetValue,
|
|
2113
|
-
tolerance: effectiveTolerance,
|
|
2114
|
-
property,
|
|
2115
|
-
attempts: attemptCount,
|
|
2116
|
-
adjustment_mode: lastAdjustmentMode,
|
|
2117
|
-
actual_state: observedState,
|
|
2118
|
-
converged: true,
|
|
2119
|
-
within_tolerance: true,
|
|
2120
|
-
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
2121
|
-
},
|
|
2122
|
-
traceSteps
|
|
2123
|
-
}) as AdjustControlResponse
|
|
2124
|
-
|
|
2125
|
-
return {
|
|
2126
|
-
...base,
|
|
2127
|
-
target_state: {
|
|
2128
|
-
property,
|
|
2129
|
-
target_value: normalizedTargetValue,
|
|
2130
|
-
tolerance: effectiveTolerance
|
|
2131
|
-
},
|
|
2132
|
-
actual_state: observedState,
|
|
2133
|
-
within_tolerance: true,
|
|
2134
|
-
converged: true,
|
|
2135
|
-
attempts: attemptCount,
|
|
2136
|
-
adjustment_mode: lastAdjustmentMode
|
|
2137
|
-
}
|
|
2138
|
-
}
|
|
2139
|
-
}
|
|
2140
|
-
|
|
2141
|
-
verification = verificationResult.verification
|
|
2142
|
-
lastObservedState = observedState
|
|
2143
|
-
|
|
2144
|
-
if (verificationResult.withinTolerance) {
|
|
2145
|
-
recordTraceStep('verify', 'success', {
|
|
2146
|
-
attempt: attemptCount,
|
|
2147
|
-
property,
|
|
2148
|
-
target_value: normalizedTargetValue,
|
|
2149
|
-
actual_state: observedState,
|
|
2150
|
-
reason: verification?.reason ?? 'control converged to target value'
|
|
2151
|
-
})
|
|
2152
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
2153
|
-
const base = buildActionExecutionResult({
|
|
2154
|
-
actionType,
|
|
2155
|
-
sourceModule: 'interact',
|
|
2156
|
-
device: actionDevice ?? currentDevice,
|
|
2157
|
-
selector: targetSelector,
|
|
2158
|
-
resolved: resolvedTarget,
|
|
2159
|
-
success: true,
|
|
2160
|
-
uiFingerprintBefore: fingerprintBefore,
|
|
2161
|
-
uiFingerprintAfter,
|
|
2162
|
-
details: {
|
|
2163
|
-
target_value: normalizedTargetValue,
|
|
2164
|
-
tolerance: effectiveTolerance,
|
|
2165
|
-
property,
|
|
2166
|
-
attempts: attemptCount,
|
|
2167
|
-
adjustment_mode: lastAdjustmentMode,
|
|
2168
|
-
actual_state: observedState,
|
|
2169
|
-
converged: true,
|
|
2170
|
-
within_tolerance: true,
|
|
2171
|
-
reason: verification?.reason ?? 'control converged to target value'
|
|
2172
|
-
},
|
|
2173
|
-
traceSteps
|
|
2174
|
-
}) as AdjustControlResponse
|
|
2175
|
-
|
|
2176
|
-
return {
|
|
2177
|
-
...base,
|
|
2178
|
-
target_state: {
|
|
2179
|
-
property,
|
|
2180
|
-
target_value: normalizedTargetValue,
|
|
2181
|
-
tolerance: effectiveTolerance
|
|
2182
|
-
},
|
|
2183
|
-
actual_state: observedState,
|
|
2184
|
-
within_tolerance: true,
|
|
2185
|
-
converged: true,
|
|
2186
|
-
attempts: attemptCount,
|
|
2187
|
-
adjustment_mode: lastAdjustmentMode
|
|
2188
|
-
}
|
|
2189
|
-
}
|
|
2190
|
-
|
|
2191
|
-
cachedResolvedMatch = inferredSliderState
|
|
2192
|
-
? null
|
|
2193
|
-
: {
|
|
2194
|
-
el: {
|
|
2195
|
-
...currentEl,
|
|
2196
|
-
state: {
|
|
2197
|
-
...(currentEl.state ?? null),
|
|
2198
|
-
...(observedState ? {
|
|
2199
|
-
[observedState.property]: observedState.value,
|
|
2200
|
-
raw_value: observedState.raw_value ?? observedState.value
|
|
2201
|
-
} : {})
|
|
2202
|
-
}
|
|
2203
|
-
},
|
|
2204
|
-
idx: resolved.match.idx
|
|
2205
|
-
}
|
|
2206
|
-
}
|
|
2207
|
-
|
|
2208
|
-
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
2209
|
-
return buildFailure('CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter)
|
|
198
|
+
static async adjustControlHandler(input: AdjustmentInput): Promise<AdjustControlResponse> {
|
|
199
|
+
return adjustControl(input, {
|
|
200
|
+
observe: (platform, deviceId) => ToolsObserve.getUITreeHandler({ platform, deviceId }),
|
|
201
|
+
fingerprint: (platform, deviceId) => ToolsInteract._captureFingerprint(platform, deviceId),
|
|
202
|
+
getHandle: id => ToolsInteract._resolvedUiElements.get(id),
|
|
203
|
+
currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
|
|
204
|
+
target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx),
|
|
205
|
+
wait: input => ToolsInteract.waitForUIHandler(input), find: input => ToolsInteract.findElementHandler(input),
|
|
206
|
+
expectState: input => ToolsInteract.expectStateHandler(input), service: (platform, deviceId) => ToolsInteract.getInteractionService(platform, deviceId),
|
|
207
|
+
tap: input => ToolsInteract.tapHandler(input), swipe: input => ToolsInteract.swipeHandler(input)
|
|
208
|
+
})
|
|
2210
209
|
}
|
|
2211
210
|
|
|
2212
211
|
static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId, timeoutMs }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string, timeoutMs?: number }) {
|
|
@@ -2229,1454 +228,40 @@ export class ToolsInteract {
|
|
|
2229
228
|
return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id)
|
|
2230
229
|
}
|
|
2231
230
|
|
|
2232
|
-
static async findElementHandler(
|
|
2233
|
-
|
|
2234
|
-
const start = Date.now()
|
|
2235
|
-
const deadline = start + timeoutMs
|
|
2236
|
-
const normalize = ToolsInteract._normalize
|
|
2237
|
-
|
|
2238
|
-
const q = normalize(query)
|
|
2239
|
-
if (!q) return { found: false, error: 'Empty query' }
|
|
2240
|
-
|
|
2241
|
-
let best: RankedResolutionCandidate | null = null
|
|
2242
|
-
let bestTree: any = null
|
|
2243
|
-
let bestIterationCandidates: RankedResolutionCandidate[] = []
|
|
2244
|
-
let shouldStop = false
|
|
2245
|
-
|
|
2246
|
-
const scoreElement = (el: UiElement | null, idx: number): RankedResolutionCandidate | null => {
|
|
2247
|
-
if (!el || !el.visible) return null
|
|
2248
|
-
const bounds = el.bounds || [0,0,0,0]
|
|
2249
|
-
if (!Array.isArray(bounds) || bounds.length < 4) return null
|
|
2250
|
-
const [l,t,r,b] = bounds
|
|
2251
|
-
if (r <= l || b <= t) return null
|
|
2252
|
-
// Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
|
|
2253
|
-
const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el))
|
|
2254
|
-
|
|
2255
|
-
const text = normalize(el.text ?? el.label ?? el.value ?? '')
|
|
2256
|
-
const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '')
|
|
2257
|
-
const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
|
|
2258
|
-
const className = normalize(el.type ?? el.class ?? '')
|
|
2259
|
-
const semanticRole = normalize(el.semantic?.semantic_role ?? '')
|
|
2260
|
-
const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : []
|
|
2261
|
-
|
|
2262
|
-
let score = 0
|
|
2263
|
-
let reason = 'best_scoring_candidate'
|
|
2264
|
-
if (exact) {
|
|
2265
|
-
if (text && text === q) {
|
|
2266
|
-
score = 1.0
|
|
2267
|
-
reason = 'exact_text_match'
|
|
2268
|
-
} else if (content && content === q) {
|
|
2269
|
-
score = 0.95
|
|
2270
|
-
reason = 'exact_content_desc_match'
|
|
2271
|
-
} else if (resourceId && resourceId === q) {
|
|
2272
|
-
score = 0.92
|
|
2273
|
-
reason = 'exact_resource_id_match'
|
|
2274
|
-
} else if (className && className === q) {
|
|
2275
|
-
score = 0.3
|
|
2276
|
-
reason = 'exact_class_match'
|
|
2277
|
-
}
|
|
2278
|
-
} else {
|
|
2279
|
-
if (text && text === q) {
|
|
2280
|
-
score = 1.0
|
|
2281
|
-
reason = 'exact_text_match'
|
|
2282
|
-
} else if (content && content === q) {
|
|
2283
|
-
score = 0.95
|
|
2284
|
-
reason = 'exact_content_desc_match'
|
|
2285
|
-
} else if (resourceId && resourceId === q) {
|
|
2286
|
-
score = 0.92
|
|
2287
|
-
reason = 'exact_resource_id_match'
|
|
2288
|
-
} else if (text && text.includes(q)) {
|
|
2289
|
-
score = 0.6
|
|
2290
|
-
reason = 'partial_text_match'
|
|
2291
|
-
} else if (content && content.includes(q)) {
|
|
2292
|
-
score = 0.55
|
|
2293
|
-
reason = 'partial_content_desc_match'
|
|
2294
|
-
} else if (resourceId && resourceId.includes(q)) {
|
|
2295
|
-
score = 0.7
|
|
2296
|
-
reason = 'partial_resource_id_match'
|
|
2297
|
-
} else if (className && className.includes(q)) {
|
|
2298
|
-
score = 0.3
|
|
2299
|
-
reason = 'partial_class_match'
|
|
2300
|
-
}
|
|
2301
|
-
}
|
|
2302
|
-
if (!exact) {
|
|
2303
|
-
if (!score && semanticRole && semanticRole.includes(q)) {
|
|
2304
|
-
score = 0.5
|
|
2305
|
-
reason = 'semantic_role_match'
|
|
2306
|
-
}
|
|
2307
|
-
if (semanticActions.some((action) => action.includes(q))) {
|
|
2308
|
-
score = Math.max(score, score > 0 ? 0.65 : 0.6)
|
|
2309
|
-
reason = 'semantic_action_match'
|
|
2310
|
-
}
|
|
2311
|
-
if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
|
|
2312
|
-
score = 0.45
|
|
2313
|
-
reason = 'semantic_control_match'
|
|
2314
|
-
}
|
|
2315
|
-
} else {
|
|
2316
|
-
if (!score && semanticRole && semanticRole === q) {
|
|
2317
|
-
score = 0.5
|
|
2318
|
-
reason = 'semantic_role_match'
|
|
2319
|
-
}
|
|
2320
|
-
if (semanticActions.some((action) => action === q)) {
|
|
2321
|
-
score = Math.max(score, score > 0 ? 0.65 : 0.6)
|
|
2322
|
-
reason = 'semantic_action_match'
|
|
2323
|
-
}
|
|
2324
|
-
}
|
|
2325
|
-
if (score > 0 && interactable) score += 0.05
|
|
2326
|
-
if (score <= 0) return null
|
|
2327
|
-
return { el, idx, score, reason, interactable }
|
|
2328
|
-
}
|
|
2329
|
-
|
|
2330
|
-
while (Date.now() <= deadline) {
|
|
2331
|
-
try {
|
|
2332
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
|
|
2333
|
-
if (tree && Array.isArray((tree as any).elements)) {
|
|
2334
|
-
const elements = ((tree as any).elements as UiElement[])
|
|
2335
|
-
const iterationCandidates: RankedResolutionCandidate[] = []
|
|
2336
|
-
let iterationImprovedBest = false
|
|
2337
|
-
for (let i = 0; i < elements.length; i++) {
|
|
2338
|
-
const el = elements[i]
|
|
2339
|
-
try {
|
|
2340
|
-
const candidate = scoreElement(el, i)
|
|
2341
|
-
if (!candidate) continue
|
|
2342
|
-
iterationCandidates.push(candidate)
|
|
2343
|
-
if (!best || candidate.score > best.score) {
|
|
2344
|
-
best = candidate
|
|
2345
|
-
bestTree = tree
|
|
2346
|
-
iterationImprovedBest = true
|
|
2347
|
-
if (best.score >= 0.95) {
|
|
2348
|
-
shouldStop = true
|
|
2349
|
-
break
|
|
2350
|
-
}
|
|
2351
|
-
}
|
|
2352
|
-
} catch (e) { console.error('Error scoring element:', e) }
|
|
2353
|
-
}
|
|
2354
|
-
if (iterationImprovedBest) {
|
|
2355
|
-
bestIterationCandidates = iterationCandidates.slice()
|
|
2356
|
-
}
|
|
2357
|
-
}
|
|
2358
|
-
} catch (e) { console.error('Error fetching UI tree:', e) }
|
|
2359
|
-
if (shouldStop || Date.now() > deadline) break
|
|
2360
|
-
await new Promise(r => setTimeout(r, 100))
|
|
2361
|
-
}
|
|
2362
|
-
|
|
2363
|
-
if (!best) return { found: false, error: 'Element not found' }
|
|
2364
|
-
|
|
2365
|
-
// If the best match is not interactable, try to resolve an actionable ancestor.
|
|
2366
|
-
try {
|
|
2367
|
-
const elements = (bestTree && Array.isArray(bestTree.elements)) ? (bestTree.elements as UiElement[]) : []
|
|
2368
|
-
const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution as UiResolution : null
|
|
2369
|
-
let chosen = best as { el: UiElement, idx: number }
|
|
2370
|
-
const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null
|
|
2371
|
-
|
|
2372
|
-
// Strategy 1: if parentId references an index, climb that chain
|
|
2373
|
-
let resolvedAncestor: { el: UiElement, idx: number } | null = null
|
|
2374
|
-
if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
|
|
2375
|
-
let cur = chosen
|
|
2376
|
-
let safety = 0
|
|
2377
|
-
while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
|
|
2378
|
-
let pid = cur.el.parentId
|
|
2379
|
-
let idx: number | null = null
|
|
2380
|
-
if (typeof pid === 'number') idx = pid
|
|
2381
|
-
else if (typeof pid === 'string' && /^\d+$/.test(pid)) idx = Number(pid)
|
|
2382
|
-
// If parentId is not an index, try to find by matching resourceId or id field
|
|
2383
|
-
if (idx !== null && elements[idx]) {
|
|
2384
|
-
cur = { el: elements[idx], idx }
|
|
2385
|
-
if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
|
|
2386
|
-
} else if (typeof pid === 'string') {
|
|
2387
|
-
// fallback: search elements for matching resourceId or id
|
|
2388
|
-
const foundIndex = elements.findIndex((el: UiElement)=> (el.resourceId === pid || el.id === pid))
|
|
2389
|
-
const found = foundIndex >= 0 ? elements[foundIndex] : null
|
|
2390
|
-
if (found) {
|
|
2391
|
-
cur = { el: found, idx: foundIndex }
|
|
2392
|
-
if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
|
|
2393
|
-
// otherwise continue climbing if this found element has its own parentId
|
|
2394
|
-
} else {
|
|
2395
|
-
break
|
|
2396
|
-
}
|
|
2397
|
-
} else {
|
|
2398
|
-
break
|
|
2399
|
-
}
|
|
2400
|
-
safety++
|
|
2401
|
-
}
|
|
2402
|
-
}
|
|
2403
|
-
|
|
2404
|
-
// Strategy 2: fallback - find a clickable element whose bounds fully contain the child's bounds
|
|
2405
|
-
if (!resolvedAncestor && childBounds) {
|
|
2406
|
-
const [cl,ct,cr,cb] = childBounds
|
|
2407
|
-
// find candidates that are clickable and contain the child bounds
|
|
2408
|
-
const candidates = elements
|
|
2409
|
-
.map((el: UiElement, idx: number) => ({ el, idx }))
|
|
2410
|
-
.filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds!.length >= 4)
|
|
2411
|
-
let bestCandidate: { el: UiElement, idx: number } | null = null
|
|
2412
|
-
let bestCandidateArea = Infinity
|
|
2413
|
-
for (const c of candidates) {
|
|
2414
|
-
const bounds = c.el.bounds as number[]
|
|
2415
|
-
const [pl,pt,pr,pb] = bounds
|
|
2416
|
-
if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
|
|
2417
|
-
const area = (pr-pl) * (pb-pt)
|
|
2418
|
-
if (area < bestCandidateArea) { bestCandidateArea = area; bestCandidate = c }
|
|
2419
|
-
}
|
|
2420
|
-
}
|
|
2421
|
-
if (bestCandidate) resolvedAncestor = bestCandidate
|
|
2422
|
-
}
|
|
2423
|
-
|
|
2424
|
-
if (resolvedAncestor) {
|
|
2425
|
-
best = {
|
|
2426
|
-
el: resolvedAncestor.el,
|
|
2427
|
-
idx: resolvedAncestor.idx,
|
|
2428
|
-
score: Math.min(1, best.score + 0.02),
|
|
2429
|
-
reason: 'clickable_parent_preferred',
|
|
2430
|
-
interactable: true
|
|
2431
|
-
}
|
|
2432
|
-
}
|
|
2433
|
-
|
|
2434
|
-
if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
|
|
2435
|
-
const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen)
|
|
2436
|
-
if (nearbyActionable) {
|
|
2437
|
-
best = {
|
|
2438
|
-
el: nearbyActionable.el,
|
|
2439
|
-
idx: nearbyActionable.idx,
|
|
2440
|
-
score: Math.min(1, best.score + 0.02),
|
|
2441
|
-
reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
|
|
2442
|
-
interactable: true
|
|
2443
|
-
}
|
|
2444
|
-
}
|
|
2445
|
-
}
|
|
2446
|
-
} catch (e) { console.error('Error resolving ancestor:', e) }
|
|
2447
|
-
|
|
2448
|
-
if (!best) return { found: false, error: 'Element not found' }
|
|
2449
|
-
|
|
2450
|
-
const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null
|
|
2451
|
-
const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null
|
|
2452
|
-
const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el))
|
|
2453
|
-
const alternateCandidates = uniqueRanked
|
|
2454
|
-
.filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
|
|
2455
|
-
.slice(0, 3)
|
|
2456
|
-
.map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate))
|
|
2457
|
-
|
|
2458
|
-
const outEl = {
|
|
2459
|
-
text: best.el.text ?? null,
|
|
2460
|
-
resourceId: best.el.resourceId ?? null,
|
|
2461
|
-
contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
|
|
2462
|
-
class: best.el.type ?? best.el.class ?? null,
|
|
2463
|
-
bounds: boundsObj,
|
|
2464
|
-
clickable: !!best.el.clickable,
|
|
2465
|
-
enabled: !!best.el.enabled,
|
|
2466
|
-
stable_id: best.el.stable_id ?? null,
|
|
2467
|
-
role: best.el.role ?? null,
|
|
2468
|
-
test_tag: best.el.test_tag ?? null,
|
|
2469
|
-
selector: best.el.selector ?? null,
|
|
2470
|
-
semantic: best.el.semantic ?? null,
|
|
2471
|
-
tapCoordinates,
|
|
2472
|
-
telemetry: {
|
|
2473
|
-
matchedIndex: best.idx ?? null,
|
|
2474
|
-
matchedInteractable: !!best.interactable,
|
|
2475
|
-
sliderLike: best.reason === 'slider_track_preferred'
|
|
2476
|
-
}
|
|
2477
|
-
}
|
|
2478
|
-
if (best.reason === 'slider_track_preferred') {
|
|
2479
|
-
const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left)
|
|
2480
|
-
const interactionHint = {
|
|
2481
|
-
kind: 'slider',
|
|
2482
|
-
axis: isVertical ? 'vertical' : 'horizontal',
|
|
2483
|
-
trackBounds: boundsObj
|
|
2484
|
-
}
|
|
2485
|
-
;(outEl as any).interactionHint = interactionHint
|
|
2486
|
-
}
|
|
2487
|
-
const scoreVal = Math.min(1, Number(best.score.toFixed(3)))
|
|
2488
|
-
const resolution: FindElementResolutionSummary = {
|
|
2489
|
-
confidence: scoreVal,
|
|
2490
|
-
reason: best.reason,
|
|
2491
|
-
fallback_available: alternateCandidates.length > 0,
|
|
2492
|
-
matched_count: uniqueRanked.length,
|
|
2493
|
-
alternates: alternateCandidates
|
|
2494
|
-
}
|
|
2495
|
-
return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution }
|
|
231
|
+
static async findElementHandler(input: FindInput): Promise<FindElementResponse> {
|
|
232
|
+
return findElement(input, target => ToolsObserve.getUITreeHandler(target))
|
|
2496
233
|
}
|
|
2497
234
|
|
|
2498
|
-
static async waitForUIHandler(
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
const requestedIndex = typeof match?.index === 'number' ? match.index : null
|
|
2502
|
-
const requested = {
|
|
2503
|
-
selector: selector ?? {},
|
|
2504
|
-
condition,
|
|
2505
|
-
match: requestedIndex === null ? null : { index: requestedIndex }
|
|
2506
|
-
}
|
|
2507
|
-
|
|
2508
|
-
// Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
|
|
2509
|
-
const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
|
|
2510
|
-
const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
|
|
2511
|
-
const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
|
|
2512
|
-
|
|
2513
|
-
if (!hasText && !hasResId && !hasAccId) {
|
|
2514
|
-
return {
|
|
2515
|
-
status: 'timeout',
|
|
2516
|
-
error: {
|
|
2517
|
-
code: 'INVALID_SELECTOR',
|
|
2518
|
-
message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
|
|
2519
|
-
},
|
|
2520
|
-
metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
|
|
2521
|
-
requested,
|
|
2522
|
-
observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
|
|
2523
|
-
};
|
|
2524
|
-
}
|
|
2525
|
-
|
|
2526
|
-
// Validate condition
|
|
2527
|
-
if (!['exists','not_exists','visible','clickable'].includes(condition)) {
|
|
2528
|
-
return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
|
|
2529
|
-
}
|
|
2530
|
-
|
|
2531
|
-
// Platform check
|
|
2532
|
-
if (platform && !['android','ios'].includes(platform)) {
|
|
2533
|
-
return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
|
|
2534
|
-
}
|
|
2535
|
-
|
|
2536
|
-
const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000))
|
|
2537
|
-
const maxAttempts = (retry && retry.max_attempts) ? Math.max(1, retry.max_attempts) : 1
|
|
2538
|
-
const backoff = (retry && retry.backoff_ms) ? Math.max(0, retry.backoff_ms) : 0
|
|
2539
|
-
|
|
2540
|
-
let attempts = 0
|
|
2541
|
-
let totalPollCount = 0
|
|
2542
|
-
let lastMatchedCount = 0
|
|
2543
|
-
let lastMatchedElement: ActionTargetResolved | null = null
|
|
2544
|
-
let lastConditionSatisfied = false
|
|
2545
|
-
let matchedAt: number | null = null
|
|
2546
|
-
let stableMatchCount = 0
|
|
2547
|
-
const stableObservationCount = singleObservation ? 1 : 2
|
|
2548
|
-
const snapshotStaleThresholdMs = 500
|
|
2549
|
-
|
|
2550
|
-
// Precompute normalized selector values and helpers (constant across polls)
|
|
2551
|
-
const normalize = ToolsInteract._normalize
|
|
2552
|
-
const containsFlag = !!selector?.contains
|
|
2553
|
-
const selText = normalize(selector?.text)
|
|
2554
|
-
const selRid = normalize(selector?.resource_id)
|
|
2555
|
-
const selAid = normalize(selector?.accessibility_id)
|
|
2556
|
-
|
|
2557
|
-
try {
|
|
2558
|
-
while (attempts < maxAttempts) {
|
|
2559
|
-
attempts++
|
|
2560
|
-
const attemptStart = Date.now()
|
|
2561
|
-
const deadline = attemptStart + (timeout_ms || 0)
|
|
2562
|
-
|
|
2563
|
-
while (Date.now() <= deadline || (singleAttempt && totalPollCount === 0)) {
|
|
2564
|
-
remainingBudget()
|
|
2565
|
-
totalPollCount++
|
|
2566
|
-
try {
|
|
2567
|
-
const acquisitionStarted = Date.now()
|
|
2568
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
2569
|
-
observeReadinessWait(tree, acquisitionStarted)
|
|
2570
|
-
// A failed acquisition cannot establish that the requested tap target is absent.
|
|
2571
|
-
if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error))) throw new Error('ACTION_TIMEOUT')
|
|
2572
|
-
const elements = (tree && Array.isArray(tree.elements)) ? tree.elements as any[] : []
|
|
2573
|
-
|
|
2574
|
-
const matches: { el: any, idx: number }[] = []
|
|
2575
|
-
|
|
2576
|
-
for (let i = 0; i < elements.length; i++) {
|
|
2577
|
-
const el = elements[i]
|
|
2578
|
-
let ok = true
|
|
2579
|
-
|
|
2580
|
-
// text
|
|
2581
|
-
if (selector.text !== undefined && selector.text !== null) {
|
|
2582
|
-
const val = normalize(el.text || el.label || el.value || '')
|
|
2583
|
-
if (containsFlag) {
|
|
2584
|
-
if (!val.includes(selText)) ok = false
|
|
2585
|
-
} else {
|
|
2586
|
-
if (val !== selText) ok = false
|
|
2587
|
-
}
|
|
2588
|
-
}
|
|
2589
|
-
|
|
2590
|
-
// resource_id
|
|
2591
|
-
if (ok && selector.resource_id !== undefined && selector.resource_id !== null) {
|
|
2592
|
-
const rid = normalize(el.resourceId || el.resourceID || el.id || '')
|
|
2593
|
-
if (containsFlag) {
|
|
2594
|
-
if (!rid.includes(selRid)) ok = false
|
|
2595
|
-
} else {
|
|
2596
|
-
if (rid !== selRid) ok = false
|
|
2597
|
-
}
|
|
2598
|
-
}
|
|
2599
|
-
|
|
2600
|
-
// accessibility_id
|
|
2601
|
-
if (ok && selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
|
|
2602
|
-
const aid = normalize(el.contentDescription || el.contentDesc || el.accessibilityLabel || el.label || '')
|
|
2603
|
-
if (containsFlag) {
|
|
2604
|
-
if (!aid.includes(selAid)) ok = false
|
|
2605
|
-
} else {
|
|
2606
|
-
if (aid !== selAid) ok = false
|
|
2607
|
-
}
|
|
2608
|
-
}
|
|
2609
|
-
|
|
2610
|
-
if (ok) matches.push({ el, idx: i })
|
|
2611
|
-
}
|
|
2612
|
-
|
|
2613
|
-
// Evaluate condition
|
|
2614
|
-
const matchedCount = matches.length
|
|
2615
|
-
if (rejectAmbiguous && matchedCount > 1 && match?.index === undefined) return { status: 'timeout', error: { code: 'AMBIGUOUS_TARGET' } }
|
|
2616
|
-
const pickIndex = (typeof match?.index === 'number') ? match!.index as number : undefined
|
|
2617
|
-
let chosen: { el: any, idx: number } | null = null
|
|
2618
|
-
if (matches.length > 0) {
|
|
2619
|
-
if (pickIndex !== undefined) {
|
|
2620
|
-
// If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
|
|
2621
|
-
if (pickIndex >= 0 && pickIndex < matches.length) chosen = matches[pickIndex]
|
|
2622
|
-
else chosen = null
|
|
2623
|
-
} else {
|
|
2624
|
-
chosen = matches[0]
|
|
2625
|
-
}
|
|
2626
|
-
} else {
|
|
2627
|
-
chosen = null
|
|
2628
|
-
}
|
|
2629
|
-
|
|
2630
|
-
let conditionMet = false
|
|
2631
|
-
let matchedElement = chosen
|
|
2632
|
-
if (condition === 'exists') {
|
|
2633
|
-
// when an index is specified, existence requires that specific index be present
|
|
2634
|
-
conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1)
|
|
2635
|
-
} else if (condition === 'not_exists') {
|
|
2636
|
-
// when an index is specified, not_exists is true if that index is absent
|
|
2637
|
-
conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0)
|
|
2638
|
-
} else if (condition === 'visible') {
|
|
2639
|
-
if (chosen) {
|
|
2640
|
-
const b = chosen.el.bounds
|
|
2641
|
-
const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
|
|
2642
|
-
conditionMet = visibleFlag
|
|
2643
|
-
} else conditionMet = false
|
|
2644
|
-
} else if (condition === 'clickable') {
|
|
2645
|
-
matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
|
|
2646
|
-
? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen as { el: UiElement, idx: number }) || chosen) : null
|
|
2647
|
-
if (matchedElement) {
|
|
2648
|
-
const b = matchedElement.el.bounds
|
|
2649
|
-
const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
|
|
2650
|
-
const enabled = !!matchedElement.el.enabled
|
|
2651
|
-
const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable
|
|
2652
|
-
conditionMet = visibleFlag && enabled && clickable
|
|
2653
|
-
} else conditionMet = false
|
|
2654
|
-
}
|
|
2655
|
-
|
|
2656
|
-
// Waiting for a label to exist/be visible is distinct from resolving its
|
|
2657
|
-
// executable control. Reuse the clickable-wait ancestor policy without
|
|
2658
|
-
// promoting hidden or disabled labels into an unrelated enabled target.
|
|
2659
|
-
if (resolveTapTarget && conditionMet && matchedElement &&
|
|
2660
|
-
ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
|
|
2661
|
-
matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement
|
|
2662
|
-
}
|
|
2663
|
-
const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
|
|
2664
|
-
const resolvedDeviceId = tree?.device?.id || deviceId
|
|
2665
|
-
lastMatchedCount = matchedCount
|
|
2666
|
-
lastConditionSatisfied = conditionMet
|
|
2667
|
-
lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
|
|
2668
|
-
const now = Date.now()
|
|
2669
|
-
|
|
2670
|
-
const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null
|
|
2671
|
-
const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs
|
|
2672
|
-
|
|
2673
|
-
if (conditionMet && snapshotFresh) {
|
|
2674
|
-
if (matchedAt === null) matchedAt = now
|
|
2675
|
-
stableMatchCount++
|
|
2676
|
-
if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
|
|
2677
|
-
const latency_ms = now - overallStart
|
|
2678
|
-
const outEl = lastMatchedElement
|
|
2679
|
-
|
|
2680
|
-
return {
|
|
2681
|
-
status: 'success',
|
|
2682
|
-
...(singleObservation ? { _tree: tree } : {}),
|
|
2683
|
-
matched: matchedCount,
|
|
2684
|
-
element: outEl,
|
|
2685
|
-
metrics: { latency_ms, poll_count: totalPollCount, attempts },
|
|
2686
|
-
requested,
|
|
2687
|
-
observed: {
|
|
2688
|
-
matched_count: matchedCount,
|
|
2689
|
-
condition_satisfied: true,
|
|
2690
|
-
selected_index: outEl?.index ?? null,
|
|
2691
|
-
last_matched_element: outEl
|
|
2692
|
-
}
|
|
2693
|
-
}
|
|
2694
|
-
}
|
|
2695
|
-
} else {
|
|
2696
|
-
stableMatchCount = 0
|
|
2697
|
-
matchedAt = null
|
|
2698
|
-
}
|
|
2699
|
-
|
|
2700
|
-
} catch (e) {
|
|
2701
|
-
// Deadline expiry does not invalidate the last successfully observed state.
|
|
2702
|
-
if (e instanceof Error && e.message === 'ACTION_TIMEOUT') throw e
|
|
2703
|
-
if (readinessTracker && !String(e).includes('READINESS_')) readinessTracker.unavailable()
|
|
2704
|
-
if (readinessTracker && isDeviceUnavailable(e)) throw e
|
|
2705
|
-
// Non-fatal per-poll error; record and continue
|
|
2706
|
-
if (!readinessTracker) console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e))
|
|
2707
|
-
}
|
|
2708
|
-
|
|
2709
|
-
if (singleAttempt) break
|
|
2710
|
-
// Sleep until next poll
|
|
2711
|
-
await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)))
|
|
2712
|
-
}
|
|
2713
|
-
|
|
2714
|
-
// Attempt timed out; if more attempts allowed, backoff then retry
|
|
2715
|
-
if (attempts < maxAttempts) {
|
|
2716
|
-
if (backoff > 0) await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)))
|
|
2717
|
-
continue
|
|
2718
|
-
}
|
|
2719
|
-
|
|
2720
|
-
// Final failure for this call
|
|
2721
|
-
const elapsed = Date.now() - overallStart
|
|
2722
|
-
const observed = {
|
|
2723
|
-
matched_count: lastMatchedCount,
|
|
2724
|
-
condition_satisfied: lastConditionSatisfied,
|
|
2725
|
-
selected_index: lastMatchedElement?.index ?? null,
|
|
2726
|
-
last_matched_element: lastMatchedElement
|
|
2727
|
-
}
|
|
2728
|
-
const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
|
|
2729
|
-
? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
|
|
2730
|
-
: ` observed ${lastMatchedCount} match(es)`
|
|
2731
|
-
return {
|
|
2732
|
-
status: 'timeout',
|
|
2733
|
-
error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
|
|
2734
|
-
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
2735
|
-
requested,
|
|
2736
|
-
observed
|
|
2737
|
-
}
|
|
2738
|
-
}
|
|
2739
|
-
|
|
2740
|
-
} catch (err) {
|
|
2741
|
-
if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT') throw err
|
|
2742
|
-
if (readinessTracker && isDeviceUnavailable(err)) throw err
|
|
2743
|
-
const elapsed = Date.now() - overallStart
|
|
2744
|
-
return {
|
|
2745
|
-
status: 'timeout',
|
|
2746
|
-
error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
|
|
2747
|
-
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
2748
|
-
requested,
|
|
2749
|
-
observed: {
|
|
2750
|
-
matched_count: lastMatchedCount,
|
|
2751
|
-
condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
|
|
2752
|
-
selected_index: lastMatchedElement?.index ?? null,
|
|
2753
|
-
last_matched_element: lastMatchedElement
|
|
2754
|
-
}
|
|
2755
|
-
}
|
|
2756
|
-
}
|
|
235
|
+
static async waitForUIHandler(input: UIWaitInput) {
|
|
236
|
+
return waitForUI(input, { observe: target => ToolsObserve.getUITreeHandler(target),
|
|
237
|
+
resolve: (platform, deviceId, element, index) => ToolsInteract._buildResolvedElement(platform, deviceId, element, index) })
|
|
2757
238
|
}
|
|
2758
239
|
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
if (!Array.isArray(logsArr)) return []
|
|
2762
|
-
return logsArr.map((l: any) => {
|
|
2763
|
-
if (typeof l === 'string') return l
|
|
2764
|
-
if (l && (l.message || l.msg)) return l.message || l.msg
|
|
2765
|
-
try { return JSON.stringify(l) } catch { return String(l) }
|
|
2766
|
-
})
|
|
240
|
+
static async waitForScreenChangeHandler(input: ScreenWaitInput) {
|
|
241
|
+
return waitForScreenChange(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target) })
|
|
2767
242
|
}
|
|
2768
243
|
|
|
2769
|
-
static async
|
|
2770
|
-
|
|
2771
|
-
let lastFingerprint: string | null = null
|
|
2772
|
-
let lastActivity: string | null = null
|
|
2773
|
-
|
|
2774
|
-
while (Date.now() - start < timeoutMs) {
|
|
2775
|
-
try {
|
|
2776
|
-
const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
2777
|
-
const fp = res?.fingerprint ?? null
|
|
2778
|
-
lastActivity = (res as any)?.activity ?? lastActivity
|
|
2779
|
-
if (fp === null || fp === undefined) {
|
|
2780
|
-
lastFingerprint = null
|
|
2781
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
2782
|
-
continue
|
|
2783
|
-
}
|
|
2784
|
-
|
|
2785
|
-
lastFingerprint = fp
|
|
2786
|
-
|
|
2787
|
-
if (fp !== previousFingerprint) {
|
|
2788
|
-
// Stability confirmation
|
|
2789
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
2790
|
-
try {
|
|
2791
|
-
const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
2792
|
-
const confirmFp = confirmRes?.fingerprint ?? null
|
|
2793
|
-
lastActivity = (confirmRes as any)?.activity ?? lastActivity
|
|
2794
|
-
if (confirmFp === fp) {
|
|
2795
|
-
return {
|
|
2796
|
-
success: true,
|
|
2797
|
-
previousFingerprint,
|
|
2798
|
-
newFingerprint: fp,
|
|
2799
|
-
elapsedMs: Date.now() - start,
|
|
2800
|
-
observed_screen: {
|
|
2801
|
-
fingerprint: fp,
|
|
2802
|
-
activity: lastActivity
|
|
2803
|
-
}
|
|
2804
|
-
}
|
|
2805
|
-
}
|
|
2806
|
-
lastFingerprint = confirmFp
|
|
2807
|
-
continue
|
|
2808
|
-
} catch (e) { console.error('Error confirming fingerprint:', e); continue }
|
|
2809
|
-
}
|
|
2810
|
-
} catch (e) { console.error('Error getting screen fingerprint:', e) }
|
|
2811
|
-
|
|
2812
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
2813
|
-
}
|
|
2814
|
-
|
|
2815
|
-
return {
|
|
2816
|
-
success: false,
|
|
2817
|
-
reason: 'timeout',
|
|
2818
|
-
previousFingerprint,
|
|
2819
|
-
lastFingerprint,
|
|
2820
|
-
elapsedMs: Date.now() - start,
|
|
2821
|
-
observed_screen: {
|
|
2822
|
-
fingerprint: lastFingerprint,
|
|
2823
|
-
activity: lastActivity
|
|
2824
|
-
}
|
|
2825
|
-
}
|
|
244
|
+
static async waitForUIChangeHandler(input: ChangeInput): Promise<WaitForUIChangeResponse> {
|
|
245
|
+
return waitForUIChange(input, { observe: target => ToolsObserve.getUITreeHandler(target), scope: (tree, scope, target) => ToolsInteract._resolveUiChangeScope(tree, scope, target) })
|
|
2826
246
|
}
|
|
2827
247
|
|
|
2828
|
-
static async
|
|
2829
|
-
|
|
2830
|
-
deviceId,
|
|
2831
|
-
timeout_ms = 60000,
|
|
2832
|
-
stability_window_ms = 300,
|
|
2833
|
-
expected_change,
|
|
2834
|
-
scope = 'screen',
|
|
2835
|
-
target = null
|
|
2836
|
-
}: {
|
|
2837
|
-
platform?: 'android' | 'ios',
|
|
2838
|
-
deviceId?: string,
|
|
2839
|
-
timeout_ms?: number,
|
|
2840
|
-
stability_window_ms?: number,
|
|
2841
|
-
expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change',
|
|
2842
|
-
scope?: 'screen' | 'subtree',
|
|
2843
|
-
target?: string | null
|
|
2844
|
-
}): Promise<WaitForUIChangeResponse> {
|
|
2845
|
-
const readinessTracker = currentReadinessWait()
|
|
2846
|
-
const start = Date.now()
|
|
2847
|
-
const pollIntervalMs = 300
|
|
2848
|
-
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
|
|
2849
|
-
let baseline: UiChangeSignatureSet | null = null
|
|
2850
|
-
let baselineScope: UiChangeScopeResult | null = null
|
|
2851
|
-
let lastObservedRevision: number | null = null
|
|
2852
|
-
let lastLoadingState: any = null
|
|
2853
|
-
let lastSnapshotFreshnessMs: number | null = null
|
|
2854
|
-
let candidateSignatures: UiChangeSignatureSet | null = null
|
|
2855
|
-
let candidateObservedChange: 'hierarchy_diff' | 'text_change' | 'state_change' | null = null
|
|
2856
|
-
let candidateSinceMs: number | null = null
|
|
2857
|
-
let lastChangeSummary: ReturnType<typeof ToolsInteract._summarizeUiChangeDelta> | null = null
|
|
2858
|
-
let lastScopeResolution: UiChangeScopeResolution = {
|
|
2859
|
-
scope: scope === 'subtree' ? 'subtree' : 'screen',
|
|
2860
|
-
target: target && typeof target === 'string' ? target : null,
|
|
2861
|
-
resolved: scope !== 'subtree',
|
|
2862
|
-
resolvedIndex: null,
|
|
2863
|
-
resolvedStableId: null,
|
|
2864
|
-
reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
|
|
2865
|
-
}
|
|
2866
|
-
|
|
2867
|
-
while (Date.now() - start < timeout_ms) {
|
|
2868
|
-
try {
|
|
2869
|
-
if (readinessTracker) remainingBudget()
|
|
2870
|
-
const acquisitionStarted = Date.now()
|
|
2871
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
2872
|
-
observeReadinessWait(tree, acquisitionStarted)
|
|
2873
|
-
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
|
|
2874
|
-
if (scopedTree.error) {
|
|
2875
|
-
lastScopeResolution = scopedTree.resolution
|
|
2876
|
-
return {
|
|
2877
|
-
success: false,
|
|
2878
|
-
observed_change: null,
|
|
2879
|
-
snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
|
|
2880
|
-
snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
|
|
2881
|
-
timeout: true,
|
|
2882
|
-
elapsed_ms: Date.now() - start,
|
|
2883
|
-
expected_change,
|
|
2884
|
-
loading_state: tree?.loading_state ?? lastLoadingState ?? null,
|
|
2885
|
-
scope: scopedTree.resolution.scope,
|
|
2886
|
-
target: scopedTree.resolution.target,
|
|
2887
|
-
stability_state: 'transient',
|
|
2888
|
-
change_summary: lastChangeSummary,
|
|
2889
|
-
reason: scopedTree.error.message,
|
|
2890
|
-
error: scopedTree.error
|
|
2891
|
-
}
|
|
2892
|
-
}
|
|
2893
|
-
|
|
2894
|
-
const scopedElements = scopedTree.elements
|
|
2895
|
-
const scopedSignatureTree = {
|
|
2896
|
-
...tree,
|
|
2897
|
-
elements: scopedElements
|
|
2898
|
-
}
|
|
2899
|
-
const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree)
|
|
2900
|
-
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
|
|
2901
|
-
lastLoadingState = tree?.loading_state ?? lastLoadingState
|
|
2902
|
-
lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs
|
|
2903
|
-
lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary
|
|
2904
|
-
lastScopeResolution = scopedTree.resolution
|
|
2905
|
-
baselineScope = baselineScope ?? scopedTree
|
|
2906
|
-
|
|
2907
|
-
if (!baseline) {
|
|
2908
|
-
baseline = signatures
|
|
2909
|
-
baselineScope = scopedTree
|
|
2910
|
-
} else {
|
|
2911
|
-
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
|
|
2912
|
-
if (observedChange) {
|
|
2913
|
-
if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
|
|
2914
|
-
candidateSignatures = signatures
|
|
2915
|
-
candidateObservedChange = observedChange
|
|
2916
|
-
candidateSinceMs = Date.now()
|
|
2917
|
-
}
|
|
2918
|
-
|
|
2919
|
-
const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
|
|
2920
|
-
if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
|
|
2921
|
-
return {
|
|
2922
|
-
success: true,
|
|
2923
|
-
observed_change: candidateObservedChange ?? observedChange,
|
|
2924
|
-
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2925
|
-
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2926
|
-
timeout: false,
|
|
2927
|
-
elapsed_ms: Date.now() - start,
|
|
2928
|
-
expected_change,
|
|
2929
|
-
loading_state: lastLoadingState ?? null,
|
|
2930
|
-
scope: lastScopeResolution.scope,
|
|
2931
|
-
target: lastScopeResolution.target,
|
|
2932
|
-
stability_state: 'stable',
|
|
2933
|
-
change_summary: lastChangeSummary,
|
|
2934
|
-
reason: 'UI change observed'
|
|
2935
|
-
}
|
|
2936
|
-
}
|
|
2937
|
-
} else {
|
|
2938
|
-
candidateSignatures = null
|
|
2939
|
-
candidateObservedChange = null
|
|
2940
|
-
candidateSinceMs = null
|
|
2941
|
-
}
|
|
2942
|
-
}
|
|
2943
|
-
} catch (error) {
|
|
2944
|
-
if (readinessTracker && !String(error).includes('READINESS_')) readinessTracker.unavailable()
|
|
2945
|
-
if (readinessTracker && isDeviceUnavailable(error)) throw error
|
|
2946
|
-
if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) { readinessTracker.deadline(); break }
|
|
2947
|
-
// Default waits retain best-effort polling.
|
|
2948
|
-
}
|
|
2949
|
-
|
|
2950
|
-
await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)))
|
|
2951
|
-
}
|
|
2952
|
-
|
|
2953
|
-
return {
|
|
2954
|
-
success: false,
|
|
2955
|
-
observed_change: null,
|
|
2956
|
-
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2957
|
-
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2958
|
-
timeout: true,
|
|
2959
|
-
elapsed_ms: Date.now() - start,
|
|
2960
|
-
expected_change,
|
|
2961
|
-
loading_state: lastLoadingState ?? null,
|
|
2962
|
-
scope: lastScopeResolution.scope,
|
|
2963
|
-
target: lastScopeResolution.target,
|
|
2964
|
-
stability_state: 'transient',
|
|
2965
|
-
change_summary: lastChangeSummary,
|
|
2966
|
-
reason: 'timeout'
|
|
2967
|
-
}
|
|
248
|
+
static async expectScreenHandler(input: ScreenExpectation): Promise<ExpectScreenResponse> {
|
|
249
|
+
return expectScreen(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), current: target => ToolsObserve.getCurrentScreenHandler(target) })
|
|
2968
250
|
}
|
|
2969
251
|
|
|
2970
|
-
static async
|
|
2971
|
-
|
|
2972
|
-
fingerprint,
|
|
2973
|
-
screen,
|
|
2974
|
-
deviceId
|
|
2975
|
-
}: {
|
|
2976
|
-
platform?: 'android' | 'ios',
|
|
2977
|
-
fingerprint?: string,
|
|
2978
|
-
screen?: string,
|
|
2979
|
-
deviceId?: string
|
|
2980
|
-
}): Promise<ExpectScreenResponse> {
|
|
2981
|
-
const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
|
|
2982
|
-
const observedScreen = {
|
|
2983
|
-
fingerprint: observedFingerprint?.fingerprint ?? null,
|
|
2984
|
-
screen: observedFingerprint?.activity ?? null
|
|
2985
|
-
}
|
|
2986
|
-
|
|
2987
|
-
let observedScreenLabel = observedScreen.screen
|
|
2988
|
-
if (!fingerprint && screen && platform !== 'ios') {
|
|
2989
|
-
try {
|
|
2990
|
-
const current = await ToolsObserve.getCurrentScreenHandler({ deviceId }) as any
|
|
2991
|
-
observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel
|
|
2992
|
-
} catch {
|
|
2993
|
-
// Keep fingerprint-derived activity when current-screen lookup is unavailable.
|
|
2994
|
-
}
|
|
2995
|
-
}
|
|
2996
|
-
|
|
2997
|
-
const expectedScreen = {
|
|
2998
|
-
fingerprint: fingerprint ?? null,
|
|
2999
|
-
screen: screen ?? null
|
|
3000
|
-
}
|
|
3001
|
-
|
|
3002
|
-
let success = false
|
|
3003
|
-
let basis: 'fingerprint' | 'screen' | 'none' = 'none'
|
|
3004
|
-
let reason = 'No fingerprint or screen expectation provided'
|
|
3005
|
-
if (fingerprint) {
|
|
3006
|
-
basis = 'fingerprint'
|
|
3007
|
-
success = observedScreen.fingerprint === fingerprint
|
|
3008
|
-
reason = success
|
|
3009
|
-
? `observed fingerprint matches expected fingerprint ${fingerprint}`
|
|
3010
|
-
: `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`
|
|
3011
|
-
} else if (screen) {
|
|
3012
|
-
basis = 'screen'
|
|
3013
|
-
const candidates = new Set<string>()
|
|
3014
|
-
if (observedScreen.screen) candidates.add(observedScreen.screen)
|
|
3015
|
-
if (observedScreenLabel) candidates.add(observedScreenLabel)
|
|
3016
|
-
success = candidates.has(screen)
|
|
3017
|
-
reason = success
|
|
3018
|
-
? `observed screen matches expected screen ${screen}`
|
|
3019
|
-
: `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`
|
|
3020
|
-
}
|
|
3021
|
-
|
|
3022
|
-
return {
|
|
3023
|
-
success,
|
|
3024
|
-
observed_screen: {
|
|
3025
|
-
fingerprint: observedScreen.fingerprint,
|
|
3026
|
-
screen: observedScreenLabel
|
|
3027
|
-
},
|
|
3028
|
-
expected_screen: expectedScreen,
|
|
3029
|
-
confidence: success ? 1 : 0,
|
|
3030
|
-
comparison: {
|
|
3031
|
-
basis,
|
|
3032
|
-
matched: success,
|
|
3033
|
-
reason
|
|
3034
|
-
},
|
|
3035
|
-
trace: buildObservationTrace({
|
|
3036
|
-
actionType: 'expect_screen',
|
|
3037
|
-
stage: 'verify',
|
|
3038
|
-
success,
|
|
3039
|
-
attempts: 1,
|
|
3040
|
-
metadata: {
|
|
3041
|
-
expected_screen: expectedScreen,
|
|
3042
|
-
observed_screen: {
|
|
3043
|
-
fingerprint: observedScreen.fingerprint,
|
|
3044
|
-
screen: observedScreenLabel
|
|
3045
|
-
},
|
|
3046
|
-
comparison: {
|
|
3047
|
-
basis,
|
|
3048
|
-
matched: success
|
|
3049
|
-
}
|
|
3050
|
-
}
|
|
3051
|
-
})
|
|
3052
|
-
}
|
|
252
|
+
static async expectElementVisibleHandler(input: VisibilityInput): Promise<ExpectElementVisibleResponse> {
|
|
253
|
+
return expectVisible(input, request => ToolsInteract.waitForUIHandler(request))
|
|
3053
254
|
}
|
|
3054
255
|
|
|
3055
|
-
static async
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
poll_interval_ms = 300,
|
|
3060
|
-
platform,
|
|
3061
|
-
deviceId
|
|
3062
|
-
}: {
|
|
3063
|
-
selector: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
|
|
3064
|
-
element_id?: string,
|
|
3065
|
-
timeout_ms?: number,
|
|
3066
|
-
poll_interval_ms?: number,
|
|
3067
|
-
platform?: 'android' | 'ios',
|
|
3068
|
-
deviceId?: string
|
|
3069
|
-
}): Promise<ExpectElementVisibleResponse> {
|
|
3070
|
-
const result = await ToolsInteract.waitForUIHandler({
|
|
3071
|
-
selector,
|
|
3072
|
-
condition: 'visible',
|
|
3073
|
-
timeout_ms,
|
|
3074
|
-
poll_interval_ms,
|
|
3075
|
-
platform,
|
|
3076
|
-
deviceId
|
|
3077
|
-
}) as any
|
|
3078
|
-
|
|
3079
|
-
if (result?.status === 'success' && result?.element) {
|
|
3080
|
-
return {
|
|
3081
|
-
success: true,
|
|
3082
|
-
selector,
|
|
3083
|
-
element_id: result.element.elementId ?? element_id ?? null,
|
|
3084
|
-
expected_condition: 'visible',
|
|
3085
|
-
element: {
|
|
3086
|
-
elementId: result.element.elementId ?? null,
|
|
3087
|
-
text: result.element.text ?? null,
|
|
3088
|
-
resource_id: result.element.resource_id ?? null,
|
|
3089
|
-
accessibility_id: result.element.accessibility_id ?? null,
|
|
3090
|
-
class: result.element.class ?? null,
|
|
3091
|
-
bounds: result.element.bounds ?? null,
|
|
3092
|
-
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
3093
|
-
state: (result.element as any).state ?? null,
|
|
3094
|
-
stable_id: (result.element as any).stable_id ?? null,
|
|
3095
|
-
role: (result.element as any).role ?? null,
|
|
3096
|
-
test_tag: (result.element as any).test_tag ?? null,
|
|
3097
|
-
selector: (result.element as any).selector ?? null,
|
|
3098
|
-
semantic: (result.element as any).semantic ?? null
|
|
3099
|
-
},
|
|
3100
|
-
observed: {
|
|
3101
|
-
status: result.status,
|
|
3102
|
-
matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
|
|
3103
|
-
condition_satisfied: true,
|
|
3104
|
-
selected_index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
3105
|
-
last_matched_element: {
|
|
3106
|
-
elementId: result.element.elementId ?? null,
|
|
3107
|
-
text: result.element.text ?? null,
|
|
3108
|
-
resource_id: result.element.resource_id ?? null,
|
|
3109
|
-
accessibility_id: result.element.accessibility_id ?? null,
|
|
3110
|
-
class: result.element.class ?? null,
|
|
3111
|
-
bounds: result.element.bounds ?? null,
|
|
3112
|
-
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
3113
|
-
state: (result.element as any).state ?? null,
|
|
3114
|
-
stable_id: (result.element as any).stable_id ?? null,
|
|
3115
|
-
role: (result.element as any).role ?? null,
|
|
3116
|
-
test_tag: (result.element as any).test_tag ?? null,
|
|
3117
|
-
selector: (result.element as any).selector ?? null,
|
|
3118
|
-
semantic: (result.element as any).semantic ?? null
|
|
3119
|
-
}
|
|
3120
|
-
},
|
|
3121
|
-
reason: 'selector is visible',
|
|
3122
|
-
trace: buildObservationTrace({
|
|
3123
|
-
actionType: 'expect_element_visible',
|
|
3124
|
-
stage: 'verify',
|
|
3125
|
-
success: true,
|
|
3126
|
-
attempts: 1,
|
|
3127
|
-
metadata: {
|
|
3128
|
-
selector,
|
|
3129
|
-
element_id: result.element.elementId ?? element_id ?? null,
|
|
3130
|
-
status: result.status
|
|
3131
|
-
}
|
|
3132
|
-
})
|
|
3133
|
-
}
|
|
3134
|
-
}
|
|
3135
|
-
|
|
3136
|
-
const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT'
|
|
3137
|
-
return {
|
|
3138
|
-
success: false,
|
|
3139
|
-
selector,
|
|
3140
|
-
element_id: element_id ?? null,
|
|
3141
|
-
expected_condition: 'visible',
|
|
3142
|
-
observed: {
|
|
3143
|
-
status: result?.status,
|
|
3144
|
-
matched_count: result?.observed?.matched_count,
|
|
3145
|
-
condition_satisfied: result?.observed?.condition_satisfied ?? false,
|
|
3146
|
-
selected_index: result?.observed?.selected_index ?? null,
|
|
3147
|
-
last_matched_element: result?.observed?.last_matched_element ?? null
|
|
3148
|
-
},
|
|
3149
|
-
reason: result?.error?.message ?? 'selector is not visible',
|
|
3150
|
-
failure_code: errorCode,
|
|
3151
|
-
retryable: errorCode === 'TIMEOUT',
|
|
3152
|
-
trace: buildObservationTrace({
|
|
3153
|
-
actionType: 'expect_element_visible',
|
|
3154
|
-
stage: 'verify',
|
|
3155
|
-
success: false,
|
|
3156
|
-
attempts: 1,
|
|
3157
|
-
metadata: {
|
|
3158
|
-
selector,
|
|
3159
|
-
element_id: element_id ?? null,
|
|
3160
|
-
status: result?.status ?? null,
|
|
3161
|
-
reason: result?.error?.message ?? 'selector is not visible'
|
|
3162
|
-
}
|
|
3163
|
-
})
|
|
3164
|
-
}
|
|
3165
|
-
}
|
|
3166
|
-
|
|
3167
|
-
static async expectStateHandler({
|
|
3168
|
-
selector,
|
|
3169
|
-
element_id,
|
|
3170
|
-
property,
|
|
3171
|
-
expected,
|
|
3172
|
-
platform,
|
|
3173
|
-
deviceId,
|
|
3174
|
-
stabilization_window_ms = 1000,
|
|
3175
|
-
stable_observation_count = 2,
|
|
3176
|
-
snapshot_stale_threshold_ms = 500,
|
|
3177
|
-
poll_interval_ms = 150,
|
|
3178
|
-
exact_budget = false
|
|
3179
|
-
}: {
|
|
3180
|
-
selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
|
|
3181
|
-
element_id?: string,
|
|
3182
|
-
property: string,
|
|
3183
|
-
expected: boolean | number | string | Record<string, unknown>,
|
|
3184
|
-
platform?: 'android' | 'ios',
|
|
3185
|
-
deviceId?: string,
|
|
3186
|
-
stabilization_window_ms?: number,
|
|
3187
|
-
stable_observation_count?: number,
|
|
3188
|
-
snapshot_stale_threshold_ms?: number,
|
|
3189
|
-
poll_interval_ms?: number,
|
|
3190
|
-
exact_budget?: boolean
|
|
3191
|
-
}): Promise<ExpectStateResponse> {
|
|
3192
|
-
const compareBoolean = (value: unknown) => typeof value === 'boolean' ? value : null
|
|
3193
|
-
const compareString = (value: unknown) => typeof value === 'string' ? value : null
|
|
3194
|
-
const compareNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value) ? value : null
|
|
3195
|
-
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
|
|
3196
|
-
const start = Date.now()
|
|
3197
|
-
const deadline = start + (exact_budget ? stabilization_window_ms : Math.max(500, stabilization_window_ms))
|
|
3198
|
-
const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2))
|
|
3199
|
-
const pollDelay = exact_budget ? Math.max(50, Math.min(poll_interval_ms || 150, 1000)) : Math.max(100, Math.min(poll_interval_ms || 150, 200))
|
|
3200
|
-
const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800))
|
|
3201
|
-
|
|
3202
|
-
let attempts = 0
|
|
3203
|
-
let stableCount = 0
|
|
3204
|
-
let lastReason = 'element not found'
|
|
3205
|
-
let lastFailureCode: 'ELEMENT_NOT_FOUND' | 'UNKNOWN' = 'ELEMENT_NOT_FOUND'
|
|
3206
|
-
let lastObservedElement: (ActionTargetResolved & { state?: UIElementState | null }) | null = null
|
|
3207
|
-
let lastObservedValue: boolean | number | string | Record<string, unknown> | null = null
|
|
3208
|
-
let lastRawValue: boolean | number | string | null = null
|
|
3209
|
-
let lastResolvedElementId: string | null = element_id ?? null
|
|
3210
|
-
const traceSteps: TraceStep[] = []
|
|
3211
|
-
let traceAttemptIndex = 0
|
|
3212
|
-
let resolveRecorded = false
|
|
3213
|
-
|
|
3214
|
-
const recordTraceStep = (
|
|
3215
|
-
stage: TraceStep['stage'],
|
|
3216
|
-
result: TraceStep['result'],
|
|
3217
|
-
metadata?: Record<string, unknown>
|
|
3218
|
-
) => {
|
|
3219
|
-
traceSteps.push(createTraceStep({
|
|
3220
|
-
stage,
|
|
3221
|
-
timestamp: Date.now(),
|
|
3222
|
-
result,
|
|
3223
|
-
attemptIndex: traceAttemptIndex++,
|
|
3224
|
-
metadata
|
|
3225
|
-
}))
|
|
3226
|
-
}
|
|
3227
|
-
|
|
3228
|
-
const buildStateTrace = (outcome: 'success' | 'failure'): ActionTrace => ({
|
|
3229
|
-
action_id: nextActionId('expect_state', Date.now()),
|
|
3230
|
-
steps: traceSteps,
|
|
3231
|
-
final_outcome: outcome,
|
|
3232
|
-
attempts
|
|
3233
|
-
})
|
|
3234
|
-
|
|
3235
|
-
while (Date.now() <= deadline) {
|
|
3236
|
-
attempts++
|
|
3237
|
-
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
3238
|
-
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
3239
|
-
const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
|
|
3240
|
-
const treeDeviceId = tree?.device?.id || deviceId
|
|
3241
|
-
const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null
|
|
3242
|
-
let matched: { el: UiElement, idx: number } | null = null
|
|
3243
|
-
|
|
3244
|
-
if (element_id) {
|
|
3245
|
-
const resolved = ToolsInteract._resolvedUiElements.get(element_id)
|
|
3246
|
-
if (resolved) {
|
|
3247
|
-
const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
|
|
3248
|
-
if (current) matched = { el: current.el, idx: current.index }
|
|
3249
|
-
}
|
|
3250
|
-
}
|
|
3251
|
-
|
|
3252
|
-
if (!matched && selector) {
|
|
3253
|
-
matched = ToolsInteract._findFirstMatchingElement(elements, selector)
|
|
3254
|
-
}
|
|
3255
|
-
|
|
3256
|
-
if (!matched) {
|
|
3257
|
-
lastReason = 'element not found'
|
|
3258
|
-
lastFailureCode = 'ELEMENT_NOT_FOUND'
|
|
3259
|
-
stableCount = 0
|
|
3260
|
-
recordTraceStep('resolve', 'retry', {
|
|
3261
|
-
selector: selector ?? null,
|
|
3262
|
-
element_id: lastResolvedElementId,
|
|
3263
|
-
matched: false,
|
|
3264
|
-
reason: lastReason,
|
|
3265
|
-
attempt: attempts
|
|
3266
|
-
})
|
|
3267
|
-
recordTraceStep('stabilize', 'retry', {
|
|
3268
|
-
stabilization_attempts: attempts,
|
|
3269
|
-
stable_observation_count: stableCount,
|
|
3270
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
3271
|
-
reason: lastReason
|
|
3272
|
-
})
|
|
3273
|
-
await sleep(pollDelay)
|
|
3274
|
-
continue
|
|
3275
|
-
}
|
|
3276
|
-
|
|
3277
|
-
const resolvedElement = ToolsInteract._resolvedTargetFromElement(
|
|
3278
|
-
ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx),
|
|
3279
|
-
matched.el,
|
|
3280
|
-
matched.idx
|
|
3281
|
-
)
|
|
3282
|
-
lastResolvedElementId = resolvedElement.elementId
|
|
3283
|
-
lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null }
|
|
3284
|
-
|
|
3285
|
-
if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
|
|
3286
|
-
lastReason = 'stale snapshot'
|
|
3287
|
-
lastFailureCode = 'UNKNOWN'
|
|
3288
|
-
stableCount = 0
|
|
3289
|
-
recordTraceStep('resolve', 'retry', {
|
|
3290
|
-
selector: selector ?? null,
|
|
3291
|
-
element_id: lastResolvedElementId,
|
|
3292
|
-
matched: true,
|
|
3293
|
-
reason: lastReason,
|
|
3294
|
-
attempt: attempts
|
|
3295
|
-
})
|
|
3296
|
-
recordTraceStep('stabilize', 'retry', {
|
|
3297
|
-
stabilization_attempts: attempts,
|
|
3298
|
-
stable_observation_count: stableCount,
|
|
3299
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
3300
|
-
reason: lastReason
|
|
3301
|
-
})
|
|
3302
|
-
await sleep(pollDelay)
|
|
3303
|
-
continue
|
|
3304
|
-
}
|
|
3305
|
-
|
|
3306
|
-
if (!resolveRecorded) {
|
|
3307
|
-
recordTraceStep('resolve', 'success', {
|
|
3308
|
-
selector: selector ?? null,
|
|
3309
|
-
element_id: lastResolvedElementId,
|
|
3310
|
-
matched: true,
|
|
3311
|
-
reason: 'element resolved'
|
|
3312
|
-
})
|
|
3313
|
-
resolveRecorded = true
|
|
3314
|
-
}
|
|
3315
|
-
|
|
3316
|
-
const observedState = matched.el.state ?? null
|
|
3317
|
-
const actual = observedState?.[property as keyof UIElementState] ?? null
|
|
3318
|
-
|
|
3319
|
-
let success = false
|
|
3320
|
-
let reason = ''
|
|
3321
|
-
let rawValue: boolean | number | string | null = null
|
|
3322
|
-
let observedValue: boolean | number | string | Record<string, unknown> | null = actual as any
|
|
3323
|
-
|
|
3324
|
-
switch (property) {
|
|
3325
|
-
case 'checked':
|
|
3326
|
-
case 'focused':
|
|
3327
|
-
case 'expanded':
|
|
3328
|
-
case 'enabled': {
|
|
3329
|
-
const expectedBool = compareBoolean(expected)
|
|
3330
|
-
const actualBool = compareBoolean(actual)
|
|
3331
|
-
if (expectedBool === null) {
|
|
3332
|
-
reason = `expected ${property} must be boolean`
|
|
3333
|
-
} else if (actualBool === null) {
|
|
3334
|
-
reason = `${property} state unavailable`
|
|
3335
|
-
} else {
|
|
3336
|
-
rawValue = actualBool
|
|
3337
|
-
success = actualBool === expectedBool
|
|
3338
|
-
reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`
|
|
3339
|
-
}
|
|
3340
|
-
observedValue = actualBool
|
|
3341
|
-
break
|
|
3342
|
-
}
|
|
3343
|
-
case 'value':
|
|
3344
|
-
case 'raw_value': {
|
|
3345
|
-
const expectedNumber = compareNumber(expected)
|
|
3346
|
-
const actualNumber = compareNumber(actual)
|
|
3347
|
-
if (expectedNumber !== null && actualNumber !== null) {
|
|
3348
|
-
success = actualNumber === expectedNumber
|
|
3349
|
-
rawValue = actualNumber
|
|
3350
|
-
observedValue = actualNumber
|
|
3351
|
-
reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`
|
|
3352
|
-
break
|
|
3353
|
-
}
|
|
3354
|
-
const expectedString = typeof expected === 'string' ? expected : null
|
|
3355
|
-
const actualString = compareString(actual)
|
|
3356
|
-
if (expectedString !== null && actualString !== null) {
|
|
3357
|
-
success = actualString === expectedString
|
|
3358
|
-
rawValue = actualString
|
|
3359
|
-
observedValue = actualString
|
|
3360
|
-
reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`
|
|
3361
|
-
} else {
|
|
3362
|
-
reason = 'value state unavailable'
|
|
3363
|
-
}
|
|
3364
|
-
break
|
|
3365
|
-
}
|
|
3366
|
-
case 'selected': {
|
|
3367
|
-
const expectedBool = typeof expected === 'boolean' ? expected : null
|
|
3368
|
-
const expectedString = typeof expected === 'string'
|
|
3369
|
-
? expected
|
|
3370
|
-
: expected && typeof expected === 'object'
|
|
3371
|
-
? String((expected as { id?: unknown; label?: unknown }).id ?? (expected as { id?: unknown; label?: unknown }).label ?? '')
|
|
3372
|
-
: null
|
|
3373
|
-
if (!observedState || observedState.selected === undefined || observedState.selected === null) {
|
|
3374
|
-
reason = 'selected state unavailable'
|
|
3375
|
-
break
|
|
3376
|
-
}
|
|
3377
|
-
if (expectedBool !== null) {
|
|
3378
|
-
const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null
|
|
3379
|
-
if (actualBool === null) {
|
|
3380
|
-
reason = 'selected state is not boolean'
|
|
3381
|
-
break
|
|
3382
|
-
}
|
|
3383
|
-
rawValue = actualBool
|
|
3384
|
-
observedValue = actualBool
|
|
3385
|
-
success = actualBool === expectedBool
|
|
3386
|
-
reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`
|
|
3387
|
-
break
|
|
3388
|
-
}
|
|
3389
|
-
const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
|
|
3390
|
-
? String((observedState.selected as { id?: unknown; label?: unknown }).id ?? (observedState.selected as { id?: unknown; label?: unknown }).label ?? '')
|
|
3391
|
-
: String(observedState.selected)
|
|
3392
|
-
const actualString = actualSelected.trim()
|
|
3393
|
-
if (!expectedString) {
|
|
3394
|
-
reason = 'expected selected must be boolean, string, or object with id/label'
|
|
3395
|
-
break
|
|
3396
|
-
}
|
|
3397
|
-
rawValue = actualString
|
|
3398
|
-
observedValue = actualString
|
|
3399
|
-
success = actualString === expectedString
|
|
3400
|
-
reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`
|
|
3401
|
-
break
|
|
3402
|
-
}
|
|
3403
|
-
case 'text_value': {
|
|
3404
|
-
const expectedString = typeof expected === 'string' ? expected : null
|
|
3405
|
-
const actualString = compareString(actual)
|
|
3406
|
-
if (!expectedString) {
|
|
3407
|
-
reason = 'expected text_value must be string'
|
|
3408
|
-
} else if (!actualString) {
|
|
3409
|
-
reason = 'text_value state unavailable'
|
|
3410
|
-
} else {
|
|
3411
|
-
success = actualString === expectedString
|
|
3412
|
-
rawValue = actualString
|
|
3413
|
-
observedValue = actualString
|
|
3414
|
-
reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`
|
|
3415
|
-
}
|
|
3416
|
-
break
|
|
3417
|
-
}
|
|
3418
|
-
default: {
|
|
3419
|
-
if (actual !== null && actual !== undefined) {
|
|
3420
|
-
success = actual === expected
|
|
3421
|
-
observedValue = actual as any
|
|
3422
|
-
rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null
|
|
3423
|
-
reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`
|
|
3424
|
-
} else {
|
|
3425
|
-
reason = `unsupported or unavailable state property: ${property}`
|
|
3426
|
-
}
|
|
3427
|
-
}
|
|
3428
|
-
}
|
|
3429
|
-
|
|
3430
|
-
if (success) {
|
|
3431
|
-
stableCount++
|
|
3432
|
-
recordTraceStep('stabilize', 'success', {
|
|
3433
|
-
stabilization_attempts: attempts,
|
|
3434
|
-
stable_observation_count: stableCount,
|
|
3435
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
3436
|
-
reason
|
|
3437
|
-
})
|
|
3438
|
-
recordTraceStep('verify', 'success', {
|
|
3439
|
-
property,
|
|
3440
|
-
expected,
|
|
3441
|
-
observed_value: observedValue,
|
|
3442
|
-
reason
|
|
3443
|
-
})
|
|
3444
|
-
if (stableCount >= stableTarget) {
|
|
3445
|
-
return {
|
|
3446
|
-
success: true,
|
|
3447
|
-
selector,
|
|
3448
|
-
element_id: lastResolvedElementId,
|
|
3449
|
-
expected_state: { property, expected },
|
|
3450
|
-
element: lastObservedElement,
|
|
3451
|
-
observed_state: {
|
|
3452
|
-
property,
|
|
3453
|
-
value: observedValue,
|
|
3454
|
-
...(rawValue !== null ? { raw_value: rawValue } : {})
|
|
3455
|
-
},
|
|
3456
|
-
reason,
|
|
3457
|
-
stabilization_attempts: attempts,
|
|
3458
|
-
stabilization_window_ms: Date.now() - start,
|
|
3459
|
-
stable_observation_count: stableCount,
|
|
3460
|
-
snapshot_freshness_ms: treeAgeMs ?? undefined,
|
|
3461
|
-
trace: buildStateTrace('success')
|
|
3462
|
-
} as ExpectStateResponse & {
|
|
3463
|
-
stabilization_attempts?: number;
|
|
3464
|
-
stabilization_window_ms?: number;
|
|
3465
|
-
stable_observation_count?: number;
|
|
3466
|
-
snapshot_freshness_ms?: number;
|
|
3467
|
-
}
|
|
3468
|
-
}
|
|
3469
|
-
} else {
|
|
3470
|
-
stableCount = 0
|
|
3471
|
-
lastReason = reason || lastReason
|
|
3472
|
-
lastFailureCode = 'UNKNOWN'
|
|
3473
|
-
recordTraceStep('stabilize', 'retry', {
|
|
3474
|
-
stabilization_attempts: attempts,
|
|
3475
|
-
stable_observation_count: stableCount,
|
|
3476
|
-
snapshot_freshness_ms: treeAgeMs,
|
|
3477
|
-
reason: lastReason
|
|
3478
|
-
})
|
|
3479
|
-
recordTraceStep('verify', 'retry', {
|
|
3480
|
-
property,
|
|
3481
|
-
expected,
|
|
3482
|
-
observed_value: observedValue,
|
|
3483
|
-
reason: lastReason
|
|
3484
|
-
})
|
|
3485
|
-
}
|
|
3486
|
-
|
|
3487
|
-
if (!success) {
|
|
3488
|
-
lastObservedValue = observedValue
|
|
3489
|
-
lastRawValue = rawValue
|
|
3490
|
-
}
|
|
3491
|
-
|
|
3492
|
-
await sleep(pollDelay)
|
|
3493
|
-
}
|
|
3494
|
-
|
|
3495
|
-
return {
|
|
3496
|
-
success: false,
|
|
3497
|
-
selector,
|
|
3498
|
-
element_id: lastResolvedElementId,
|
|
3499
|
-
expected_state: { property, expected },
|
|
3500
|
-
element: lastObservedElement,
|
|
3501
|
-
observed_state: {
|
|
3502
|
-
property,
|
|
3503
|
-
value: lastObservedValue,
|
|
3504
|
-
...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
|
|
3505
|
-
},
|
|
3506
|
-
reason: lastReason,
|
|
3507
|
-
failure_code: lastFailureCode,
|
|
3508
|
-
retryable: true,
|
|
3509
|
-
trace: buildStateTrace('failure')
|
|
3510
|
-
}
|
|
256
|
+
static async expectStateHandler(input: StateInput): Promise<ExpectStateResponse> {
|
|
257
|
+
return expectState(input, { observe: target => ToolsObserve.getUITreeHandler(target), getHandle: id => ToolsInteract._resolvedUiElements.get(id),
|
|
258
|
+
currentHandle: (elements, platform, deviceId, handle) => ToolsInteract._findCurrentResolvedElement(elements, platform, deviceId, handle),
|
|
259
|
+
target: (platform, deviceId, match) => ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(platform, deviceId, match.el, match.idx), match.el, match.idx) })
|
|
3511
260
|
}
|
|
3512
261
|
|
|
3513
|
-
static async waitForUICore(
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
// Clamp polling interval to 250-500ms for consistent behavior
|
|
3519
|
-
const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500))
|
|
3520
|
-
|
|
3521
|
-
// Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
|
|
3522
|
-
let initialFingerprint: string | null = null
|
|
3523
|
-
let baselineLastLine: string | null = null
|
|
3524
|
-
try {
|
|
3525
|
-
const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as Promise<ScreenFingerprintResponse | null>
|
|
3526
|
-
const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as Promise<any>
|
|
3527
|
-
const withTimeout = (p: Promise<any>, ms: number) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))])
|
|
3528
|
-
const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)])
|
|
3529
|
-
if (fpRes && typeof fpRes === 'object') initialFingerprint = (fpRes as ScreenFingerprintResponse).fingerprint ?? null
|
|
3530
|
-
if (gl) {
|
|
3531
|
-
const logsArr = Array.isArray((gl as any).logs) ? (gl as any).logs : []
|
|
3532
|
-
// Normalize to last message string for baseline comparison
|
|
3533
|
-
const msgs = ToolsInteract._logsToMessages(logsArr)
|
|
3534
|
-
baselineLastLine = msgs.length ? msgs[msgs.length - 1] : null
|
|
3535
|
-
}
|
|
3536
|
-
} catch (err) {
|
|
3537
|
-
try { console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err)) } catch { }
|
|
3538
|
-
}
|
|
3539
|
-
|
|
3540
|
-
// Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
|
|
3541
|
-
let lastChangeAt = Date.now()
|
|
3542
|
-
let prevFingerprint = initialFingerprint
|
|
3543
|
-
|
|
3544
|
-
const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))
|
|
3545
|
-
|
|
3546
|
-
// Optional initial observation delay requested by caller
|
|
3547
|
-
if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
|
|
3548
|
-
try { console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`) } catch { }
|
|
3549
|
-
await sleep(observationDelayMs)
|
|
3550
|
-
}
|
|
3551
|
-
|
|
3552
|
-
// Telemetry
|
|
3553
|
-
let pollCount = 0
|
|
3554
|
-
let matchedAt: number | null = null
|
|
3555
|
-
let lastObservedState: boolean | null = null
|
|
3556
|
-
let stableDuration = 0
|
|
3557
|
-
let matchSource: string | null = null
|
|
3558
|
-
|
|
3559
|
-
while (Date.now() <= deadline) {
|
|
3560
|
-
pollCount++
|
|
3561
|
-
const now = Date.now()
|
|
3562
|
-
// Evaluate condition per type
|
|
3563
|
-
if (type === 'ui') {
|
|
3564
|
-
try {
|
|
3565
|
-
// Prefer using the public findElementHandler which tests can override. This avoids relying
|
|
3566
|
-
// on resolveObserve/getUITree for unit tests which may not have devices available.
|
|
3567
|
-
try {
|
|
3568
|
-
const findRes = await (ToolsInteract as any).findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId })
|
|
3569
|
-
const isPresent = !!(findRes && (findRes as any).found)
|
|
3570
|
-
const conditionTrue = (match === 'present') ? isPresent : !isPresent
|
|
3571
|
-
if (conditionTrue) {
|
|
3572
|
-
if (matchedAt === null) matchedAt = Date.now()
|
|
3573
|
-
stableDuration = Date.now() - (matchedAt as number)
|
|
3574
|
-
lastObservedState = true
|
|
3575
|
-
if (stableDuration >= stability_ms) {
|
|
3576
|
-
matchSource = 'ui-find'
|
|
3577
|
-
const element = isPresent ? (findRes as any).element : null
|
|
3578
|
-
const now2 = Date.now()
|
|
3579
|
-
return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null }
|
|
3580
|
-
}
|
|
3581
|
-
} else {
|
|
3582
|
-
matchedAt = null
|
|
3583
|
-
stableDuration = 0
|
|
3584
|
-
lastObservedState = false
|
|
3585
|
-
}
|
|
3586
|
-
} catch (err) { console.error('waitForUI(ui) find error:', err) }
|
|
3587
|
-
} catch (err) { console.error('waitForUI(ui) outer error:', err) }
|
|
3588
|
-
} else if (type === 'log') {
|
|
3589
|
-
try {
|
|
3590
|
-
// Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
|
|
3591
|
-
const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 }) as any
|
|
3592
|
-
const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : []
|
|
3593
|
-
for (const ent of entries) {
|
|
3594
|
-
const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : ''
|
|
3595
|
-
if (q && String(msg).includes(q)) {
|
|
3596
|
-
const now2 = Date.now()
|
|
3597
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true }
|
|
3598
|
-
}
|
|
3599
|
-
}
|
|
3600
|
-
|
|
3601
|
-
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as any
|
|
3602
|
-
const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : []
|
|
3603
|
-
// Normalize to messages for comparison
|
|
3604
|
-
const msgs = ToolsInteract._logsToMessages(logsArr)
|
|
3605
|
-
let startIndex = 0
|
|
3606
|
-
if (baselineLastLine) {
|
|
3607
|
-
const idx = msgs.lastIndexOf(baselineLastLine)
|
|
3608
|
-
startIndex = idx >= 0 ? idx + 1 : 0
|
|
3609
|
-
}
|
|
3610
|
-
for (let i = startIndex; i < msgs.length; i++) {
|
|
3611
|
-
const line = msgs[i]
|
|
3612
|
-
if (q && String(line).includes(q)) {
|
|
3613
|
-
const now2 = Date.now()
|
|
3614
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true }
|
|
3615
|
-
}
|
|
3616
|
-
}
|
|
3617
|
-
} catch (err) { console.error('waitForUI(log) error:', err) }
|
|
3618
|
-
} else if (type === 'screen') {
|
|
3619
|
-
try {
|
|
3620
|
-
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
3621
|
-
const fp = fpRes?.fingerprint ?? null
|
|
3622
|
-
if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
|
|
3623
|
-
// when screen changed, require stability_ms where fingerprint remains the same
|
|
3624
|
-
if (matchedAt === null) matchedAt = now
|
|
3625
|
-
const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null)?.fingerprint ?? null
|
|
3626
|
-
if (confirmFp === fp) {
|
|
3627
|
-
stableDuration = Date.now() - (matchedAt as number)
|
|
3628
|
-
lastObservedState = true
|
|
3629
|
-
if (stableDuration >= stability_ms) {
|
|
3630
|
-
const now2 = Date.now()
|
|
3631
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null }
|
|
3632
|
-
}
|
|
3633
|
-
} else {
|
|
3634
|
-
matchedAt = null
|
|
3635
|
-
stableDuration = 0
|
|
3636
|
-
lastObservedState = false
|
|
3637
|
-
}
|
|
3638
|
-
}
|
|
3639
|
-
} catch (err) { console.error('waitForUI(screen) error:', err) }
|
|
3640
|
-
} else if (type === 'idle') {
|
|
3641
|
-
try {
|
|
3642
|
-
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
3643
|
-
const fp = fpRes?.fingerprint ?? null
|
|
3644
|
-
if (fp !== prevFingerprint) {
|
|
3645
|
-
prevFingerprint = fp
|
|
3646
|
-
lastChangeAt = Date.now()
|
|
3647
|
-
matchedAt = null
|
|
3648
|
-
stableDuration = 0
|
|
3649
|
-
lastObservedState = false
|
|
3650
|
-
} else {
|
|
3651
|
-
const idleMs = Date.now() - lastChangeAt
|
|
3652
|
-
lastObservedState = true
|
|
3653
|
-
if (idleMs >= stability_ms) {
|
|
3654
|
-
const now2 = Date.now()
|
|
3655
|
-
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null }
|
|
3656
|
-
}
|
|
3657
|
-
}
|
|
3658
|
-
} catch (err) { console.error('waitForUI(idle) error:', err) }
|
|
3659
|
-
}
|
|
3660
|
-
|
|
3661
|
-
// Respect poll interval and avoid tight loop
|
|
3662
|
-
await sleep(pollInterval)
|
|
3663
|
-
}
|
|
3664
|
-
|
|
3665
|
-
// On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
|
|
3666
|
-
let snapshot: any = null
|
|
3667
|
-
if (includeSnapshotOnFailure) {
|
|
3668
|
-
try {
|
|
3669
|
-
// Use dynamic import to avoid circular-initialization issues where the ToolsObserve
|
|
3670
|
-
// binding captured earlier may not reflect test-time overrides. Importing at call
|
|
3671
|
-
// time ensures the latest exported ToolsObserve object is used.
|
|
3672
|
-
const Obs = await import('../observe/index.js')
|
|
3673
|
-
snapshot = await (Obs as any).ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId })
|
|
3674
|
-
} catch (err) {
|
|
3675
|
-
snapshot = { error: err instanceof Error ? err.message : String(err) }
|
|
3676
|
-
}
|
|
3677
|
-
}
|
|
3678
|
-
|
|
3679
|
-
const elapsed = Date.now() - start
|
|
3680
|
-
return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null }
|
|
262
|
+
static async waitForUICore(input: LegacyWaitInput) {
|
|
263
|
+
return waitForUICore(input, { fingerprint: target => ToolsObserve.getScreenFingerprintHandler(target), logs: target => ToolsObserve.getLogsHandler(target),
|
|
264
|
+
stream: target => ToolsObserve.readLogStreamHandler(target), find: input => ToolsInteract.findElementHandler(input),
|
|
265
|
+
snapshot: async input => { const { ToolsObserve } = await import('../observe/index.js'); return ToolsObserve.captureDebugSnapshotHandler(input) } })
|
|
3681
266
|
}
|
|
3682
267
|
}
|