mobile-debug-mcp 0.26.3 → 0.26.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +456 -0
- package/dist/observe/ios.js +1 -1
- package/dist/server/tool-definitions.js +56 -0
- package/dist/server/tool-handlers.js +25 -0
- package/dist/server-core.js +1 -1
- package/dist/utils/android/utils.js +2 -2
- package/docs/CHANGELOG.md +3 -0
- package/docs/ROADMAP.md +64 -9
- package/docs/rfcs/008-adjustable-control-support-and-semantic-value-manipulation.md +273 -0
- package/docs/specs/mcp-tooling-spec-v1.md +1 -1
- package/docs/tools/interact.md +21 -0
- package/package.json +1 -1
- package/src/interact/index.ts +585 -0
- package/src/observe/ios.ts +1 -1
- package/src/server/tool-definitions.ts +56 -0
- package/src/server/tool-handlers.ts +26 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +17 -0
- package/src/utils/android/utils.ts +2 -2
- package/test/unit/interact/adjust_control.test.ts +365 -0
- package/test/unit/observe/state_extraction.test.ts +24 -0
- package/test/unit/server/contract.test.ts +8 -0
- package/test/unit/server/response_shapes.test.ts +39 -0
package/src/interact/index.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { buildActionExecutionResult } from '../server/common.js'
|
|
|
10
10
|
import type {
|
|
11
11
|
ActionFailureCode,
|
|
12
12
|
ActionTargetResolved,
|
|
13
|
+
AdjustControlResponse,
|
|
13
14
|
FindElementResponse,
|
|
14
15
|
ExpectElementVisibleResponse,
|
|
15
16
|
ExpectStateResponse,
|
|
@@ -334,6 +335,89 @@ export class ToolsInteract {
|
|
|
334
335
|
}
|
|
335
336
|
}
|
|
336
337
|
|
|
338
|
+
private static _isAdjustableControl(el: UiElement | null): boolean {
|
|
339
|
+
if (!el) return false
|
|
340
|
+
const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
|
|
341
|
+
const role = ToolsInteract._normalize(el.role ?? '')
|
|
342
|
+
return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
|
|
346
|
+
if (!el?.state) return null
|
|
347
|
+
const stateValue = el.state[property as keyof UIElementState]
|
|
348
|
+
if (typeof stateValue === 'number' && Number.isFinite(stateValue)) return stateValue
|
|
349
|
+
if (property === 'value' || property === 'raw_value') {
|
|
350
|
+
const fallback = el.state.raw_value ?? el.state.value
|
|
351
|
+
if (typeof fallback === 'number' && Number.isFinite(fallback)) return fallback
|
|
352
|
+
}
|
|
353
|
+
return null
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
private static _buildControlPoint(bounds: [number, number, number, number], ratio: number, axis: 'horizontal' | 'vertical') {
|
|
357
|
+
const clampedRatio = Math.max(0, Math.min(1, ratio))
|
|
358
|
+
const [left, top, right, bottom] = bounds
|
|
359
|
+
const width = Math.max(1, right - left)
|
|
360
|
+
const height = Math.max(1, bottom - top)
|
|
361
|
+
const insetX = Math.max(8, Math.floor(width * 0.08))
|
|
362
|
+
const insetY = Math.max(8, Math.floor(height * 0.08))
|
|
363
|
+
if (axis === 'vertical') {
|
|
364
|
+
const usableHeight = Math.max(1, height - (insetY * 2))
|
|
365
|
+
return {
|
|
366
|
+
x: Math.floor((left + right) / 2),
|
|
367
|
+
y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
const usableWidth = Math.max(1, width - (insetX * 2))
|
|
371
|
+
return {
|
|
372
|
+
x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
|
|
373
|
+
y: Math.floor((top + bottom) / 2)
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
private static _buildConservativeControlPoint(
|
|
378
|
+
bounds: [number, number, number, number],
|
|
379
|
+
targetValue: number,
|
|
380
|
+
currentValue: number | null,
|
|
381
|
+
min: number,
|
|
382
|
+
max: number,
|
|
383
|
+
axis: 'horizontal' | 'vertical'
|
|
384
|
+
) {
|
|
385
|
+
const range = Math.max(1, max - min)
|
|
386
|
+
const targetRatio = (targetValue - min) / range
|
|
387
|
+
const stepRatio = 1 / range
|
|
388
|
+
const centerBias = stepRatio / 2
|
|
389
|
+
const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue)
|
|
390
|
+
const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0])
|
|
391
|
+
const edgeWindow = Math.max(3, Math.floor(range * 0.1))
|
|
392
|
+
const isNearLowEdge = targetValue - min <= edgeWindow
|
|
393
|
+
const isNearHighEdge = max - targetValue <= edgeWindow
|
|
394
|
+
const directionBias = direction > 0
|
|
395
|
+
? -stepRatio * 0.15
|
|
396
|
+
: direction < 0
|
|
397
|
+
? stepRatio * 0.65
|
|
398
|
+
: 0
|
|
399
|
+
const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx))
|
|
400
|
+
const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin)
|
|
401
|
+
const edgeBias = isNearLowEdge
|
|
402
|
+
? endpointMargin
|
|
403
|
+
: isNearHighEdge
|
|
404
|
+
? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
|
|
405
|
+
: 0
|
|
406
|
+
const safeRatio = Math.min(
|
|
407
|
+
1 - (endpointMargin * 0.25),
|
|
408
|
+
Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias)
|
|
409
|
+
)
|
|
410
|
+
return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
|
|
414
|
+
const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
|
|
415
|
+
const role = ToolsInteract._normalize(el.role ?? '')
|
|
416
|
+
if (/vertical/.test(type) || /vertical/.test(role)) return 'vertical'
|
|
417
|
+
if (/horizontal/.test(type) || /horizontal/.test(role)) return 'horizontal'
|
|
418
|
+
return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal'
|
|
419
|
+
}
|
|
420
|
+
|
|
337
421
|
private static _actionFailure(
|
|
338
422
|
actionType: string,
|
|
339
423
|
selector: Record<string, unknown> | null,
|
|
@@ -570,6 +654,507 @@ export class ToolsInteract {
|
|
|
570
654
|
})
|
|
571
655
|
}
|
|
572
656
|
|
|
657
|
+
static async adjustControlHandler({
|
|
658
|
+
selector,
|
|
659
|
+
element_id,
|
|
660
|
+
property = 'value',
|
|
661
|
+
targetValue,
|
|
662
|
+
tolerance = 0,
|
|
663
|
+
maxAttempts = 3,
|
|
664
|
+
platform,
|
|
665
|
+
deviceId
|
|
666
|
+
}: {
|
|
667
|
+
selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
|
|
668
|
+
element_id?: string,
|
|
669
|
+
property?: string,
|
|
670
|
+
targetValue: number,
|
|
671
|
+
tolerance?: number,
|
|
672
|
+
maxAttempts?: number,
|
|
673
|
+
platform?: 'android' | 'ios',
|
|
674
|
+
deviceId?: string
|
|
675
|
+
}): Promise<AdjustControlResponse> {
|
|
676
|
+
const actionType = 'adjust_control'
|
|
677
|
+
const targetSelector = selector ?? (element_id ? { elementId: element_id } : null)
|
|
678
|
+
const normalizedTolerance = Number.isFinite(tolerance) ? Math.max(0, tolerance) : 0
|
|
679
|
+
const attemptsLimit = Math.max(1, Math.floor(Number(maxAttempts) || 1))
|
|
680
|
+
const sourcePlatform: 'android' | 'ios' = platform || 'android'
|
|
681
|
+
let resolvedPlatform = sourcePlatform
|
|
682
|
+
let resolvedDeviceId = deviceId
|
|
683
|
+
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
684
|
+
let semanticFallbackElement: FindElementResponse['element'] | null = null
|
|
685
|
+
|
|
686
|
+
const buildFailure = (
|
|
687
|
+
failureCode: ActionFailureCode,
|
|
688
|
+
reason: string,
|
|
689
|
+
resolved: ActionTargetResolved | null,
|
|
690
|
+
device: any,
|
|
691
|
+
actualState: { property: string; value: number | null; raw_value?: number | null } | null,
|
|
692
|
+
attempts: number,
|
|
693
|
+
adjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture',
|
|
694
|
+
retryable = false,
|
|
695
|
+
uiFingerprintAfter: string | null = null
|
|
696
|
+
): AdjustControlResponse => {
|
|
697
|
+
const base = buildActionExecutionResult({
|
|
698
|
+
actionType,
|
|
699
|
+
sourceModule: 'interact',
|
|
700
|
+
device,
|
|
701
|
+
selector: targetSelector,
|
|
702
|
+
resolved,
|
|
703
|
+
success: false,
|
|
704
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
705
|
+
uiFingerprintAfter,
|
|
706
|
+
failure: { failureCode, retryable },
|
|
707
|
+
details: {
|
|
708
|
+
target_value: targetValue,
|
|
709
|
+
tolerance: normalizedTolerance,
|
|
710
|
+
property,
|
|
711
|
+
attempts,
|
|
712
|
+
adjustment_mode: adjustmentMode,
|
|
713
|
+
actual_state: actualState,
|
|
714
|
+
converged: false,
|
|
715
|
+
within_tolerance: false,
|
|
716
|
+
reason
|
|
717
|
+
}
|
|
718
|
+
}) as AdjustControlResponse
|
|
719
|
+
|
|
720
|
+
return {
|
|
721
|
+
...base,
|
|
722
|
+
target_state: {
|
|
723
|
+
property,
|
|
724
|
+
target_value: targetValue,
|
|
725
|
+
tolerance: normalizedTolerance
|
|
726
|
+
},
|
|
727
|
+
actual_state: actualState,
|
|
728
|
+
within_tolerance: false,
|
|
729
|
+
converged: false,
|
|
730
|
+
attempts,
|
|
731
|
+
adjustment_mode: adjustmentMode
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
const resolveCurrentMatch = async (): Promise<{
|
|
736
|
+
tree: any
|
|
737
|
+
device: any
|
|
738
|
+
match: { el: UiElement, idx: number } | null
|
|
739
|
+
resolvedTarget: ActionTargetResolved | null
|
|
740
|
+
} | null> => {
|
|
741
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
|
|
742
|
+
resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform
|
|
743
|
+
resolvedDeviceId = tree?.device?.id || resolvedDeviceId
|
|
744
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
745
|
+
|
|
746
|
+
if (element_id) {
|
|
747
|
+
const stored = ToolsInteract._resolvedUiElements.get(element_id)
|
|
748
|
+
if (!stored) {
|
|
749
|
+
return null
|
|
750
|
+
}
|
|
751
|
+
const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored)
|
|
752
|
+
if (!current) {
|
|
753
|
+
return null
|
|
754
|
+
}
|
|
755
|
+
return {
|
|
756
|
+
tree,
|
|
757
|
+
device: tree?.device,
|
|
758
|
+
match: { el: current.el, idx: current.index },
|
|
759
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
760
|
+
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index),
|
|
761
|
+
current.el,
|
|
762
|
+
current.index
|
|
763
|
+
)
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
if (semanticFallbackElement) {
|
|
768
|
+
const fallbackBounds = ToolsInteract._normalizeBounds(
|
|
769
|
+
Array.isArray(semanticFallbackElement.bounds)
|
|
770
|
+
? semanticFallbackElement.bounds
|
|
771
|
+
: semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
|
|
772
|
+
? [
|
|
773
|
+
Number((semanticFallbackElement.bounds as any).left),
|
|
774
|
+
Number((semanticFallbackElement.bounds as any).top),
|
|
775
|
+
Number((semanticFallbackElement.bounds as any).right),
|
|
776
|
+
Number((semanticFallbackElement.bounds as any).bottom)
|
|
777
|
+
]
|
|
778
|
+
: null
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
let matchedIndex = -1
|
|
782
|
+
if (fallbackBounds) {
|
|
783
|
+
matchedIndex = elements.findIndex((el) => {
|
|
784
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
785
|
+
return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3]
|
|
786
|
+
})
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
if (matchedIndex === -1 && fallbackBounds) {
|
|
790
|
+
const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2)
|
|
791
|
+
const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2)
|
|
792
|
+
let bestDistance = Infinity
|
|
793
|
+
for (let i = 0; i < elements.length; i++) {
|
|
794
|
+
const el = elements[i]
|
|
795
|
+
if (!ToolsInteract._isAdjustableControl(el)) continue
|
|
796
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
797
|
+
if (!bounds) continue
|
|
798
|
+
const centerX = Math.floor((bounds[0] + bounds[2]) / 2)
|
|
799
|
+
const centerY = Math.floor((bounds[1] + bounds[3]) / 2)
|
|
800
|
+
const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY)
|
|
801
|
+
if (distance < bestDistance) {
|
|
802
|
+
bestDistance = distance
|
|
803
|
+
matchedIndex = i
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
if (matchedIndex >= 0 && elements[matchedIndex]) {
|
|
809
|
+
const matched = { el: elements[matchedIndex], idx: matchedIndex }
|
|
810
|
+
return {
|
|
811
|
+
tree,
|
|
812
|
+
device: tree?.device,
|
|
813
|
+
match: matched,
|
|
814
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
815
|
+
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
|
|
816
|
+
matched.el,
|
|
817
|
+
matched.idx
|
|
818
|
+
)
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
if (selector) {
|
|
824
|
+
const matched = ToolsInteract._findFirstMatchingElement(elements, selector)
|
|
825
|
+
if (!matched) {
|
|
826
|
+
return null
|
|
827
|
+
}
|
|
828
|
+
return {
|
|
829
|
+
tree,
|
|
830
|
+
device: tree?.device,
|
|
831
|
+
match: matched,
|
|
832
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
833
|
+
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
|
|
834
|
+
matched.el,
|
|
835
|
+
matched.idx
|
|
836
|
+
)
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
return null
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
if (!selector && !element_id) {
|
|
844
|
+
return buildFailure('ELEMENT_NOT_FOUND', 'selector or element_id is required', null, undefined, null, 0, 'gesture', false)
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
if (selector && !element_id) {
|
|
848
|
+
const waitResult = await ToolsInteract.waitForUIHandler({
|
|
849
|
+
selector,
|
|
850
|
+
condition: 'clickable',
|
|
851
|
+
timeout_ms: 5000,
|
|
852
|
+
poll_interval_ms: 300,
|
|
853
|
+
platform: resolvedPlatform,
|
|
854
|
+
deviceId: resolvedDeviceId
|
|
855
|
+
}) as any
|
|
856
|
+
|
|
857
|
+
if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
|
|
858
|
+
const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
|
|
859
|
+
if (!semanticQuery) {
|
|
860
|
+
return buildFailure(
|
|
861
|
+
waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
|
|
862
|
+
waitResult?.error?.message ?? 'adjustable control not found',
|
|
863
|
+
null,
|
|
864
|
+
waitResult?.device,
|
|
865
|
+
null,
|
|
866
|
+
0,
|
|
867
|
+
'gesture',
|
|
868
|
+
waitResult?.error?.code === 'ELEMENT_NOT_FOUND'
|
|
869
|
+
)
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
const fallback = await ToolsInteract.findElementHandler({
|
|
873
|
+
query: semanticQuery,
|
|
874
|
+
exact: false,
|
|
875
|
+
timeoutMs: 3000,
|
|
876
|
+
platform: resolvedPlatform,
|
|
877
|
+
deviceId: resolvedDeviceId
|
|
878
|
+
})
|
|
879
|
+
|
|
880
|
+
if (!fallback.found || !fallback.element) {
|
|
881
|
+
return buildFailure(
|
|
882
|
+
'ELEMENT_NOT_FOUND',
|
|
883
|
+
waitResult?.error?.message ?? 'adjustable control not found',
|
|
884
|
+
null,
|
|
885
|
+
waitResult?.device,
|
|
886
|
+
null,
|
|
887
|
+
0,
|
|
888
|
+
'gesture',
|
|
889
|
+
true
|
|
890
|
+
)
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
semanticFallbackElement = fallback.element
|
|
894
|
+
} else {
|
|
895
|
+
element_id = waitResult.element.elementId
|
|
896
|
+
semanticFallbackElement = null
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
let lastObservedState: { property: string; value: number | null; raw_value?: number | null } | null = null
|
|
901
|
+
let lastAdjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture'
|
|
902
|
+
let resolvedTarget: ActionTargetResolved | null = null
|
|
903
|
+
let currentDevice: any = undefined
|
|
904
|
+
let attemptCount = 0
|
|
905
|
+
let cachedResolvedMatch: { el: UiElement, idx: number } | null = null
|
|
906
|
+
|
|
907
|
+
for (let attempt = 0; attempt < attemptsLimit; attempt++) {
|
|
908
|
+
const resolved: {
|
|
909
|
+
tree: any
|
|
910
|
+
device: any
|
|
911
|
+
match: { el: UiElement, idx: number } | null
|
|
912
|
+
resolvedTarget: ActionTargetResolved | null
|
|
913
|
+
} | null = cachedResolvedMatch
|
|
914
|
+
? {
|
|
915
|
+
tree: null,
|
|
916
|
+
device: currentDevice,
|
|
917
|
+
match: cachedResolvedMatch,
|
|
918
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(
|
|
919
|
+
ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx),
|
|
920
|
+
cachedResolvedMatch.el,
|
|
921
|
+
cachedResolvedMatch.idx
|
|
922
|
+
)
|
|
923
|
+
}
|
|
924
|
+
: await resolveCurrentMatch()
|
|
925
|
+
if (!resolved || !resolved.match || !resolved.resolvedTarget) {
|
|
926
|
+
return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true)
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
currentDevice = resolved.device
|
|
930
|
+
resolvedTarget = resolved.resolvedTarget
|
|
931
|
+
const currentEl: UiElement = resolved.match.el
|
|
932
|
+
cachedResolvedMatch = resolved.match
|
|
933
|
+
const bounds = ToolsInteract._normalizeBounds(currentEl.bounds)
|
|
934
|
+
const valueRange = currentEl.state?.value_range ?? null
|
|
935
|
+
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property)
|
|
936
|
+
const actualState = currentValue !== null
|
|
937
|
+
? { property, value: currentValue, raw_value: typeof currentEl.state?.raw_value === 'number' ? currentEl.state.raw_value : undefined }
|
|
938
|
+
: null
|
|
939
|
+
|
|
940
|
+
lastObservedState = actualState
|
|
941
|
+
|
|
942
|
+
if (property !== 'value' && property !== 'raw_value') {
|
|
943
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
|
|
947
|
+
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
948
|
+
const base = buildActionExecutionResult({
|
|
949
|
+
actionType,
|
|
950
|
+
sourceModule: 'interact',
|
|
951
|
+
device: currentDevice,
|
|
952
|
+
selector: targetSelector,
|
|
953
|
+
resolved: resolvedTarget,
|
|
954
|
+
success: true,
|
|
955
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
956
|
+
uiFingerprintAfter,
|
|
957
|
+
details: {
|
|
958
|
+
target_value: targetValue,
|
|
959
|
+
tolerance: normalizedTolerance,
|
|
960
|
+
property,
|
|
961
|
+
attempts: attemptCount,
|
|
962
|
+
adjustment_mode: 'semantic',
|
|
963
|
+
actual_state: actualState,
|
|
964
|
+
converged: true,
|
|
965
|
+
within_tolerance: true,
|
|
966
|
+
reason: 'control already within tolerance'
|
|
967
|
+
}
|
|
968
|
+
}) as AdjustControlResponse
|
|
969
|
+
|
|
970
|
+
return {
|
|
971
|
+
...base,
|
|
972
|
+
target_state: {
|
|
973
|
+
property,
|
|
974
|
+
target_value: targetValue,
|
|
975
|
+
tolerance: normalizedTolerance
|
|
976
|
+
},
|
|
977
|
+
actual_state: actualState,
|
|
978
|
+
within_tolerance: true,
|
|
979
|
+
converged: true,
|
|
980
|
+
attempts: attemptCount,
|
|
981
|
+
adjustment_mode: 'semantic'
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
if (!ToolsInteract._isAdjustableControl(currentEl)) {
|
|
986
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
if (!bounds) {
|
|
990
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
const min = typeof valueRange?.min === 'number' ? valueRange.min : null
|
|
994
|
+
const max = typeof valueRange?.max === 'number' ? valueRange.max : null
|
|
995
|
+
if (min === null || max === null || !Number.isFinite(min) || !Number.isFinite(max) || max <= min) {
|
|
996
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
if (targetValue < min || targetValue > max) {
|
|
1000
|
+
return buildFailure('UNKNOWN', `targetValue ${targetValue} is outside the control range ${min}..${max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
const axis = ToolsInteract._controlAxis(currentEl, bounds)
|
|
1004
|
+
const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
|
|
1005
|
+
const currentPoint = currentValue !== null
|
|
1006
|
+
? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
|
|
1007
|
+
: ToolsInteract._buildControlPoint(bounds, 0.5, axis)
|
|
1008
|
+
|
|
1009
|
+
const runVerification = async (): Promise<{
|
|
1010
|
+
verification: any
|
|
1011
|
+
observedState: { property: string; value: number | null; raw_value?: number | null } | null
|
|
1012
|
+
withinTolerance: boolean
|
|
1013
|
+
}> => {
|
|
1014
|
+
const verification = await ToolsInteract.expectStateHandler({
|
|
1015
|
+
element_id: resolvedTarget?.elementId ?? element_id,
|
|
1016
|
+
selector: selector ?? undefined,
|
|
1017
|
+
property,
|
|
1018
|
+
expected: targetValue,
|
|
1019
|
+
platform: resolvedPlatform,
|
|
1020
|
+
deviceId: resolvedDeviceId
|
|
1021
|
+
}) as any
|
|
1022
|
+
|
|
1023
|
+
const observedValue = typeof verification?.observed_state?.value === 'number'
|
|
1024
|
+
? verification.observed_state.value
|
|
1025
|
+
: typeof verification?.observed_state?.raw_value === 'number'
|
|
1026
|
+
? verification.observed_state.raw_value
|
|
1027
|
+
: null
|
|
1028
|
+
const observedState = observedValue !== null
|
|
1029
|
+
? {
|
|
1030
|
+
property,
|
|
1031
|
+
value: observedValue,
|
|
1032
|
+
raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
|
|
1033
|
+
}
|
|
1034
|
+
: actualState
|
|
1035
|
+
|
|
1036
|
+
return {
|
|
1037
|
+
verification,
|
|
1038
|
+
observedState,
|
|
1039
|
+
withinTolerance: observedValue !== null && Math.abs(observedValue - targetValue) <= normalizedTolerance
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
lastAdjustmentMode = 'coordinate'
|
|
1044
|
+
const primaryActionResult = await ToolsInteract.tapHandler({
|
|
1045
|
+
platform: resolvedPlatform,
|
|
1046
|
+
x: targetPoint.x,
|
|
1047
|
+
y: targetPoint.y,
|
|
1048
|
+
deviceId: resolvedDeviceId
|
|
1049
|
+
})
|
|
1050
|
+
let actionDevice = primaryActionResult.device ?? currentDevice
|
|
1051
|
+
attemptCount++
|
|
1052
|
+
|
|
1053
|
+
if (!primaryActionResult.success) {
|
|
1054
|
+
lastAdjustmentMode = 'gesture'
|
|
1055
|
+
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
1056
|
+
platform: resolvedPlatform,
|
|
1057
|
+
x1: currentPoint.x,
|
|
1058
|
+
y1: currentPoint.y,
|
|
1059
|
+
x2: targetPoint.x,
|
|
1060
|
+
y2: targetPoint.y,
|
|
1061
|
+
duration: 220,
|
|
1062
|
+
deviceId: resolvedDeviceId
|
|
1063
|
+
})
|
|
1064
|
+
attemptCount++
|
|
1065
|
+
|
|
1066
|
+
if (!fallbackActionResult.success) {
|
|
1067
|
+
return buildFailure('UNKNOWN', fallbackActionResult.error ?? primaryActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? primaryActionResult.device, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
actionDevice = fallbackActionResult.device ?? actionDevice
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
let verificationResult = await runVerification()
|
|
1074
|
+
let observedState = verificationResult.observedState
|
|
1075
|
+
lastObservedState = observedState
|
|
1076
|
+
|
|
1077
|
+
if (!verificationResult.withinTolerance && currentValue !== null) {
|
|
1078
|
+
lastAdjustmentMode = 'gesture'
|
|
1079
|
+
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
1080
|
+
platform: resolvedPlatform,
|
|
1081
|
+
x1: currentPoint.x,
|
|
1082
|
+
y1: currentPoint.y,
|
|
1083
|
+
x2: targetPoint.x,
|
|
1084
|
+
y2: targetPoint.y,
|
|
1085
|
+
duration: 220,
|
|
1086
|
+
deviceId: resolvedDeviceId
|
|
1087
|
+
})
|
|
1088
|
+
attemptCount++
|
|
1089
|
+
if (!fallbackActionResult.success) {
|
|
1090
|
+
return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
verificationResult = await runVerification()
|
|
1094
|
+
observedState = verificationResult.observedState
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
const verification = verificationResult.verification
|
|
1098
|
+
lastObservedState = observedState
|
|
1099
|
+
|
|
1100
|
+
if (verificationResult.withinTolerance) {
|
|
1101
|
+
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1102
|
+
const base = buildActionExecutionResult({
|
|
1103
|
+
actionType,
|
|
1104
|
+
sourceModule: 'interact',
|
|
1105
|
+
device: actionDevice ?? currentDevice,
|
|
1106
|
+
selector: targetSelector,
|
|
1107
|
+
resolved: resolvedTarget,
|
|
1108
|
+
success: true,
|
|
1109
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
1110
|
+
uiFingerprintAfter,
|
|
1111
|
+
details: {
|
|
1112
|
+
target_value: targetValue,
|
|
1113
|
+
tolerance: normalizedTolerance,
|
|
1114
|
+
property,
|
|
1115
|
+
attempts: attemptCount,
|
|
1116
|
+
adjustment_mode: lastAdjustmentMode,
|
|
1117
|
+
actual_state: observedState,
|
|
1118
|
+
converged: true,
|
|
1119
|
+
within_tolerance: true,
|
|
1120
|
+
reason: verification?.reason ?? 'control converged to target value'
|
|
1121
|
+
}
|
|
1122
|
+
}) as AdjustControlResponse
|
|
1123
|
+
|
|
1124
|
+
return {
|
|
1125
|
+
...base,
|
|
1126
|
+
target_state: {
|
|
1127
|
+
property,
|
|
1128
|
+
target_value: targetValue,
|
|
1129
|
+
tolerance: normalizedTolerance
|
|
1130
|
+
},
|
|
1131
|
+
actual_state: observedState,
|
|
1132
|
+
within_tolerance: true,
|
|
1133
|
+
converged: true,
|
|
1134
|
+
attempts: attemptCount,
|
|
1135
|
+
adjustment_mode: lastAdjustmentMode
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
cachedResolvedMatch = {
|
|
1140
|
+
el: {
|
|
1141
|
+
...currentEl,
|
|
1142
|
+
state: {
|
|
1143
|
+
...(currentEl.state ?? null),
|
|
1144
|
+
...(observedState ? {
|
|
1145
|
+
[observedState.property]: observedState.value,
|
|
1146
|
+
raw_value: observedState.raw_value ?? observedState.value
|
|
1147
|
+
} : {})
|
|
1148
|
+
}
|
|
1149
|
+
},
|
|
1150
|
+
idx: resolved.match.idx
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1155
|
+
return buildFailure('TIMEOUT', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter)
|
|
1156
|
+
}
|
|
1157
|
+
|
|
573
1158
|
static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string }) {
|
|
574
1159
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId)
|
|
575
1160
|
return await interact.swipe(x1, y1, x2, y2, duration, resolved.id)
|
package/src/observe/ios.ts
CHANGED
|
@@ -121,7 +121,7 @@ function buildIOSSemantic(type: string, traits: string[]): UIElementSemanticMeta
|
|
|
121
121
|
}
|
|
122
122
|
|
|
123
123
|
function isIOSAdjustable(node: IDBElement, type: string, traits: string[]): boolean {
|
|
124
|
-
return /slider|adjustable|stepper
|
|
124
|
+
return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait))
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
function extractIOSState(node: IDBElement, type: string, label: string | null, value: string | null, traits: string[]): UIElementState | null {
|
|
@@ -555,6 +555,62 @@ Failure Handling:
|
|
|
555
555
|
required: ['property', 'expected']
|
|
556
556
|
}
|
|
557
557
|
},
|
|
558
|
+
{
|
|
559
|
+
name: 'adjust_control',
|
|
560
|
+
description: `Purpose:
|
|
561
|
+
Adjust a numeric control value with verification.
|
|
562
|
+
|
|
563
|
+
This is the initial adjustable-control surface for slider-like controls and other controls that expose a numeric value or value_range.
|
|
564
|
+
|
|
565
|
+
Inputs:
|
|
566
|
+
- selector or element_id
|
|
567
|
+
- property (defaults to "value")
|
|
568
|
+
- targetValue
|
|
569
|
+
- tolerance (optional)
|
|
570
|
+
- maxAttempts (optional)
|
|
571
|
+
- platform/deviceId (optional)
|
|
572
|
+
|
|
573
|
+
Output Structure:
|
|
574
|
+
- action_id, timestamp (ISO 8601), action_type
|
|
575
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
576
|
+
- source_module: runtime source of the action envelope
|
|
577
|
+
- target_state / actual_state / within_tolerance / converged / attempts / adjustment_mode
|
|
578
|
+
- target.selector = original selector or element handle
|
|
579
|
+
- success = true when the control converges within tolerance
|
|
580
|
+
|
|
581
|
+
Verification Guidance:
|
|
582
|
+
- Prefer direct target placement when value_range is available; fall back to a drag only if the direct tap does not converge
|
|
583
|
+
- Use expect_state for the control value readback
|
|
584
|
+
- Treat coordinate fallback as degraded mode
|
|
585
|
+
|
|
586
|
+
Failure Handling:
|
|
587
|
+
- ELEMENT_NOT_FOUND → re-resolve the control
|
|
588
|
+
- ELEMENT_NOT_INTERACTABLE → the control cannot be adjusted through the current runtime
|
|
589
|
+
- TIMEOUT → the control did not converge within bounded retries
|
|
590
|
+
- UNKNOWN → capture a snapshot and stop`,
|
|
591
|
+
inputSchema: {
|
|
592
|
+
type: 'object',
|
|
593
|
+
properties: {
|
|
594
|
+
selector: {
|
|
595
|
+
type: 'object',
|
|
596
|
+
properties: {
|
|
597
|
+
text: { type: 'string' },
|
|
598
|
+
resource_id: { type: 'string' },
|
|
599
|
+
accessibility_id: { type: 'string' },
|
|
600
|
+
contains: { type: 'boolean', default: false }
|
|
601
|
+
}
|
|
602
|
+
},
|
|
603
|
+
element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
|
|
604
|
+
property: { type: 'string', description: 'Readable numeric state property to adjust.', default: 'value' },
|
|
605
|
+
targetValue: { type: 'number', description: 'Target numeric value.' },
|
|
606
|
+
tolerance: { type: 'number', description: 'Accepted numeric tolerance around the target value.', default: 0 },
|
|
607
|
+
maxAttempts: { type: 'number', description: 'Maximum adjustment attempts.', default: 3 },
|
|
608
|
+
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
|
|
609
|
+
deviceId: { type: 'string', description: 'Optional device serial/udid' }
|
|
610
|
+
},
|
|
611
|
+
required: ['targetValue']
|
|
612
|
+
}
|
|
613
|
+
},
|
|
558
614
|
{
|
|
559
615
|
name: 'wait_for_ui',
|
|
560
616
|
description: `Purpose:
|