mobile-debug-mcp 0.26.3 → 0.26.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,7 @@ import { buildActionExecutionResult } from '../server/common.js'
10
10
  import type {
11
11
  ActionFailureCode,
12
12
  ActionTargetResolved,
13
+ AdjustControlResponse,
13
14
  FindElementResponse,
14
15
  ExpectElementVisibleResponse,
15
16
  ExpectStateResponse,
@@ -334,6 +335,89 @@ export class ToolsInteract {
334
335
  }
335
336
  }
336
337
 
338
+ private static _isAdjustableControl(el: UiElement | null): boolean {
339
+ if (!el) return false
340
+ const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
341
+ const role = ToolsInteract._normalize(el.role ?? '')
342
+ return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
343
+ }
344
+
345
+ private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
346
+ if (!el?.state) return null
347
+ const stateValue = el.state[property as keyof UIElementState]
348
+ if (typeof stateValue === 'number' && Number.isFinite(stateValue)) return stateValue
349
+ if (property === 'value' || property === 'raw_value') {
350
+ const fallback = el.state.raw_value ?? el.state.value
351
+ if (typeof fallback === 'number' && Number.isFinite(fallback)) return fallback
352
+ }
353
+ return null
354
+ }
355
+
356
+ private static _buildControlPoint(bounds: [number, number, number, number], ratio: number, axis: 'horizontal' | 'vertical') {
357
+ const clampedRatio = Math.max(0, Math.min(1, ratio))
358
+ const [left, top, right, bottom] = bounds
359
+ const width = Math.max(1, right - left)
360
+ const height = Math.max(1, bottom - top)
361
+ const insetX = Math.max(8, Math.floor(width * 0.08))
362
+ const insetY = Math.max(8, Math.floor(height * 0.08))
363
+ if (axis === 'vertical') {
364
+ const usableHeight = Math.max(1, height - (insetY * 2))
365
+ return {
366
+ x: Math.floor((left + right) / 2),
367
+ y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
368
+ }
369
+ }
370
+ const usableWidth = Math.max(1, width - (insetX * 2))
371
+ return {
372
+ x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
373
+ y: Math.floor((top + bottom) / 2)
374
+ }
375
+ }
376
+
377
+ private static _buildConservativeControlPoint(
378
+ bounds: [number, number, number, number],
379
+ targetValue: number,
380
+ currentValue: number | null,
381
+ min: number,
382
+ max: number,
383
+ axis: 'horizontal' | 'vertical'
384
+ ) {
385
+ const range = Math.max(1, max - min)
386
+ const targetRatio = (targetValue - min) / range
387
+ const stepRatio = 1 / range
388
+ const centerBias = stepRatio / 2
389
+ const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue)
390
+ const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0])
391
+ const edgeWindow = Math.max(3, Math.floor(range * 0.1))
392
+ const isNearLowEdge = targetValue - min <= edgeWindow
393
+ const isNearHighEdge = max - targetValue <= edgeWindow
394
+ const directionBias = direction > 0
395
+ ? -stepRatio * 0.15
396
+ : direction < 0
397
+ ? stepRatio * 0.65
398
+ : 0
399
+ const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx))
400
+ const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin)
401
+ const edgeBias = isNearLowEdge
402
+ ? endpointMargin
403
+ : isNearHighEdge
404
+ ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
405
+ : 0
406
+ const safeRatio = Math.min(
407
+ 1 - (endpointMargin * 0.25),
408
+ Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias)
409
+ )
410
+ return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
411
+ }
412
+
413
+ private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
414
+ const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
415
+ const role = ToolsInteract._normalize(el.role ?? '')
416
+ if (/vertical/.test(type) || /vertical/.test(role)) return 'vertical'
417
+ if (/horizontal/.test(type) || /horizontal/.test(role)) return 'horizontal'
418
+ return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal'
419
+ }
420
+
337
421
  private static _actionFailure(
338
422
  actionType: string,
339
423
  selector: Record<string, unknown> | null,
@@ -570,6 +654,507 @@ export class ToolsInteract {
570
654
  })
571
655
  }
572
656
 
657
+ static async adjustControlHandler({
658
+ selector,
659
+ element_id,
660
+ property = 'value',
661
+ targetValue,
662
+ tolerance = 0,
663
+ maxAttempts = 3,
664
+ platform,
665
+ deviceId
666
+ }: {
667
+ selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
668
+ element_id?: string,
669
+ property?: string,
670
+ targetValue: number,
671
+ tolerance?: number,
672
+ maxAttempts?: number,
673
+ platform?: 'android' | 'ios',
674
+ deviceId?: string
675
+ }): Promise<AdjustControlResponse> {
676
+ const actionType = 'adjust_control'
677
+ const targetSelector = selector ?? (element_id ? { elementId: element_id } : null)
678
+ const normalizedTolerance = Number.isFinite(tolerance) ? Math.max(0, tolerance) : 0
679
+ const attemptsLimit = Math.max(1, Math.floor(Number(maxAttempts) || 1))
680
+ const sourcePlatform: 'android' | 'ios' = platform || 'android'
681
+ let resolvedPlatform = sourcePlatform
682
+ let resolvedDeviceId = deviceId
683
+ const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
684
+ let semanticFallbackElement: FindElementResponse['element'] | null = null
685
+
686
+ const buildFailure = (
687
+ failureCode: ActionFailureCode,
688
+ reason: string,
689
+ resolved: ActionTargetResolved | null,
690
+ device: any,
691
+ actualState: { property: string; value: number | null; raw_value?: number | null } | null,
692
+ attempts: number,
693
+ adjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture',
694
+ retryable = false,
695
+ uiFingerprintAfter: string | null = null
696
+ ): AdjustControlResponse => {
697
+ const base = buildActionExecutionResult({
698
+ actionType,
699
+ sourceModule: 'interact',
700
+ device,
701
+ selector: targetSelector,
702
+ resolved,
703
+ success: false,
704
+ uiFingerprintBefore: fingerprintBefore,
705
+ uiFingerprintAfter,
706
+ failure: { failureCode, retryable },
707
+ details: {
708
+ target_value: targetValue,
709
+ tolerance: normalizedTolerance,
710
+ property,
711
+ attempts,
712
+ adjustment_mode: adjustmentMode,
713
+ actual_state: actualState,
714
+ converged: false,
715
+ within_tolerance: false,
716
+ reason
717
+ }
718
+ }) as AdjustControlResponse
719
+
720
+ return {
721
+ ...base,
722
+ target_state: {
723
+ property,
724
+ target_value: targetValue,
725
+ tolerance: normalizedTolerance
726
+ },
727
+ actual_state: actualState,
728
+ within_tolerance: false,
729
+ converged: false,
730
+ attempts,
731
+ adjustment_mode: adjustmentMode
732
+ }
733
+ }
734
+
735
+ const resolveCurrentMatch = async (): Promise<{
736
+ tree: any
737
+ device: any
738
+ match: { el: UiElement, idx: number } | null
739
+ resolvedTarget: ActionTargetResolved | null
740
+ } | null> => {
741
+ const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
742
+ resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform
743
+ resolvedDeviceId = tree?.device?.id || resolvedDeviceId
744
+ const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
745
+
746
+ if (element_id) {
747
+ const stored = ToolsInteract._resolvedUiElements.get(element_id)
748
+ if (!stored) {
749
+ return null
750
+ }
751
+ const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored)
752
+ if (!current) {
753
+ return null
754
+ }
755
+ return {
756
+ tree,
757
+ device: tree?.device,
758
+ match: { el: current.el, idx: current.index },
759
+ resolvedTarget: ToolsInteract._resolvedTargetFromElement(
760
+ ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index),
761
+ current.el,
762
+ current.index
763
+ )
764
+ }
765
+ }
766
+
767
+ if (semanticFallbackElement) {
768
+ const fallbackBounds = ToolsInteract._normalizeBounds(
769
+ Array.isArray(semanticFallbackElement.bounds)
770
+ ? semanticFallbackElement.bounds
771
+ : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
772
+ ? [
773
+ Number((semanticFallbackElement.bounds as any).left),
774
+ Number((semanticFallbackElement.bounds as any).top),
775
+ Number((semanticFallbackElement.bounds as any).right),
776
+ Number((semanticFallbackElement.bounds as any).bottom)
777
+ ]
778
+ : null
779
+ )
780
+
781
+ let matchedIndex = -1
782
+ if (fallbackBounds) {
783
+ matchedIndex = elements.findIndex((el) => {
784
+ const bounds = ToolsInteract._normalizeBounds(el.bounds)
785
+ return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3]
786
+ })
787
+ }
788
+
789
+ if (matchedIndex === -1 && fallbackBounds) {
790
+ const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2)
791
+ const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2)
792
+ let bestDistance = Infinity
793
+ for (let i = 0; i < elements.length; i++) {
794
+ const el = elements[i]
795
+ if (!ToolsInteract._isAdjustableControl(el)) continue
796
+ const bounds = ToolsInteract._normalizeBounds(el.bounds)
797
+ if (!bounds) continue
798
+ const centerX = Math.floor((bounds[0] + bounds[2]) / 2)
799
+ const centerY = Math.floor((bounds[1] + bounds[3]) / 2)
800
+ const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY)
801
+ if (distance < bestDistance) {
802
+ bestDistance = distance
803
+ matchedIndex = i
804
+ }
805
+ }
806
+ }
807
+
808
+ if (matchedIndex >= 0 && elements[matchedIndex]) {
809
+ const matched = { el: elements[matchedIndex], idx: matchedIndex }
810
+ return {
811
+ tree,
812
+ device: tree?.device,
813
+ match: matched,
814
+ resolvedTarget: ToolsInteract._resolvedTargetFromElement(
815
+ ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
816
+ matched.el,
817
+ matched.idx
818
+ )
819
+ }
820
+ }
821
+ }
822
+
823
+ if (selector) {
824
+ const matched = ToolsInteract._findFirstMatchingElement(elements, selector)
825
+ if (!matched) {
826
+ return null
827
+ }
828
+ return {
829
+ tree,
830
+ device: tree?.device,
831
+ match: matched,
832
+ resolvedTarget: ToolsInteract._resolvedTargetFromElement(
833
+ ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
834
+ matched.el,
835
+ matched.idx
836
+ )
837
+ }
838
+ }
839
+
840
+ return null
841
+ }
842
+
843
+ if (!selector && !element_id) {
844
+ return buildFailure('ELEMENT_NOT_FOUND', 'selector or element_id is required', null, undefined, null, 0, 'gesture', false)
845
+ }
846
+
847
+ if (selector && !element_id) {
848
+ const waitResult = await ToolsInteract.waitForUIHandler({
849
+ selector,
850
+ condition: 'clickable',
851
+ timeout_ms: 5000,
852
+ poll_interval_ms: 300,
853
+ platform: resolvedPlatform,
854
+ deviceId: resolvedDeviceId
855
+ }) as any
856
+
857
+ if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
858
+ const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
859
+ if (!semanticQuery) {
860
+ return buildFailure(
861
+ waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
862
+ waitResult?.error?.message ?? 'adjustable control not found',
863
+ null,
864
+ waitResult?.device,
865
+ null,
866
+ 0,
867
+ 'gesture',
868
+ waitResult?.error?.code === 'ELEMENT_NOT_FOUND'
869
+ )
870
+ }
871
+
872
+ const fallback = await ToolsInteract.findElementHandler({
873
+ query: semanticQuery,
874
+ exact: false,
875
+ timeoutMs: 3000,
876
+ platform: resolvedPlatform,
877
+ deviceId: resolvedDeviceId
878
+ })
879
+
880
+ if (!fallback.found || !fallback.element) {
881
+ return buildFailure(
882
+ 'ELEMENT_NOT_FOUND',
883
+ waitResult?.error?.message ?? 'adjustable control not found',
884
+ null,
885
+ waitResult?.device,
886
+ null,
887
+ 0,
888
+ 'gesture',
889
+ true
890
+ )
891
+ }
892
+
893
+ semanticFallbackElement = fallback.element
894
+ } else {
895
+ element_id = waitResult.element.elementId
896
+ semanticFallbackElement = null
897
+ }
898
+ }
899
+
900
+ let lastObservedState: { property: string; value: number | null; raw_value?: number | null } | null = null
901
+ let lastAdjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture'
902
+ let resolvedTarget: ActionTargetResolved | null = null
903
+ let currentDevice: any = undefined
904
+ let attemptCount = 0
905
+ let cachedResolvedMatch: { el: UiElement, idx: number } | null = null
906
+
907
+ for (let attempt = 0; attempt < attemptsLimit; attempt++) {
908
+ const resolved: {
909
+ tree: any
910
+ device: any
911
+ match: { el: UiElement, idx: number } | null
912
+ resolvedTarget: ActionTargetResolved | null
913
+ } | null = cachedResolvedMatch
914
+ ? {
915
+ tree: null,
916
+ device: currentDevice,
917
+ match: cachedResolvedMatch,
918
+ resolvedTarget: ToolsInteract._resolvedTargetFromElement(
919
+ ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx),
920
+ cachedResolvedMatch.el,
921
+ cachedResolvedMatch.idx
922
+ )
923
+ }
924
+ : await resolveCurrentMatch()
925
+ if (!resolved || !resolved.match || !resolved.resolvedTarget) {
926
+ return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true)
927
+ }
928
+
929
+ currentDevice = resolved.device
930
+ resolvedTarget = resolved.resolvedTarget
931
+ const currentEl: UiElement = resolved.match.el
932
+ cachedResolvedMatch = resolved.match
933
+ const bounds = ToolsInteract._normalizeBounds(currentEl.bounds)
934
+ const valueRange = currentEl.state?.value_range ?? null
935
+ const currentValue = ToolsInteract._readNumericControlValue(currentEl, property)
936
+ const actualState = currentValue !== null
937
+ ? { property, value: currentValue, raw_value: typeof currentEl.state?.raw_value === 'number' ? currentEl.state.raw_value : undefined }
938
+ : null
939
+
940
+ lastObservedState = actualState
941
+
942
+ if (property !== 'value' && property !== 'raw_value') {
943
+ return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
944
+ }
945
+
946
+ if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
947
+ const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
948
+ const base = buildActionExecutionResult({
949
+ actionType,
950
+ sourceModule: 'interact',
951
+ device: currentDevice,
952
+ selector: targetSelector,
953
+ resolved: resolvedTarget,
954
+ success: true,
955
+ uiFingerprintBefore: fingerprintBefore,
956
+ uiFingerprintAfter,
957
+ details: {
958
+ target_value: targetValue,
959
+ tolerance: normalizedTolerance,
960
+ property,
961
+ attempts: attemptCount,
962
+ adjustment_mode: 'semantic',
963
+ actual_state: actualState,
964
+ converged: true,
965
+ within_tolerance: true,
966
+ reason: 'control already within tolerance'
967
+ }
968
+ }) as AdjustControlResponse
969
+
970
+ return {
971
+ ...base,
972
+ target_state: {
973
+ property,
974
+ target_value: targetValue,
975
+ tolerance: normalizedTolerance
976
+ },
977
+ actual_state: actualState,
978
+ within_tolerance: true,
979
+ converged: true,
980
+ attempts: attemptCount,
981
+ adjustment_mode: 'semantic'
982
+ }
983
+ }
984
+
985
+ if (!ToolsInteract._isAdjustableControl(currentEl)) {
986
+ return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
987
+ }
988
+
989
+ if (!bounds) {
990
+ return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
991
+ }
992
+
993
+ const min = typeof valueRange?.min === 'number' ? valueRange.min : null
994
+ const max = typeof valueRange?.max === 'number' ? valueRange.max : null
995
+ if (min === null || max === null || !Number.isFinite(min) || !Number.isFinite(max) || max <= min) {
996
+ return buildFailure('ELEMENT_NOT_INTERACTABLE', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
997
+ }
998
+
999
+ if (targetValue < min || targetValue > max) {
1000
+ return buildFailure('UNKNOWN', `targetValue ${targetValue} is outside the control range ${min}..${max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
1001
+ }
1002
+
1003
+ const axis = ToolsInteract._controlAxis(currentEl, bounds)
1004
+ const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
1005
+ const currentPoint = currentValue !== null
1006
+ ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
1007
+ : ToolsInteract._buildControlPoint(bounds, 0.5, axis)
1008
+
1009
+ const runVerification = async (): Promise<{
1010
+ verification: any
1011
+ observedState: { property: string; value: number | null; raw_value?: number | null } | null
1012
+ withinTolerance: boolean
1013
+ }> => {
1014
+ const verification = await ToolsInteract.expectStateHandler({
1015
+ element_id: resolvedTarget?.elementId ?? element_id,
1016
+ selector: selector ?? undefined,
1017
+ property,
1018
+ expected: targetValue,
1019
+ platform: resolvedPlatform,
1020
+ deviceId: resolvedDeviceId
1021
+ }) as any
1022
+
1023
+ const observedValue = typeof verification?.observed_state?.value === 'number'
1024
+ ? verification.observed_state.value
1025
+ : typeof verification?.observed_state?.raw_value === 'number'
1026
+ ? verification.observed_state.raw_value
1027
+ : null
1028
+ const observedState = observedValue !== null
1029
+ ? {
1030
+ property,
1031
+ value: observedValue,
1032
+ raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
1033
+ }
1034
+ : actualState
1035
+
1036
+ return {
1037
+ verification,
1038
+ observedState,
1039
+ withinTolerance: observedValue !== null && Math.abs(observedValue - targetValue) <= normalizedTolerance
1040
+ }
1041
+ }
1042
+
1043
+ lastAdjustmentMode = 'coordinate'
1044
+ const primaryActionResult = await ToolsInteract.tapHandler({
1045
+ platform: resolvedPlatform,
1046
+ x: targetPoint.x,
1047
+ y: targetPoint.y,
1048
+ deviceId: resolvedDeviceId
1049
+ })
1050
+ let actionDevice = primaryActionResult.device ?? currentDevice
1051
+ attemptCount++
1052
+
1053
+ if (!primaryActionResult.success) {
1054
+ lastAdjustmentMode = 'gesture'
1055
+ const fallbackActionResult = await ToolsInteract.swipeHandler({
1056
+ platform: resolvedPlatform,
1057
+ x1: currentPoint.x,
1058
+ y1: currentPoint.y,
1059
+ x2: targetPoint.x,
1060
+ y2: targetPoint.y,
1061
+ duration: 220,
1062
+ deviceId: resolvedDeviceId
1063
+ })
1064
+ attemptCount++
1065
+
1066
+ if (!fallbackActionResult.success) {
1067
+ return buildFailure('UNKNOWN', fallbackActionResult.error ?? primaryActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? primaryActionResult.device, actualState, attemptCount, lastAdjustmentMode, false)
1068
+ }
1069
+
1070
+ actionDevice = fallbackActionResult.device ?? actionDevice
1071
+ }
1072
+
1073
+ let verificationResult = await runVerification()
1074
+ let observedState = verificationResult.observedState
1075
+ lastObservedState = observedState
1076
+
1077
+ if (!verificationResult.withinTolerance && currentValue !== null) {
1078
+ lastAdjustmentMode = 'gesture'
1079
+ const fallbackActionResult = await ToolsInteract.swipeHandler({
1080
+ platform: resolvedPlatform,
1081
+ x1: currentPoint.x,
1082
+ y1: currentPoint.y,
1083
+ x2: targetPoint.x,
1084
+ y2: targetPoint.y,
1085
+ duration: 220,
1086
+ deviceId: resolvedDeviceId
1087
+ })
1088
+ attemptCount++
1089
+ if (!fallbackActionResult.success) {
1090
+ return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
1091
+ }
1092
+
1093
+ verificationResult = await runVerification()
1094
+ observedState = verificationResult.observedState
1095
+ }
1096
+
1097
+ const verification = verificationResult.verification
1098
+ lastObservedState = observedState
1099
+
1100
+ if (verificationResult.withinTolerance) {
1101
+ const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1102
+ const base = buildActionExecutionResult({
1103
+ actionType,
1104
+ sourceModule: 'interact',
1105
+ device: actionDevice ?? currentDevice,
1106
+ selector: targetSelector,
1107
+ resolved: resolvedTarget,
1108
+ success: true,
1109
+ uiFingerprintBefore: fingerprintBefore,
1110
+ uiFingerprintAfter,
1111
+ details: {
1112
+ target_value: targetValue,
1113
+ tolerance: normalizedTolerance,
1114
+ property,
1115
+ attempts: attemptCount,
1116
+ adjustment_mode: lastAdjustmentMode,
1117
+ actual_state: observedState,
1118
+ converged: true,
1119
+ within_tolerance: true,
1120
+ reason: verification?.reason ?? 'control converged to target value'
1121
+ }
1122
+ }) as AdjustControlResponse
1123
+
1124
+ return {
1125
+ ...base,
1126
+ target_state: {
1127
+ property,
1128
+ target_value: targetValue,
1129
+ tolerance: normalizedTolerance
1130
+ },
1131
+ actual_state: observedState,
1132
+ within_tolerance: true,
1133
+ converged: true,
1134
+ attempts: attemptCount,
1135
+ adjustment_mode: lastAdjustmentMode
1136
+ }
1137
+ }
1138
+
1139
+ cachedResolvedMatch = {
1140
+ el: {
1141
+ ...currentEl,
1142
+ state: {
1143
+ ...(currentEl.state ?? null),
1144
+ ...(observedState ? {
1145
+ [observedState.property]: observedState.value,
1146
+ raw_value: observedState.raw_value ?? observedState.value
1147
+ } : {})
1148
+ }
1149
+ },
1150
+ idx: resolved.match.idx
1151
+ }
1152
+ }
1153
+
1154
+ const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1155
+ return buildFailure('TIMEOUT', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter)
1156
+ }
1157
+
573
1158
  static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string }) {
574
1159
  const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId)
575
1160
  return await interact.swipe(x1, y1, x2, y2, duration, resolved.id)
@@ -121,7 +121,7 @@ function buildIOSSemantic(type: string, traits: string[]): UIElementSemanticMeta
121
121
  }
122
122
 
123
123
  function isIOSAdjustable(node: IDBElement, type: string, traits: string[]): boolean {
124
- return /slider|adjustable|stepper|progress/i.test(type) || traits.some((trait) => /adjustable|slider|progress/i.test(trait))
124
+ return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait))
125
125
  }
126
126
 
127
127
  function extractIOSState(node: IDBElement, type: string, label: string | null, value: string | null, traits: string[]): UIElementState | null {
@@ -555,6 +555,62 @@ Failure Handling:
555
555
  required: ['property', 'expected']
556
556
  }
557
557
  },
558
+ {
559
+ name: 'adjust_control',
560
+ description: `Purpose:
561
+ Adjust a numeric control value with verification.
562
+
563
+ This is the initial adjustable-control surface for slider-like controls and other controls that expose a numeric value or value_range.
564
+
565
+ Inputs:
566
+ - selector or element_id
567
+ - property (defaults to "value")
568
+ - targetValue
569
+ - tolerance (optional)
570
+ - maxAttempts (optional)
571
+ - platform/deviceId (optional)
572
+
573
+ Output Structure:
574
+ - action_id, timestamp (ISO 8601), action_type
575
+ - lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
576
+ - source_module: runtime source of the action envelope
577
+ - target_state / actual_state / within_tolerance / converged / attempts / adjustment_mode
578
+ - target.selector = original selector or element handle
579
+ - success = true when the control converges within tolerance
580
+
581
+ Verification Guidance:
582
+ - Prefer direct target placement when value_range is available; fall back to a drag only if the direct tap does not converge
583
+ - Use expect_state for the control value readback
584
+ - Treat coordinate fallback as degraded mode
585
+
586
+ Failure Handling:
587
+ - ELEMENT_NOT_FOUND → re-resolve the control
588
+ - ELEMENT_NOT_INTERACTABLE → the control cannot be adjusted through the current runtime
589
+ - TIMEOUT → the control did not converge within bounded retries
590
+ - UNKNOWN → capture a snapshot and stop`,
591
+ inputSchema: {
592
+ type: 'object',
593
+ properties: {
594
+ selector: {
595
+ type: 'object',
596
+ properties: {
597
+ text: { type: 'string' },
598
+ resource_id: { type: 'string' },
599
+ accessibility_id: { type: 'string' },
600
+ contains: { type: 'boolean', default: false }
601
+ }
602
+ },
603
+ element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
604
+ property: { type: 'string', description: 'Readable numeric state property to adjust.', default: 'value' },
605
+ targetValue: { type: 'number', description: 'Target numeric value.' },
606
+ tolerance: { type: 'number', description: 'Accepted numeric tolerance around the target value.', default: 0 },
607
+ maxAttempts: { type: 'number', description: 'Maximum adjustment attempts.', default: 3 },
608
+ platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
609
+ deviceId: { type: 'string', description: 'Optional device serial/udid' }
610
+ },
611
+ required: ['targetValue']
612
+ }
613
+ },
558
614
  {
559
615
  name: 'wait_for_ui',
560
616
  description: `Purpose: