mobile-debug-mcp 0.26.3 → 0.26.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +496 -7
- package/dist/observe/ios.js +48 -4
- package/dist/server/tool-definitions.js +56 -0
- package/dist/server/tool-handlers.js +25 -0
- package/dist/server-core.js +1 -1
- package/dist/utils/android/utils.js +37 -5
- package/docs/CHANGELOG.md +6 -0
- package/docs/ROADMAP.md +69 -14
- package/docs/rfcs/008-adjustable-control-support-and-semantic-value-manipulation.md +273 -0
- package/docs/rfcs/009-semantic-control-modeling-for-custom-and-composite-controls.md +238 -0
- package/docs/specs/mcp-tooling-spec-v1.md +23 -1
- package/docs/tools/interact.md +21 -0
- package/package.json +1 -1
- package/src/interact/index.ts +625 -8
- package/src/observe/ios.ts +43 -4
- package/src/server/tool-definitions.ts +56 -0
- package/src/server/tool-handlers.ts +26 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +21 -0
- package/src/utils/android/utils.ts +32 -5
- package/test/unit/interact/adjust_control.test.ts +365 -0
- package/test/unit/observe/find_element.test.ts +46 -0
- package/test/unit/observe/state_extraction.test.ts +89 -2
- package/test/unit/server/contract.test.ts +8 -0
- package/test/unit/server/response_shapes.test.ts +39 -0
package/src/observe/ios.ts
CHANGED
|
@@ -73,6 +73,9 @@ function normalizeIOSType(value: unknown): string {
|
|
|
73
73
|
|
|
74
74
|
function inferIOSRole(type: string, traits: string[]): string | null {
|
|
75
75
|
if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait))) return 'slider'
|
|
76
|
+
if (/stepper/.test(type)) return 'stepper'
|
|
77
|
+
if (/picker|pop up button|dropdown/.test(type)) return 'dropdown'
|
|
78
|
+
if (/segmented control/.test(type)) return 'segmented_control'
|
|
76
79
|
if (/button/.test(type) || traits.some((trait) => /button/.test(trait))) return 'button'
|
|
77
80
|
if (/cell/.test(type)) return 'cell'
|
|
78
81
|
if (/switch/.test(type)) return 'switch'
|
|
@@ -113,15 +116,51 @@ function buildIOSSelector(type: string, label: string | null, value: string | nu
|
|
|
113
116
|
return null
|
|
114
117
|
}
|
|
115
118
|
|
|
116
|
-
function buildIOSSemantic(type: string, traits: string[]): UIElementSemanticMetadata {
|
|
117
|
-
|
|
119
|
+
function buildIOSSemantic(type: string, traits: string[], role: string | null, value: string | null): UIElementSemanticMetadata {
|
|
120
|
+
const semantic: UIElementSemanticMetadata = {
|
|
118
121
|
is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
|
|
119
122
|
is_container: /window|application|group|scroll view|collection view/.test(type)
|
|
120
123
|
}
|
|
124
|
+
|
|
125
|
+
if (role === 'slider') {
|
|
126
|
+
semantic.semantic_role = 'slider'
|
|
127
|
+
semantic.adjustable = true
|
|
128
|
+
semantic.supported_actions = ['adjust']
|
|
129
|
+
semantic.state_shape = 'continuous'
|
|
130
|
+
} else if (role === 'stepper') {
|
|
131
|
+
semantic.semantic_role = 'stepper'
|
|
132
|
+
semantic.adjustable = true
|
|
133
|
+
semantic.supported_actions = ['increment', 'decrement']
|
|
134
|
+
semantic.state_shape = 'discrete'
|
|
135
|
+
} else if (role === 'dropdown') {
|
|
136
|
+
semantic.semantic_role = 'dropdown'
|
|
137
|
+
semantic.supported_actions = ['tap', 'expand']
|
|
138
|
+
semantic.state_shape = 'semantic'
|
|
139
|
+
} else if (role === 'segmented_control') {
|
|
140
|
+
semantic.semantic_role = 'segmented_control'
|
|
141
|
+
semantic.supported_actions = ['tap']
|
|
142
|
+
semantic.state_shape = 'discrete'
|
|
143
|
+
} else if (traits.some((trait) => /adjustable|slider/i.test(trait)) || /adjustable|slider/.test(type)) {
|
|
144
|
+
semantic.semantic_role = 'custom_adjustable'
|
|
145
|
+
semantic.adjustable = true
|
|
146
|
+
semantic.supported_actions = ['adjust']
|
|
147
|
+
semantic.state_shape = 'continuous'
|
|
148
|
+
} else if (semantic.is_clickable) {
|
|
149
|
+
semantic.supported_actions = ['tap']
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (semantic.state_shape === undefined && semantic.adjustable && value !== null) {
|
|
153
|
+
const numericValue = parseIOSNumber(value)
|
|
154
|
+
if (numericValue !== null && numericValue >= 0 && numericValue <= 1) {
|
|
155
|
+
semantic.state_shape = 'continuous'
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return semantic
|
|
121
160
|
}
|
|
122
161
|
|
|
123
162
|
function isIOSAdjustable(node: IDBElement, type: string, traits: string[]): boolean {
|
|
124
|
-
return /slider|adjustable|stepper
|
|
163
|
+
return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait))
|
|
125
164
|
}
|
|
126
165
|
|
|
127
166
|
function extractIOSState(node: IDBElement, type: string, label: string | null, value: string | null, traits: string[]): UIElementState | null {
|
|
@@ -184,8 +223,8 @@ export function traverseIDBNode(node: IDBElement, elements: UIElement[], parentI
|
|
|
184
223
|
const normalizedType = normalizeIOSType(type)
|
|
185
224
|
const stableId = getIOSStableId(node)
|
|
186
225
|
const selector = buildIOSSelector(type, label, value, stableId)
|
|
187
|
-
const semantic = buildIOSSemantic(normalizedType, traits)
|
|
188
226
|
const role = inferIOSRole(normalizedType, traits)
|
|
227
|
+
const semantic = buildIOSSemantic(normalizedType, traits, role, value)
|
|
189
228
|
|
|
190
229
|
const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
|
|
191
230
|
|
|
@@ -555,6 +555,62 @@ Failure Handling:
|
|
|
555
555
|
required: ['property', 'expected']
|
|
556
556
|
}
|
|
557
557
|
},
|
|
558
|
+
{
|
|
559
|
+
name: 'adjust_control',
|
|
560
|
+
description: `Purpose:
|
|
561
|
+
Adjust a numeric control value with verification.
|
|
562
|
+
|
|
563
|
+
This is the initial adjustable-control surface for slider-like controls and other controls that expose a numeric value or value_range.
|
|
564
|
+
|
|
565
|
+
Inputs:
|
|
566
|
+
- selector or element_id
|
|
567
|
+
- property (defaults to "value")
|
|
568
|
+
- targetValue
|
|
569
|
+
- tolerance (optional)
|
|
570
|
+
- maxAttempts (optional)
|
|
571
|
+
- platform/deviceId (optional)
|
|
572
|
+
|
|
573
|
+
Output Structure:
|
|
574
|
+
- action_id, timestamp (ISO 8601), action_type
|
|
575
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
576
|
+
- source_module: runtime source of the action envelope
|
|
577
|
+
- target_state / actual_state / within_tolerance / converged / attempts / adjustment_mode
|
|
578
|
+
- target.selector = original selector or element handle
|
|
579
|
+
- success = true when the control converges within tolerance
|
|
580
|
+
|
|
581
|
+
Verification Guidance:
|
|
582
|
+
- Prefer direct target placement when value_range is available; fall back to a drag only if the direct tap does not converge
|
|
583
|
+
- Use expect_state for the control value readback
|
|
584
|
+
- Treat coordinate fallback as degraded mode
|
|
585
|
+
|
|
586
|
+
Failure Handling:
|
|
587
|
+
- ELEMENT_NOT_FOUND → re-resolve the control
|
|
588
|
+
- ELEMENT_NOT_INTERACTABLE → the control cannot be adjusted through the current runtime
|
|
589
|
+
- TIMEOUT → the control did not converge within bounded retries
|
|
590
|
+
- UNKNOWN → capture a snapshot and stop`,
|
|
591
|
+
inputSchema: {
|
|
592
|
+
type: 'object',
|
|
593
|
+
properties: {
|
|
594
|
+
selector: {
|
|
595
|
+
type: 'object',
|
|
596
|
+
properties: {
|
|
597
|
+
text: { type: 'string' },
|
|
598
|
+
resource_id: { type: 'string' },
|
|
599
|
+
accessibility_id: { type: 'string' },
|
|
600
|
+
contains: { type: 'boolean', default: false }
|
|
601
|
+
}
|
|
602
|
+
},
|
|
603
|
+
element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
|
|
604
|
+
property: { type: 'string', description: 'Readable numeric state property to adjust.', default: 'value' },
|
|
605
|
+
targetValue: { type: 'number', description: 'Target numeric value.' },
|
|
606
|
+
tolerance: { type: 'number', description: 'Accepted numeric tolerance around the target value.', default: 0 },
|
|
607
|
+
maxAttempts: { type: 'number', description: 'Maximum adjustment attempts.', default: 3 },
|
|
608
|
+
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
|
|
609
|
+
deviceId: { type: 'string', description: 'Optional device serial/udid' }
|
|
610
|
+
},
|
|
611
|
+
required: ['targetValue']
|
|
612
|
+
}
|
|
613
|
+
},
|
|
558
614
|
{
|
|
559
615
|
name: 'wait_for_ui',
|
|
560
616
|
description: `Purpose:
|
|
@@ -277,6 +277,31 @@ async function handleExpectState(args: ToolCallArgs) {
|
|
|
277
277
|
return wrapResponse(res)
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
+
async function handleAdjustControl(args: ToolCallArgs) {
|
|
281
|
+
const selector = getObjectArg<ExpectElementSelectorArg>(args, 'selector')
|
|
282
|
+
const element_id = getStringArg(args, 'element_id')
|
|
283
|
+
const property = getStringArg(args, 'property') ?? 'value'
|
|
284
|
+
const targetValue = requireNumberArg(args, 'targetValue')
|
|
285
|
+
const tolerance = getNumberArg(args, 'tolerance') ?? 0
|
|
286
|
+
const maxAttempts = getNumberArg(args, 'maxAttempts') ?? 3
|
|
287
|
+
const platform = getStringArg(args, 'platform') as PlatformArg | undefined
|
|
288
|
+
const deviceId = getStringArg(args, 'deviceId')
|
|
289
|
+
if (!selector && !element_id) {
|
|
290
|
+
throw new Error('Missing selector or element_id argument')
|
|
291
|
+
}
|
|
292
|
+
const res = await ToolsInteract.adjustControlHandler({
|
|
293
|
+
selector: selector ?? undefined,
|
|
294
|
+
element_id: element_id ?? undefined,
|
|
295
|
+
property,
|
|
296
|
+
targetValue,
|
|
297
|
+
tolerance,
|
|
298
|
+
maxAttempts,
|
|
299
|
+
platform,
|
|
300
|
+
deviceId
|
|
301
|
+
})
|
|
302
|
+
return wrapResponse(res)
|
|
303
|
+
}
|
|
304
|
+
|
|
280
305
|
async function handleWaitForUI(args: ToolCallArgs) {
|
|
281
306
|
const selector = getObjectArg<ExpectElementSelectorArg>(args, 'selector')
|
|
282
307
|
const condition = (getStringArg(args, 'condition') as 'exists' | 'not_exists' | 'visible' | 'clickable' | undefined) ?? 'exists'
|
|
@@ -496,6 +521,7 @@ export const toolHandlers: Record<string, ToolHandler> = {
|
|
|
496
521
|
expect_screen: handleExpectScreen,
|
|
497
522
|
expect_element_visible: handleExpectElementVisible,
|
|
498
523
|
expect_state: handleExpectState,
|
|
524
|
+
adjust_control: handleAdjustControl,
|
|
499
525
|
wait_for_ui: handleWaitForUI,
|
|
500
526
|
find_element: handleFindElement,
|
|
501
527
|
tap: handleTap,
|
package/src/server-core.ts
CHANGED
package/src/types.ts
CHANGED
|
@@ -107,6 +107,10 @@ export interface UIResolutionSelector {
|
|
|
107
107
|
export interface UIElementSemanticMetadata {
|
|
108
108
|
is_clickable: boolean;
|
|
109
109
|
is_container: boolean;
|
|
110
|
+
semantic_role?: 'slider' | 'stepper' | 'dropdown' | 'segmented_control' | 'custom_adjustable' | 'composite_control' | null;
|
|
111
|
+
supported_actions?: string[] | null;
|
|
112
|
+
adjustable?: boolean | null;
|
|
113
|
+
state_shape?: 'continuous' | 'discrete' | 'semantic' | null;
|
|
110
114
|
}
|
|
111
115
|
|
|
112
116
|
export interface LoadingState {
|
|
@@ -413,6 +417,23 @@ export interface ExpectStateResponse {
|
|
|
413
417
|
retryable?: boolean;
|
|
414
418
|
}
|
|
415
419
|
|
|
420
|
+
export interface AdjustControlResponse extends ActionExecutionResult {
|
|
421
|
+
target_state: {
|
|
422
|
+
property: string;
|
|
423
|
+
target_value: number;
|
|
424
|
+
tolerance: number;
|
|
425
|
+
};
|
|
426
|
+
actual_state: {
|
|
427
|
+
property: string;
|
|
428
|
+
value: number | null;
|
|
429
|
+
raw_value?: number | null;
|
|
430
|
+
} | null;
|
|
431
|
+
within_tolerance: boolean;
|
|
432
|
+
converged: boolean;
|
|
433
|
+
attempts: number;
|
|
434
|
+
adjustment_mode: 'semantic' | 'gesture' | 'coordinate';
|
|
435
|
+
}
|
|
436
|
+
|
|
416
437
|
export interface WaitForUIChangeResponse {
|
|
417
438
|
success: boolean;
|
|
418
439
|
observed_change: 'hierarchy_diff' | 'text_change' | 'state_change' | null;
|
|
@@ -341,7 +341,10 @@ function normalizeClassName(value: unknown): string {
|
|
|
341
341
|
}
|
|
342
342
|
|
|
343
343
|
function inferAndroidRole(className: string): string | null {
|
|
344
|
-
if (/seekbar|slider
|
|
344
|
+
if (/seekbar|slider/.test(className)) return 'slider'
|
|
345
|
+
if (/stepper|numberpicker/.test(className)) return 'stepper'
|
|
346
|
+
if (/spinner|dropdown/.test(className)) return 'dropdown'
|
|
347
|
+
if (/segment|tablayout/.test(className)) return 'segmented_control'
|
|
345
348
|
if (/switch|toggle/.test(className)) return 'switch'
|
|
346
349
|
if (/checkbox/.test(className)) return 'checkbox'
|
|
347
350
|
if (/radiobutton|radio/.test(className)) return 'radio'
|
|
@@ -375,16 +378,40 @@ function buildAndroidSelector(text: string | null, contentDescription: string |
|
|
|
375
378
|
return null
|
|
376
379
|
}
|
|
377
380
|
|
|
378
|
-
function buildAndroidSemantic(clickable: boolean, className: string): UIElementSemanticMetadata {
|
|
379
|
-
|
|
381
|
+
function buildAndroidSemantic(clickable: boolean, className: string, role: string | null): UIElementSemanticMetadata {
|
|
382
|
+
const semantic: UIElementSemanticMetadata = {
|
|
380
383
|
is_clickable: clickable,
|
|
381
384
|
is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
|
|
382
385
|
}
|
|
386
|
+
|
|
387
|
+
if (role === 'slider') {
|
|
388
|
+
semantic.semantic_role = 'slider'
|
|
389
|
+
semantic.adjustable = true
|
|
390
|
+
semantic.supported_actions = ['adjust']
|
|
391
|
+
semantic.state_shape = 'continuous'
|
|
392
|
+
} else if (role === 'stepper') {
|
|
393
|
+
semantic.semantic_role = 'stepper'
|
|
394
|
+
semantic.adjustable = true
|
|
395
|
+
semantic.supported_actions = ['increment', 'decrement']
|
|
396
|
+
semantic.state_shape = 'discrete'
|
|
397
|
+
} else if (role === 'dropdown') {
|
|
398
|
+
semantic.semantic_role = 'dropdown'
|
|
399
|
+
semantic.supported_actions = ['tap', 'expand']
|
|
400
|
+
semantic.state_shape = 'semantic'
|
|
401
|
+
} else if (role === 'segmented_control') {
|
|
402
|
+
semantic.semantic_role = 'segmented_control'
|
|
403
|
+
semantic.supported_actions = ['tap']
|
|
404
|
+
semantic.state_shape = 'discrete'
|
|
405
|
+
} else if (clickable) {
|
|
406
|
+
semantic.supported_actions = ['tap']
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
return semantic
|
|
383
410
|
}
|
|
384
411
|
|
|
385
412
|
function isSliderLikeAndroid(node: any): boolean {
|
|
386
413
|
const className = String(node['@_class'] || '').toLowerCase()
|
|
387
|
-
return /seekbar|slider|range
|
|
414
|
+
return /seekbar|slider|range/i.test(className)
|
|
388
415
|
}
|
|
389
416
|
|
|
390
417
|
function extractAndroidState(node: any): UIElementState | null {
|
|
@@ -459,7 +486,7 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
|
|
|
459
486
|
const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null)
|
|
460
487
|
const testTag = stableId
|
|
461
488
|
const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className))
|
|
462
|
-
const semantic = buildAndroidSemantic(clickable, normalizeClassName(className))
|
|
489
|
+
const semantic = buildAndroidSemantic(clickable, normalizeClassName(className), role)
|
|
463
490
|
|
|
464
491
|
const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
|
|
465
492
|
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
import assert from 'assert'
|
|
2
|
+
import { ToolsInteract } from '../../../src/interact/index.js'
|
|
3
|
+
import * as Observe from '../../../src/observe/index.js'
|
|
4
|
+
|
|
5
|
+
async function run() {
|
|
6
|
+
console.log('Starting adjust_control unit tests...')
|
|
7
|
+
|
|
8
|
+
const originalGetUITreeHandler = (Observe as any).ToolsObserve.getUITreeHandler
|
|
9
|
+
const originalGetScreenFingerprintHandler = (Observe as any).ToolsObserve.getScreenFingerprintHandler
|
|
10
|
+
const originalTapHandler = (ToolsInteract as any).tapHandler
|
|
11
|
+
const originalSwipeHandler = (ToolsInteract as any).swipeHandler
|
|
12
|
+
const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
|
|
13
|
+
|
|
14
|
+
try {
|
|
15
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
16
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
17
|
+
screen: '',
|
|
18
|
+
resolution: { width: 1080, height: 2400 },
|
|
19
|
+
elements: [
|
|
20
|
+
{
|
|
21
|
+
text: 'Duration',
|
|
22
|
+
type: 'android.widget.SeekBar',
|
|
23
|
+
contentDescription: null,
|
|
24
|
+
clickable: true,
|
|
25
|
+
enabled: true,
|
|
26
|
+
visible: true,
|
|
27
|
+
bounds: [0, 0, 200, 40],
|
|
28
|
+
resourceId: 'seek_duration',
|
|
29
|
+
state: {
|
|
30
|
+
value: 10,
|
|
31
|
+
raw_value: 10,
|
|
32
|
+
value_range: { min: 0, max: 100 }
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
]
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => ({ fingerprint: 'fp_slider', activity: 'MainActivity' })
|
|
39
|
+
|
|
40
|
+
const wait = await ToolsInteract.waitForUIHandler({
|
|
41
|
+
selector: { text: 'Duration' },
|
|
42
|
+
condition: 'clickable',
|
|
43
|
+
timeout_ms: 200,
|
|
44
|
+
poll_interval_ms: 50,
|
|
45
|
+
platform: 'android'
|
|
46
|
+
})
|
|
47
|
+
assert.strictEqual(wait.status, 'success')
|
|
48
|
+
assert.ok(wait.element?.elementId)
|
|
49
|
+
|
|
50
|
+
const tapCalls: Array<{ platform?: string, x: number, y: number, deviceId?: string }> = []
|
|
51
|
+
const swipeCalls: Array<{ platform?: string, x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string }> = []
|
|
52
|
+
;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
|
|
53
|
+
tapCalls.push({ platform, x, y, deviceId })
|
|
54
|
+
return {
|
|
55
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
56
|
+
success: true,
|
|
57
|
+
x,
|
|
58
|
+
y
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
|
|
62
|
+
swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
|
|
63
|
+
return {
|
|
64
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
65
|
+
success: true,
|
|
66
|
+
start: [x1, y1],
|
|
67
|
+
end: [x2, y2],
|
|
68
|
+
duration
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
73
|
+
success: true,
|
|
74
|
+
selector: { text: 'Duration' },
|
|
75
|
+
element_id: wait.element.elementId,
|
|
76
|
+
expected_state: { property: 'value', expected: 30 },
|
|
77
|
+
element: {
|
|
78
|
+
elementId: wait.element.elementId,
|
|
79
|
+
text: 'Duration',
|
|
80
|
+
resource_id: 'seek_duration',
|
|
81
|
+
accessibility_id: null,
|
|
82
|
+
class: 'android.widget.SeekBar',
|
|
83
|
+
bounds: [0, 0, 200, 40],
|
|
84
|
+
index: 0,
|
|
85
|
+
state: { value: 30, raw_value: 30, value_range: { min: 0, max: 100 } }
|
|
86
|
+
},
|
|
87
|
+
observed_state: { property: 'value', value: 30, raw_value: 30 },
|
|
88
|
+
reason: 'value matches expected value'
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
const adjust = await ToolsInteract.adjustControlHandler({
|
|
92
|
+
element_id: wait.element.elementId,
|
|
93
|
+
property: 'value',
|
|
94
|
+
targetValue: 30,
|
|
95
|
+
tolerance: 0.5,
|
|
96
|
+
maxAttempts: 2,
|
|
97
|
+
platform: 'android'
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
assert.strictEqual(adjust.success, true)
|
|
101
|
+
assert.strictEqual(adjust.converged, true)
|
|
102
|
+
assert.strictEqual(adjust.within_tolerance, true)
|
|
103
|
+
assert.strictEqual(adjust.adjustment_mode, 'coordinate')
|
|
104
|
+
assert.strictEqual(adjust.target_state.target_value, 30)
|
|
105
|
+
assert.strictEqual(adjust.attempts, 1)
|
|
106
|
+
assert.strictEqual(tapCalls.length, 1)
|
|
107
|
+
assert.strictEqual(swipeCalls.length, 0)
|
|
108
|
+
assert.ok(tapCalls[0].x <= 66, 'tap should bias inward from the exact target point')
|
|
109
|
+
assert.strictEqual(adjust.action_type, 'adjust_control')
|
|
110
|
+
assert.strictEqual(adjust.target.selector.elementId, wait.element.elementId)
|
|
111
|
+
|
|
112
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
113
|
+
success: true,
|
|
114
|
+
selector: { text: 'Duration' },
|
|
115
|
+
element_id: wait.element.elementId,
|
|
116
|
+
expected_state: { property: 'value', expected: 2 },
|
|
117
|
+
element: {
|
|
118
|
+
elementId: wait.element.elementId,
|
|
119
|
+
text: 'Duration',
|
|
120
|
+
resource_id: 'seek_duration',
|
|
121
|
+
accessibility_id: null,
|
|
122
|
+
class: 'android.widget.SeekBar',
|
|
123
|
+
bounds: [0, 0, 200, 40],
|
|
124
|
+
index: 0,
|
|
125
|
+
state: { value: 2, raw_value: 2, value_range: { min: 0, max: 100 } }
|
|
126
|
+
},
|
|
127
|
+
observed_state: { property: 'value', value: 2, raw_value: 2 },
|
|
128
|
+
reason: 'value matches expected value'
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
const lowEndAdjust = await ToolsInteract.adjustControlHandler({
|
|
132
|
+
element_id: wait.element.elementId,
|
|
133
|
+
property: 'value',
|
|
134
|
+
targetValue: 2,
|
|
135
|
+
tolerance: 0.5,
|
|
136
|
+
maxAttempts: 2,
|
|
137
|
+
platform: 'android'
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
assert.strictEqual(lowEndAdjust.success, true)
|
|
141
|
+
assert.strictEqual(lowEndAdjust.converged, true)
|
|
142
|
+
assert.strictEqual(lowEndAdjust.within_tolerance, true)
|
|
143
|
+
assert.strictEqual(lowEndAdjust.attempts, 1)
|
|
144
|
+
assert.strictEqual(tapCalls.length, 2)
|
|
145
|
+
assert.strictEqual(swipeCalls.length, 0)
|
|
146
|
+
assert.ok(tapCalls[1].x >= 22, 'low-end tap should stay inside the first step instead of hugging the edge')
|
|
147
|
+
|
|
148
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
149
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
150
|
+
screen: '',
|
|
151
|
+
resolution: { width: 1080, height: 2400 },
|
|
152
|
+
elements: [
|
|
153
|
+
{
|
|
154
|
+
text: 'Duration',
|
|
155
|
+
type: 'android.widget.SeekBar',
|
|
156
|
+
contentDescription: null,
|
|
157
|
+
clickable: true,
|
|
158
|
+
enabled: true,
|
|
159
|
+
visible: true,
|
|
160
|
+
bounds: [0, 0, 200, 40],
|
|
161
|
+
resourceId: 'seek_duration',
|
|
162
|
+
state: {
|
|
163
|
+
value: 18,
|
|
164
|
+
raw_value: 18,
|
|
165
|
+
value_range: { min: 0, max: 20 }
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
]
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
172
|
+
success: true,
|
|
173
|
+
selector: { text: 'Duration' },
|
|
174
|
+
element_id: wait.element.elementId,
|
|
175
|
+
expected_state: { property: 'value', expected: 20 },
|
|
176
|
+
element: {
|
|
177
|
+
elementId: wait.element.elementId,
|
|
178
|
+
text: 'Duration',
|
|
179
|
+
resource_id: 'seek_duration',
|
|
180
|
+
accessibility_id: null,
|
|
181
|
+
class: 'android.widget.SeekBar',
|
|
182
|
+
bounds: [0, 0, 200, 40],
|
|
183
|
+
index: 0,
|
|
184
|
+
state: { value: 20, raw_value: 20, value_range: { min: 0, max: 20 } }
|
|
185
|
+
},
|
|
186
|
+
observed_state: { property: 'value', value: 20, raw_value: 20 },
|
|
187
|
+
reason: 'value matches expected value'
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
const highEndAdjust = await ToolsInteract.adjustControlHandler({
|
|
191
|
+
element_id: wait.element.elementId,
|
|
192
|
+
property: 'value',
|
|
193
|
+
targetValue: 20,
|
|
194
|
+
tolerance: 0.5,
|
|
195
|
+
maxAttempts: 2,
|
|
196
|
+
platform: 'android'
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
assert.strictEqual(highEndAdjust.success, true)
|
|
200
|
+
assert.strictEqual(highEndAdjust.converged, true)
|
|
201
|
+
assert.strictEqual(highEndAdjust.within_tolerance, true)
|
|
202
|
+
assert.strictEqual(highEndAdjust.attempts, 1)
|
|
203
|
+
assert.strictEqual(tapCalls.length, 3)
|
|
204
|
+
assert.strictEqual(swipeCalls.length, 0)
|
|
205
|
+
assert.ok(tapCalls[2].x >= 180, 'high-end tap should bias into the last step without hitting the edge')
|
|
206
|
+
|
|
207
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
208
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
209
|
+
screen: '',
|
|
210
|
+
resolution: { width: 1440, height: 3200 },
|
|
211
|
+
elements: [
|
|
212
|
+
{
|
|
213
|
+
text: 'Precision',
|
|
214
|
+
type: 'android.widget.SeekBar',
|
|
215
|
+
contentDescription: null,
|
|
216
|
+
clickable: true,
|
|
217
|
+
enabled: true,
|
|
218
|
+
visible: true,
|
|
219
|
+
bounds: [0, 0, 3000, 40],
|
|
220
|
+
resourceId: 'seek_precision',
|
|
221
|
+
state: {
|
|
222
|
+
value: 9000,
|
|
223
|
+
raw_value: 9000,
|
|
224
|
+
value_range: { min: 0, max: 10000 }
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
]
|
|
228
|
+
})
|
|
229
|
+
|
|
230
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
231
|
+
success: true,
|
|
232
|
+
selector: { text: 'Precision' },
|
|
233
|
+
element_id: wait.element.elementId,
|
|
234
|
+
expected_state: { property: 'value', expected: 9999 },
|
|
235
|
+
element: {
|
|
236
|
+
elementId: wait.element.elementId,
|
|
237
|
+
text: 'Precision',
|
|
238
|
+
resource_id: 'seek_precision',
|
|
239
|
+
accessibility_id: null,
|
|
240
|
+
class: 'android.widget.SeekBar',
|
|
241
|
+
bounds: [0, 0, 3000, 40],
|
|
242
|
+
index: 0,
|
|
243
|
+
state: { value: 9999, raw_value: 9999, value_range: { min: 0, max: 10000 } }
|
|
244
|
+
},
|
|
245
|
+
observed_state: { property: 'value', value: 9999, raw_value: 9999 },
|
|
246
|
+
reason: 'value matches expected value'
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
const precisionAdjust = await ToolsInteract.adjustControlHandler({
|
|
250
|
+
selector: { text: 'Precision' },
|
|
251
|
+
property: 'value',
|
|
252
|
+
targetValue: 9999,
|
|
253
|
+
tolerance: 0.5,
|
|
254
|
+
maxAttempts: 2,
|
|
255
|
+
platform: 'android'
|
|
256
|
+
})
|
|
257
|
+
|
|
258
|
+
assert.strictEqual(precisionAdjust.success, true)
|
|
259
|
+
assert.strictEqual(precisionAdjust.converged, true)
|
|
260
|
+
assert.strictEqual(precisionAdjust.within_tolerance, true)
|
|
261
|
+
assert.strictEqual(precisionAdjust.attempts, 1)
|
|
262
|
+
assert.strictEqual(tapCalls.length, 4)
|
|
263
|
+
assert.strictEqual(swipeCalls.length, 0)
|
|
264
|
+
assert.ok(tapCalls[3].x > 2750, 'wide, high-range control should not be clamped to a 3% endpoint margin')
|
|
265
|
+
|
|
266
|
+
let treeFetches = 0
|
|
267
|
+
let retryVerificationCount = 0
|
|
268
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
|
|
269
|
+
treeFetches++
|
|
270
|
+
return {
|
|
271
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
272
|
+
screen: '',
|
|
273
|
+
resolution: { width: 1080, height: 2400 },
|
|
274
|
+
elements: [
|
|
275
|
+
{
|
|
276
|
+
text: 'Duration',
|
|
277
|
+
type: 'android.widget.SeekBar',
|
|
278
|
+
contentDescription: null,
|
|
279
|
+
clickable: true,
|
|
280
|
+
enabled: true,
|
|
281
|
+
visible: true,
|
|
282
|
+
bounds: [0, 0, 200, 40],
|
|
283
|
+
resourceId: 'seek_duration',
|
|
284
|
+
state: {
|
|
285
|
+
value: 10,
|
|
286
|
+
raw_value: 10,
|
|
287
|
+
value_range: { min: 0, max: 20 }
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
]
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
|
|
295
|
+
tapCalls.push({ platform, x, y, deviceId })
|
|
296
|
+
return {
|
|
297
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
298
|
+
success: false,
|
|
299
|
+
error: 'tap failed'
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
|
|
304
|
+
swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
|
|
305
|
+
return {
|
|
306
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
307
|
+
success: true,
|
|
308
|
+
start: [x1, y1],
|
|
309
|
+
end: [x2, y2],
|
|
310
|
+
duration
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
;(ToolsInteract as any).expectStateHandler = async () => {
|
|
315
|
+
retryVerificationCount++
|
|
316
|
+
const value = retryVerificationCount === 1 ? 11 : 12
|
|
317
|
+
return {
|
|
318
|
+
success: true,
|
|
319
|
+
selector: { text: 'Duration' },
|
|
320
|
+
element_id: wait.element.elementId,
|
|
321
|
+
expected_state: { property: 'value', expected: 12 },
|
|
322
|
+
element: {
|
|
323
|
+
elementId: wait.element.elementId,
|
|
324
|
+
text: 'Duration',
|
|
325
|
+
resource_id: 'seek_duration',
|
|
326
|
+
accessibility_id: null,
|
|
327
|
+
class: 'android.widget.SeekBar',
|
|
328
|
+
bounds: [0, 0, 200, 40],
|
|
329
|
+
index: 0,
|
|
330
|
+
state: { value, raw_value: value, value_range: { min: 0, max: 20 } }
|
|
331
|
+
},
|
|
332
|
+
observed_state: { property: 'value', value, raw_value: value },
|
|
333
|
+
reason: value === 12 ? 'value matches expected value' : 'value still below target'
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const cachedResolveAdjust = await ToolsInteract.adjustControlHandler({
|
|
338
|
+
element_id: wait.element.elementId,
|
|
339
|
+
property: 'value',
|
|
340
|
+
targetValue: 12,
|
|
341
|
+
tolerance: 0.5,
|
|
342
|
+
maxAttempts: 2,
|
|
343
|
+
platform: 'android'
|
|
344
|
+
})
|
|
345
|
+
|
|
346
|
+
assert.strictEqual(cachedResolveAdjust.success, true)
|
|
347
|
+
assert.strictEqual(cachedResolveAdjust.converged, true)
|
|
348
|
+
assert.strictEqual(cachedResolveAdjust.within_tolerance, true)
|
|
349
|
+
assert.strictEqual(cachedResolveAdjust.attempts, 3)
|
|
350
|
+
assert.strictEqual(treeFetches, 1, 'second attempt should reuse the resolved element instead of refetching the UI tree')
|
|
351
|
+
|
|
352
|
+
console.log('adjust_control unit tests passed')
|
|
353
|
+
} finally {
|
|
354
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = originalGetUITreeHandler
|
|
355
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = originalGetScreenFingerprintHandler
|
|
356
|
+
;(ToolsInteract as any).tapHandler = originalTapHandler
|
|
357
|
+
;(ToolsInteract as any).swipeHandler = originalSwipeHandler
|
|
358
|
+
;(ToolsInteract as any).expectStateHandler = originalExpectStateHandler
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
run().catch((error) => {
|
|
363
|
+
console.error(error)
|
|
364
|
+
process.exit(1)
|
|
365
|
+
})
|