mobile-debug-mcp 0.30.1 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -5
- package/dist/interact/index.js +546 -133
- package/dist/manage/ios.js +1 -1
- package/dist/server/tool-definitions.js +13 -9
- package/dist/server/tool-handlers.js +4 -4
- package/dist/server-core.js +1 -1
- package/dist/system/index.js +485 -48
- package/dist/system/ios.js +15 -7
- package/dist/utils/android/utils.js +4 -4
- package/dist/utils/cli/idb/check-idb.js +10 -3
- package/dist/utils/cli/idb/idb-helper.js +47 -13
- package/dist/utils/ios/utils.js +5 -9
- package/docs/CHANGELOG.md +6 -0
- package/docs/ROADMAP.md +88 -212
- package/docs/specs/baseline-spec-v0.md +1 -1
- package/docs/specs-deltas/015-adjustable-control-support.md +541 -0
- package/docs/specs-deltas/016-environment-tools.md +533 -0
- package/docs/tools/TOOLS.md +2 -0
- package/docs/tools/manage.md +4 -2
- package/docs/tools/system.md +9 -2
- package/package.json +1 -1
- package/skills/spec-review/SKILL.md +52 -0
- package/skills/spec-review/references/spec-review-checklist.md +13 -0
- package/skills/spec-review/references/spec-review-template.md +28 -0
- package/src/interact/index.ts +581 -138
- package/src/manage/ios.ts +1 -1
- package/src/server/tool-definitions.ts +13 -9
- package/src/server/tool-handlers.ts +4 -4
- package/src/server-core.ts +1 -1
- package/src/system/index.ts +572 -50
- package/src/system/ios.ts +14 -7
- package/src/types.ts +2 -0
- package/src/utils/android/utils.ts +3 -3
- package/src/utils/cli/idb/check-idb.ts +11 -2
- package/src/utils/cli/idb/idb-helper.ts +49 -12
- package/src/utils/ios/utils.ts +4 -8
- package/test/unit/interact/adjust_control.test.ts +586 -33
- package/test/unit/interact/tap_element.test.ts +1 -1
- package/test/unit/server/contract.test.ts +4 -0
- package/test/unit/system/get_system_status.test.ts +3 -0
- package/test/unit/system/system_status.test.ts +40 -0
- package/test/unit/utils/idb_health.test.ts +74 -0
- package/skills/rfc-review/SKILL.md +0 -52
- package/skills/rfc-review/references/rfc-review-checklist.md +0 -12
- package/skills/rfc-review/references/rfc-review-template.md +0 -28
- /package/docs/{rfcs → specs-deltas}/001-state-verification.md +0 -0
- /package/docs/{rfcs → specs-deltas}/002-richer-element-identity +0 -0
- /package/docs/{rfcs → specs-deltas}/003-wait-and-synchronization-reliability.md +0 -0
- /package/docs/{rfcs → specs-deltas}/004-action-verification-routing.md +0 -0
- /package/docs/{rfcs → specs-deltas}/005-unified-action-execution-and-verification-model.md +0 -0
- /package/docs/{rfcs → specs-deltas}/006-runtime-action-instrumentation-and-binding-layer.md +0 -0
- /package/docs/{rfcs → specs-deltas}/007-actionability-resolution-and-executable-target-selection.md +0 -0
- /package/docs/{rfcs → specs-deltas}/008-adjustable-control-support-and-semantic-value-manipulation.md +0 -0
- /package/docs/{rfcs → specs-deltas}/009-semantic-control-modeling-for-custom-and-composite-controls.md +0 -0
- /package/docs/{rfcs → specs-deltas}/010-verification-stabilization-and-temporal-convergence.md +0 -0
- /package/docs/{rfcs → specs-deltas}/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows.md +0 -0
- /package/docs/{rfcs → specs-deltas}/011.1-recovery-contract-types-and-runtime-wiring-spec.md +0 -0
- /package/docs/{rfcs → specs-deltas}/012-action-trace-and-xecution-observability.md +0 -0
- /package/docs/{rfcs → specs-deltas}/013-wait-and-synchronization-reliability.md +0 -0
- /package/docs/{rfcs → specs-deltas}/014-actionability-resolution.md +0 -0
|
@@ -10,6 +10,7 @@ async function run() {
|
|
|
10
10
|
const originalTapHandler = (ToolsInteract as any).tapHandler
|
|
11
11
|
const originalSwipeHandler = (ToolsInteract as any).swipeHandler
|
|
12
12
|
const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
|
|
13
|
+
const originalGetInteractionService = (ToolsInteract as any).getInteractionService
|
|
13
14
|
|
|
14
15
|
try {
|
|
15
16
|
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
@@ -298,6 +299,253 @@ async function run() {
|
|
|
298
299
|
assert.strictEqual(disabledAdjust.success, false)
|
|
299
300
|
assert.strictEqual(disabledAdjust.failure_code, 'ELEMENT_NOT_INTERACTABLE')
|
|
300
301
|
|
|
302
|
+
const bothTargetsAdjust = await ToolsInteract.adjustControlHandler({
|
|
303
|
+
selector: { text: 'Duration' },
|
|
304
|
+
element_id: wait.element.elementId,
|
|
305
|
+
property: 'value',
|
|
306
|
+
targetValue: 8,
|
|
307
|
+
platform: 'android'
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
assert.strictEqual(bothTargetsAdjust.success, false)
|
|
311
|
+
assert.strictEqual(bothTargetsAdjust.failure_code, 'ELEMENT_NOT_FOUND')
|
|
312
|
+
|
|
313
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
314
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
315
|
+
screen: '',
|
|
316
|
+
resolution: { width: 1080, height: 2400 },
|
|
317
|
+
elements: [
|
|
318
|
+
{
|
|
319
|
+
text: 'Plain button',
|
|
320
|
+
type: 'android.widget.Button',
|
|
321
|
+
contentDescription: null,
|
|
322
|
+
clickable: true,
|
|
323
|
+
enabled: true,
|
|
324
|
+
visible: true,
|
|
325
|
+
bounds: [0, 0, 200, 40],
|
|
326
|
+
resourceId: 'plain_button',
|
|
327
|
+
state: null
|
|
328
|
+
}
|
|
329
|
+
]
|
|
330
|
+
})
|
|
331
|
+
|
|
332
|
+
const nonAdjustable = await ToolsInteract.adjustControlHandler({
|
|
333
|
+
selector: { text: 'Plain button' },
|
|
334
|
+
property: 'value',
|
|
335
|
+
targetValue: 1,
|
|
336
|
+
platform: 'android'
|
|
337
|
+
})
|
|
338
|
+
|
|
339
|
+
assert.strictEqual(nonAdjustable.success, false)
|
|
340
|
+
assert.strictEqual(nonAdjustable.failure_code, 'ELEMENT_NOT_INTERACTABLE')
|
|
341
|
+
|
|
342
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
343
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
344
|
+
screen: '',
|
|
345
|
+
resolution: { width: 1080, height: 2400 },
|
|
346
|
+
elements: [
|
|
347
|
+
{
|
|
348
|
+
text: 'Direct slider',
|
|
349
|
+
type: 'android.widget.SeekBar',
|
|
350
|
+
contentDescription: null,
|
|
351
|
+
clickable: true,
|
|
352
|
+
enabled: true,
|
|
353
|
+
visible: true,
|
|
354
|
+
bounds: [0, 0, 200, 40],
|
|
355
|
+
resourceId: 'seek_direct',
|
|
356
|
+
actions: ['ACTION_SET_PROGRESS'],
|
|
357
|
+
state: {
|
|
358
|
+
value: 10,
|
|
359
|
+
raw_value: 10,
|
|
360
|
+
value_range: { min: 0, max: 100 }
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
]
|
|
364
|
+
})
|
|
365
|
+
|
|
366
|
+
let directCalls = 0
|
|
367
|
+
;(ToolsInteract as any).getInteractionService = async () => ({
|
|
368
|
+
resolved: { id: 'mock-device' },
|
|
369
|
+
interact: {
|
|
370
|
+
setAdjustableValue: async ({ value }: any) => {
|
|
371
|
+
directCalls++
|
|
372
|
+
assert.strictEqual(value, 42)
|
|
373
|
+
return {
|
|
374
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
375
|
+
success: true
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
},
|
|
379
|
+
platform: 'android'
|
|
380
|
+
})
|
|
381
|
+
|
|
382
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
383
|
+
success: true,
|
|
384
|
+
selector: { text: 'Direct slider' },
|
|
385
|
+
element_id: null,
|
|
386
|
+
expected_state: { property: 'value', expected: 42 },
|
|
387
|
+
observed_state: { property: 'value', value: 42, raw_value: 42 },
|
|
388
|
+
reason: 'value matches expected value'
|
|
389
|
+
})
|
|
390
|
+
|
|
391
|
+
const directAdjust = await ToolsInteract.adjustControlHandler({
|
|
392
|
+
selector: { text: 'Direct slider' },
|
|
393
|
+
property: 'value',
|
|
394
|
+
targetValue: 42,
|
|
395
|
+
platform: 'android'
|
|
396
|
+
})
|
|
397
|
+
|
|
398
|
+
assert.strictEqual(directAdjust.success, true)
|
|
399
|
+
assert.strictEqual(directAdjust.adjustment_mode, 'semantic')
|
|
400
|
+
assert.strictEqual(directAdjust.attempts, 1)
|
|
401
|
+
assert.strictEqual(directCalls, 1)
|
|
402
|
+
|
|
403
|
+
let directRefreshValue = 10
|
|
404
|
+
let directRefreshTreeFetches = 0
|
|
405
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
|
|
406
|
+
directRefreshTreeFetches++
|
|
407
|
+
return {
|
|
408
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
409
|
+
screen: '',
|
|
410
|
+
resolution: { width: 1080, height: 2400 },
|
|
411
|
+
elements: [
|
|
412
|
+
{
|
|
413
|
+
text: 'Retrying direct slider',
|
|
414
|
+
type: 'android.widget.SeekBar',
|
|
415
|
+
contentDescription: null,
|
|
416
|
+
clickable: true,
|
|
417
|
+
enabled: true,
|
|
418
|
+
visible: true,
|
|
419
|
+
bounds: [0, 0, 200, 40],
|
|
420
|
+
resourceId: 'seek_direct_retry',
|
|
421
|
+
actions: ['ACTION_SET_PROGRESS'],
|
|
422
|
+
state: {
|
|
423
|
+
value: directRefreshValue,
|
|
424
|
+
raw_value: directRefreshValue,
|
|
425
|
+
value_range: { min: 0, max: 100 }
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
]
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
let retryingDirectCalls = 0
|
|
433
|
+
;(ToolsInteract as any).getInteractionService = async () => ({
|
|
434
|
+
resolved: { id: 'mock-device' },
|
|
435
|
+
interact: {
|
|
436
|
+
setAdjustableValue: async () => {
|
|
437
|
+
retryingDirectCalls++
|
|
438
|
+
directRefreshValue = retryingDirectCalls === 1 ? 60 : 80
|
|
439
|
+
return {
|
|
440
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
441
|
+
success: true
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
},
|
|
445
|
+
platform: 'android'
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
;(ToolsInteract as any).tapHandler = async () => {
|
|
449
|
+
throw new Error('coordinate fallback should not run before direct retry refreshes state')
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
453
|
+
success: directRefreshValue === 80,
|
|
454
|
+
selector: { text: 'Retrying direct slider' },
|
|
455
|
+
element_id: null,
|
|
456
|
+
expected_state: { property: 'value', expected: 80 },
|
|
457
|
+
observed_state: { property: 'value', value: directRefreshValue, raw_value: directRefreshValue },
|
|
458
|
+
reason: directRefreshValue === 80 ? 'value matches expected value' : 'direct adjustment partially applied'
|
|
459
|
+
})
|
|
460
|
+
|
|
461
|
+
const retryingDirectAdjust = await ToolsInteract.adjustControlHandler({
|
|
462
|
+
selector: { text: 'Retrying direct slider' },
|
|
463
|
+
property: 'value',
|
|
464
|
+
targetValue: 80,
|
|
465
|
+
maxAttempts: 2,
|
|
466
|
+
platform: 'android'
|
|
467
|
+
})
|
|
468
|
+
|
|
469
|
+
assert.strictEqual(retryingDirectAdjust.success, true)
|
|
470
|
+
assert.strictEqual(retryingDirectAdjust.adjustment_mode, 'semantic')
|
|
471
|
+
assert.strictEqual(retryingDirectAdjust.attempts, 2)
|
|
472
|
+
assert.strictEqual(retryingDirectCalls, 2)
|
|
473
|
+
assert.ok(directRefreshTreeFetches >= 2, 'direct retry should refetch the UI tree instead of cascading into stale coordinate fallback')
|
|
474
|
+
|
|
475
|
+
let incrementalTreeValue = 1
|
|
476
|
+
let incrementalTreeFetches = 0
|
|
477
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
|
|
478
|
+
incrementalTreeFetches++
|
|
479
|
+
return {
|
|
480
|
+
device: { platform: 'ios', id: 'booted', osVersion: '17', model: 'Simulator', simulator: true },
|
|
481
|
+
screen: '',
|
|
482
|
+
resolution: { width: 390, height: 844 },
|
|
483
|
+
elements: [
|
|
484
|
+
{
|
|
485
|
+
text: 'Stepper',
|
|
486
|
+
type: 'XCUIElementTypeStepper',
|
|
487
|
+
contentDescription: null,
|
|
488
|
+
clickable: true,
|
|
489
|
+
enabled: true,
|
|
490
|
+
visible: true,
|
|
491
|
+
bounds: [0, 0, 120, 44],
|
|
492
|
+
resourceId: 'stepper',
|
|
493
|
+
actions: ['increment', 'decrement'],
|
|
494
|
+
state: {
|
|
495
|
+
value: incrementalTreeValue,
|
|
496
|
+
raw_value: incrementalTreeValue,
|
|
497
|
+
value_range: { min: 0, max: 5, step: 1 },
|
|
498
|
+
step: 1
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
]
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
const directions: string[] = []
|
|
506
|
+
;(ToolsInteract as any).getInteractionService = async () => ({
|
|
507
|
+
resolved: { id: 'booted' },
|
|
508
|
+
interact: {
|
|
509
|
+
adjustAccessibleValue: async ({ direction }: any) => {
|
|
510
|
+
directions.push(direction)
|
|
511
|
+
incrementalTreeValue += direction === 'increment' ? 1 : -1
|
|
512
|
+
return {
|
|
513
|
+
device: { platform: 'ios', id: 'booted', osVersion: '17', model: 'Simulator', simulator: true },
|
|
514
|
+
success: true
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
},
|
|
518
|
+
platform: 'ios'
|
|
519
|
+
})
|
|
520
|
+
|
|
521
|
+
;(ToolsInteract as any).expectStateHandler = async () => {
|
|
522
|
+
return {
|
|
523
|
+
success: incrementalTreeValue === 3,
|
|
524
|
+
selector: { text: 'Stepper' },
|
|
525
|
+
element_id: null,
|
|
526
|
+
expected_state: { property: 'value', expected: 3 },
|
|
527
|
+
observed_state: { property: 'value', value: incrementalTreeValue, raw_value: incrementalTreeValue },
|
|
528
|
+
reason: incrementalTreeValue === 3 ? 'value matches expected value' : 'value still below target'
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const incrementalAdjust = await ToolsInteract.adjustControlHandler({
|
|
533
|
+
selector: { text: 'Stepper' },
|
|
534
|
+
property: 'value',
|
|
535
|
+
targetValue: 3,
|
|
536
|
+
maxAttempts: 5,
|
|
537
|
+
platform: 'ios'
|
|
538
|
+
})
|
|
539
|
+
|
|
540
|
+
assert.strictEqual(incrementalAdjust.success, true)
|
|
541
|
+
assert.strictEqual(incrementalAdjust.adjustment_mode, 'semantic')
|
|
542
|
+
assert.strictEqual(incrementalAdjust.attempts, 2)
|
|
543
|
+
assert.deepStrictEqual(directions, ['increment', 'increment'])
|
|
544
|
+
assert.strictEqual(incrementalAdjust.target_state.tolerance, 0)
|
|
545
|
+
assert.ok(incrementalTreeFetches >= 2, 'incremental retry should refetch the UI tree instead of cascading into stale coordinate fallback')
|
|
546
|
+
|
|
547
|
+
;(ToolsInteract as any).getInteractionService = originalGetInteractionService
|
|
548
|
+
|
|
301
549
|
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
302
550
|
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
303
551
|
screen: '',
|
|
@@ -399,8 +647,9 @@ async function run() {
|
|
|
399
647
|
tapCalls.push({ platform, x, y, deviceId })
|
|
400
648
|
return {
|
|
401
649
|
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
402
|
-
success:
|
|
403
|
-
|
|
650
|
+
success: true,
|
|
651
|
+
x,
|
|
652
|
+
y
|
|
404
653
|
}
|
|
405
654
|
}
|
|
406
655
|
|
|
@@ -450,37 +699,43 @@ async function run() {
|
|
|
450
699
|
assert.strictEqual(cachedResolveAdjust.success, true)
|
|
451
700
|
assert.strictEqual(cachedResolveAdjust.converged, true)
|
|
452
701
|
assert.strictEqual(cachedResolveAdjust.within_tolerance, true)
|
|
453
|
-
assert.
|
|
454
|
-
assert.
|
|
702
|
+
assert.strictEqual(cachedResolveAdjust.attempts, 2)
|
|
703
|
+
assert.ok(treeFetches >= 2, 'second attempt should refetch after a non-converged coordinate tap updates observed state')
|
|
455
704
|
|
|
456
705
|
const probeTapStart = tapCalls.length
|
|
457
706
|
const probeSwipeStart = swipeCalls.length
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
{
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
707
|
+
const probeSwipeFetches: number[] = []
|
|
708
|
+
let probeTreeValue = 10
|
|
709
|
+
let probeTreeFetches = 0
|
|
710
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
|
|
711
|
+
probeTreeFetches++
|
|
712
|
+
return {
|
|
713
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
714
|
+
screen: '',
|
|
715
|
+
resolution: { width: 1080, height: 2400 },
|
|
716
|
+
elements: [
|
|
717
|
+
{
|
|
718
|
+
text: 'Duration',
|
|
719
|
+
type: 'android.widget.SeekBar',
|
|
720
|
+
contentDescription: null,
|
|
721
|
+
clickable: true,
|
|
722
|
+
enabled: true,
|
|
723
|
+
visible: true,
|
|
724
|
+
bounds: [0, 0, 200, 40],
|
|
725
|
+
resourceId: 'seek_duration',
|
|
726
|
+
state: {
|
|
727
|
+
value: probeTreeValue,
|
|
728
|
+
raw_value: probeTreeValue,
|
|
729
|
+
value_range: { min: 0, max: 20 }
|
|
730
|
+
}
|
|
477
731
|
}
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
}
|
|
732
|
+
]
|
|
733
|
+
}
|
|
734
|
+
}
|
|
481
735
|
|
|
482
736
|
;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
|
|
483
737
|
tapCalls.push({ platform, x, y, deviceId })
|
|
738
|
+
probeTreeValue = probeTreeValue === 10 ? 11 : 12
|
|
484
739
|
return {
|
|
485
740
|
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
486
741
|
success: true,
|
|
@@ -488,10 +743,19 @@ async function run() {
|
|
|
488
743
|
y
|
|
489
744
|
}
|
|
490
745
|
}
|
|
746
|
+
;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
|
|
747
|
+
probeSwipeFetches.push(probeTreeFetches)
|
|
748
|
+
swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
|
|
749
|
+
return {
|
|
750
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
751
|
+
success: true,
|
|
752
|
+
start: [x1, y1],
|
|
753
|
+
end: [x2, y2],
|
|
754
|
+
duration
|
|
755
|
+
}
|
|
756
|
+
}
|
|
491
757
|
|
|
492
758
|
;(ToolsInteract as any).expectStateHandler = async () => {
|
|
493
|
-
probeVerificationCount++
|
|
494
|
-
const value = probeVerificationCount === 1 ? 11 : 12
|
|
495
759
|
return {
|
|
496
760
|
success: true,
|
|
497
761
|
selector: { text: 'Duration' },
|
|
@@ -505,10 +769,10 @@ async function run() {
|
|
|
505
769
|
class: 'android.widget.SeekBar',
|
|
506
770
|
bounds: [0, 0, 200, 40],
|
|
507
771
|
index: 0,
|
|
508
|
-
state: { value, raw_value:
|
|
772
|
+
state: { value: probeTreeValue, raw_value: probeTreeValue, value_range: { min: 0, max: 20 } }
|
|
509
773
|
},
|
|
510
|
-
observed_state: { property: 'value', value, raw_value:
|
|
511
|
-
reason:
|
|
774
|
+
observed_state: { property: 'value', value: probeTreeValue, raw_value: probeTreeValue },
|
|
775
|
+
reason: probeTreeValue === 12 ? 'value matches expected value' : 'value still below target'
|
|
512
776
|
}
|
|
513
777
|
}
|
|
514
778
|
|
|
@@ -525,9 +789,297 @@ async function run() {
|
|
|
525
789
|
assert.strictEqual(probeAdjust.converged, true)
|
|
526
790
|
assert.strictEqual(probeAdjust.within_tolerance, true)
|
|
527
791
|
assert.strictEqual(probeAdjust.adjustment_mode, 'coordinate')
|
|
528
|
-
assert.strictEqual(probeAdjust.attempts,
|
|
792
|
+
assert.strictEqual(probeAdjust.attempts, 3)
|
|
529
793
|
assert.strictEqual(tapCalls.length, probeTapStart + 2)
|
|
530
|
-
assert.strictEqual(swipeCalls.length, probeSwipeStart)
|
|
794
|
+
assert.strictEqual(swipeCalls.length, probeSwipeStart + 1)
|
|
795
|
+
assert.ok(probeTreeFetches >= 2, 'coordinate retry should refetch the UI tree instead of cascading into stale swipe fallback')
|
|
796
|
+
assert.ok(probeSwipeFetches[0] >= 2, 'swipe fallback should run only after a fresh UI tree read')
|
|
797
|
+
|
|
798
|
+
let defaultToleranceVerificationCount = 0
|
|
799
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
800
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
801
|
+
screen: '',
|
|
802
|
+
resolution: { width: 1080, height: 2400 },
|
|
803
|
+
elements: [
|
|
804
|
+
{
|
|
805
|
+
text: 'Default tolerance',
|
|
806
|
+
type: 'android.widget.SeekBar',
|
|
807
|
+
contentDescription: null,
|
|
808
|
+
clickable: true,
|
|
809
|
+
enabled: true,
|
|
810
|
+
visible: true,
|
|
811
|
+
bounds: [0, 0, 200, 40],
|
|
812
|
+
resourceId: 'seek_default',
|
|
813
|
+
state: {
|
|
814
|
+
value: 10,
|
|
815
|
+
raw_value: 10,
|
|
816
|
+
value_range: { min: 0, max: 100 }
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
]
|
|
820
|
+
})
|
|
821
|
+
|
|
822
|
+
;(ToolsInteract as any).expectStateHandler = async () => {
|
|
823
|
+
defaultToleranceVerificationCount++
|
|
824
|
+
return {
|
|
825
|
+
success: false,
|
|
826
|
+
selector: { text: 'Default tolerance' },
|
|
827
|
+
element_id: null,
|
|
828
|
+
expected_state: { property: 'value', expected: 50 },
|
|
829
|
+
observed_state: { property: 'value', value: 50.6, raw_value: 50.6 },
|
|
830
|
+
reason: 'exact value mismatch'
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
const defaultToleranceAdjust = await ToolsInteract.adjustControlHandler({
|
|
835
|
+
selector: { text: 'Default tolerance' },
|
|
836
|
+
property: 'value',
|
|
837
|
+
targetValue: 50,
|
|
838
|
+
platform: 'android'
|
|
839
|
+
})
|
|
840
|
+
|
|
841
|
+
assert.strictEqual(defaultToleranceAdjust.success, true)
|
|
842
|
+
assert.strictEqual(defaultToleranceAdjust.target_state.tolerance, 1)
|
|
843
|
+
assert.strictEqual(defaultToleranceVerificationCount, 1)
|
|
844
|
+
|
|
845
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
846
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
847
|
+
screen: '',
|
|
848
|
+
resolution: { width: 1080, height: 2400 },
|
|
849
|
+
elements: [
|
|
850
|
+
{
|
|
851
|
+
text: 'Discrete slider',
|
|
852
|
+
type: 'android.widget.SeekBar',
|
|
853
|
+
contentDescription: null,
|
|
854
|
+
clickable: true,
|
|
855
|
+
enabled: true,
|
|
856
|
+
visible: true,
|
|
857
|
+
bounds: [0, 0, 200, 40],
|
|
858
|
+
resourceId: 'seek_discrete',
|
|
859
|
+
state: {
|
|
860
|
+
value: 0,
|
|
861
|
+
raw_value: 0,
|
|
862
|
+
value_range: { min: 0, max: 20, step: 5 },
|
|
863
|
+
step: 5
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
]
|
|
867
|
+
})
|
|
868
|
+
|
|
869
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
870
|
+
success: true,
|
|
871
|
+
selector: { text: 'Discrete slider' },
|
|
872
|
+
element_id: null,
|
|
873
|
+
expected_state: { property: 'value', expected: 10 },
|
|
874
|
+
observed_state: { property: 'value', value: 10, raw_value: 10 },
|
|
875
|
+
reason: 'value matches expected value'
|
|
876
|
+
})
|
|
877
|
+
|
|
878
|
+
const discreteAdjust = await ToolsInteract.adjustControlHandler({
|
|
879
|
+
selector: { text: 'Discrete slider' },
|
|
880
|
+
property: 'value',
|
|
881
|
+
targetValue: 12,
|
|
882
|
+
platform: 'android'
|
|
883
|
+
})
|
|
884
|
+
|
|
885
|
+
assert.strictEqual(discreteAdjust.success, true)
|
|
886
|
+
assert.strictEqual(discreteAdjust.target_state.target_value, 10)
|
|
887
|
+
assert.strictEqual(discreteAdjust.target_state.tolerance, 0)
|
|
888
|
+
|
|
889
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
890
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
891
|
+
screen: '',
|
|
892
|
+
resolution: { width: 1080, height: 2400 },
|
|
893
|
+
elements: [
|
|
894
|
+
{
|
|
895
|
+
text: 'Scientific step slider',
|
|
896
|
+
type: 'android.widget.SeekBar',
|
|
897
|
+
contentDescription: null,
|
|
898
|
+
clickable: true,
|
|
899
|
+
enabled: true,
|
|
900
|
+
visible: true,
|
|
901
|
+
bounds: [0, 0, 200, 40],
|
|
902
|
+
resourceId: 'seek_scientific_step',
|
|
903
|
+
state: {
|
|
904
|
+
value: 0,
|
|
905
|
+
raw_value: 0,
|
|
906
|
+
value_range: { min: 0, max: 0.000001, step: 1e-7 },
|
|
907
|
+
step: 1e-7
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
]
|
|
911
|
+
})
|
|
912
|
+
|
|
913
|
+
;(ToolsInteract as any).expectStateHandler = async () => ({
|
|
914
|
+
success: true,
|
|
915
|
+
selector: { text: 'Scientific step slider' },
|
|
916
|
+
element_id: null,
|
|
917
|
+
expected_state: { property: 'value', expected: 0.0000003 },
|
|
918
|
+
observed_state: { property: 'value', value: 0.0000003, raw_value: 0.0000003 },
|
|
919
|
+
reason: 'value matches expected value'
|
|
920
|
+
})
|
|
921
|
+
|
|
922
|
+
const scientificStepAdjust = await ToolsInteract.adjustControlHandler({
|
|
923
|
+
selector: { text: 'Scientific step slider' },
|
|
924
|
+
property: 'value',
|
|
925
|
+
targetValue: 0.00000034,
|
|
926
|
+
platform: 'android'
|
|
927
|
+
})
|
|
928
|
+
|
|
929
|
+
assert.strictEqual(scientificStepAdjust.success, true)
|
|
930
|
+
assert.strictEqual(scientificStepAdjust.target_state.target_value, 0.0000003)
|
|
931
|
+
assert.strictEqual(scientificStepAdjust.target_state.tolerance, 0)
|
|
932
|
+
|
|
933
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
934
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
935
|
+
screen: '',
|
|
936
|
+
resolution: { width: 1080, height: 2400 },
|
|
937
|
+
elements: [
|
|
938
|
+
{
|
|
939
|
+
text: 'Out of range',
|
|
940
|
+
type: 'android.widget.SeekBar',
|
|
941
|
+
contentDescription: null,
|
|
942
|
+
clickable: true,
|
|
943
|
+
enabled: true,
|
|
944
|
+
visible: true,
|
|
945
|
+
bounds: [0, 0, 200, 40],
|
|
946
|
+
resourceId: 'seek_range',
|
|
947
|
+
state: {
|
|
948
|
+
value: 10,
|
|
949
|
+
raw_value: 10,
|
|
950
|
+
value_range: { min: 0, max: 20 }
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
]
|
|
954
|
+
})
|
|
955
|
+
|
|
956
|
+
const outOfRangeAdjust = await ToolsInteract.adjustControlHandler({
|
|
957
|
+
selector: { text: 'Out of range' },
|
|
958
|
+
property: 'value',
|
|
959
|
+
targetValue: 30,
|
|
960
|
+
platform: 'android'
|
|
961
|
+
})
|
|
962
|
+
|
|
963
|
+
assert.strictEqual(outOfRangeAdjust.success, false)
|
|
964
|
+
assert.strictEqual(outOfRangeAdjust.failure_code, 'CONTROL_CONVERGENCE_FAILED')
|
|
965
|
+
|
|
966
|
+
let inferredLabelTreeValue = 3
|
|
967
|
+
const inferredLabelTree = () => ({
|
|
968
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
969
|
+
screen: '',
|
|
970
|
+
resolution: { width: 1280, height: 2856 },
|
|
971
|
+
elements: [
|
|
972
|
+
{
|
|
973
|
+
text: 'Duration',
|
|
974
|
+
type: 'android.widget.TextView',
|
|
975
|
+
contentDescription: null,
|
|
976
|
+
clickable: false,
|
|
977
|
+
enabled: true,
|
|
978
|
+
visible: true,
|
|
979
|
+
bounds: [132, 1855, 404, 1951],
|
|
980
|
+
resourceId: null,
|
|
981
|
+
state: { value: 'Duration' }
|
|
982
|
+
},
|
|
983
|
+
{
|
|
984
|
+
text: String(inferredLabelTreeValue),
|
|
985
|
+
type: 'android.widget.TextView',
|
|
986
|
+
contentDescription: null,
|
|
987
|
+
clickable: false,
|
|
988
|
+
enabled: true,
|
|
989
|
+
visible: true,
|
|
990
|
+
bounds: [987, 1827, 1063, 1985],
|
|
991
|
+
resourceId: null,
|
|
992
|
+
state: { value: String(inferredLabelTreeValue) }
|
|
993
|
+
},
|
|
994
|
+
{
|
|
995
|
+
text: 'min',
|
|
996
|
+
type: 'android.widget.TextView',
|
|
997
|
+
contentDescription: null,
|
|
998
|
+
clickable: false,
|
|
999
|
+
enabled: true,
|
|
1000
|
+
visible: true,
|
|
1001
|
+
bounds: [1075, 1891, 1148, 1951],
|
|
1002
|
+
resourceId: null,
|
|
1003
|
+
state: { value: 'min' }
|
|
1004
|
+
},
|
|
1005
|
+
{
|
|
1006
|
+
text: '1 min',
|
|
1007
|
+
type: 'android.widget.TextView',
|
|
1008
|
+
contentDescription: null,
|
|
1009
|
+
clickable: false,
|
|
1010
|
+
enabled: true,
|
|
1011
|
+
visible: true,
|
|
1012
|
+
bounds: [132, 2177, 228, 2225],
|
|
1013
|
+
resourceId: null,
|
|
1014
|
+
state: { value: '1 min' }
|
|
1015
|
+
},
|
|
1016
|
+
{
|
|
1017
|
+
text: '15 min',
|
|
1018
|
+
type: 'android.widget.TextView',
|
|
1019
|
+
contentDescription: null,
|
|
1020
|
+
clickable: false,
|
|
1021
|
+
enabled: true,
|
|
1022
|
+
visible: true,
|
|
1023
|
+
bounds: [1031, 2177, 1148, 2225],
|
|
1024
|
+
resourceId: null,
|
|
1025
|
+
state: { value: '15 min' }
|
|
1026
|
+
},
|
|
1027
|
+
{
|
|
1028
|
+
text: null,
|
|
1029
|
+
type: 'android.view.View',
|
|
1030
|
+
contentDescription: null,
|
|
1031
|
+
clickable: true,
|
|
1032
|
+
enabled: true,
|
|
1033
|
+
visible: true,
|
|
1034
|
+
bounds: [84, 2369, 1196, 2513],
|
|
1035
|
+
resourceId: null,
|
|
1036
|
+
state: { selected: true, enabled: true }
|
|
1037
|
+
}
|
|
1038
|
+
]
|
|
1039
|
+
})
|
|
1040
|
+
|
|
1041
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => inferredLabelTree()
|
|
1042
|
+
;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
|
|
1043
|
+
tapCalls.push({ platform, x, y, deviceId })
|
|
1044
|
+
return {
|
|
1045
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
1046
|
+
success: true,
|
|
1047
|
+
x,
|
|
1048
|
+
y
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
|
|
1052
|
+
swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
|
|
1053
|
+
inferredLabelTreeValue = 5
|
|
1054
|
+
return {
|
|
1055
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
1056
|
+
success: true,
|
|
1057
|
+
start: [x1, y1],
|
|
1058
|
+
end: [x2, y2],
|
|
1059
|
+
duration
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
const inferredTapStart = tapCalls.length
|
|
1064
|
+
const inferredSwipeStart = swipeCalls.length
|
|
1065
|
+
const inferredLabelAdjust = await ToolsInteract.adjustControlHandler({
|
|
1066
|
+
selector: { text: 'Duration' },
|
|
1067
|
+
property: 'value',
|
|
1068
|
+
targetValue: 5,
|
|
1069
|
+
tolerance: 0.5,
|
|
1070
|
+
platform: 'android'
|
|
1071
|
+
})
|
|
1072
|
+
|
|
1073
|
+
assert.strictEqual(inferredLabelAdjust.success, true)
|
|
1074
|
+
assert.strictEqual(inferredLabelAdjust.adjustment_mode, 'gesture')
|
|
1075
|
+
assert.strictEqual(inferredLabelAdjust.target_state.target_value, 5)
|
|
1076
|
+
assert.strictEqual(inferredLabelAdjust.actual_state?.value, 5)
|
|
1077
|
+
assert.strictEqual(tapCalls.length, inferredTapStart + 1)
|
|
1078
|
+
assert.strictEqual(swipeCalls.length, inferredSwipeStart + 1)
|
|
1079
|
+
assert.strictEqual(swipeCalls[swipeCalls.length - 1].duration, 520)
|
|
1080
|
+
assert.ok(tapCalls[tapCalls.length - 1].y < 2369, 'inferred slider tap should avoid the clickable view below the min/max labels')
|
|
1081
|
+
assert.ok(swipeCalls[swipeCalls.length - 1].y1 < 2369, 'inferred slider drag should start above the clickable view below the min/max labels')
|
|
1082
|
+
assert.ok(swipeCalls[swipeCalls.length - 1].y2 < 2369, 'inferred slider drag should end above the clickable view below the min/max labels')
|
|
531
1083
|
|
|
532
1084
|
console.log('adjust_control unit tests passed')
|
|
533
1085
|
} finally {
|
|
@@ -536,6 +1088,7 @@ async function run() {
|
|
|
536
1088
|
;(ToolsInteract as any).tapHandler = originalTapHandler
|
|
537
1089
|
;(ToolsInteract as any).swipeHandler = originalSwipeHandler
|
|
538
1090
|
;(ToolsInteract as any).expectStateHandler = originalExpectStateHandler
|
|
1091
|
+
;(ToolsInteract as any).getInteractionService = originalGetInteractionService
|
|
539
1092
|
}
|
|
540
1093
|
}
|
|
541
1094
|
|