mobile-debug-mcp 0.30.1 → 0.30.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/interact/index.js +546 -133
  2. package/dist/server/tool-definitions.js +13 -9
  3. package/dist/server/tool-handlers.js +4 -4
  4. package/dist/server-core.js +1 -1
  5. package/docs/CHANGELOG.md +3 -0
  6. package/docs/ROADMAP.md +85 -209
  7. package/docs/specs-deltas/015-adjustable-control-support.md +541 -0
  8. package/package.json +1 -1
  9. package/skills/spec-review/SKILL.md +52 -0
  10. package/skills/spec-review/references/spec-review-checklist.md +13 -0
  11. package/skills/spec-review/references/spec-review-template.md +28 -0
  12. package/src/interact/index.ts +581 -138
  13. package/src/server/tool-definitions.ts +13 -9
  14. package/src/server/tool-handlers.ts +4 -4
  15. package/src/server-core.ts +1 -1
  16. package/src/types.ts +2 -0
  17. package/test/unit/interact/adjust_control.test.ts +586 -33
  18. package/test/unit/server/contract.test.ts +4 -0
  19. package/skills/rfc-review/SKILL.md +0 -52
  20. package/skills/rfc-review/references/rfc-review-checklist.md +0 -12
  21. package/skills/rfc-review/references/rfc-review-template.md +0 -28
  22. /package/docs/{rfcs → specs-deltas}/001-state-verification.md +0 -0
  23. /package/docs/{rfcs → specs-deltas}/002-richer-element-identity +0 -0
  24. /package/docs/{rfcs → specs-deltas}/003-wait-and-synchronization-reliability.md +0 -0
  25. /package/docs/{rfcs → specs-deltas}/004-action-verification-routing.md +0 -0
  26. /package/docs/{rfcs → specs-deltas}/005-unified-action-execution-and-verification-model.md +0 -0
  27. /package/docs/{rfcs → specs-deltas}/006-runtime-action-instrumentation-and-binding-layer.md +0 -0
  28. /package/docs/{rfcs → specs-deltas}/007-actionability-resolution-and-executable-target-selection.md +0 -0
  29. /package/docs/{rfcs → specs-deltas}/008-adjustable-control-support-and-semantic-value-manipulation.md +0 -0
  30. /package/docs/{rfcs → specs-deltas}/009-semantic-control-modeling-for-custom-and-composite-controls.md +0 -0
  31. /package/docs/{rfcs → specs-deltas}/010-verification-stabilization-and-temporal-convergence.md +0 -0
  32. /package/docs/{rfcs → specs-deltas}/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows.md +0 -0
  33. /package/docs/{rfcs → specs-deltas}/011.1-recovery-contract-types-and-runtime-wiring-spec.md +0 -0
  34. /package/docs/{rfcs → specs-deltas}/012-action-trace-and-xecution-observability.md +0 -0
  35. /package/docs/{rfcs → specs-deltas}/013-wait-and-synchronization-reliability.md +0 -0
  36. /package/docs/{rfcs → specs-deltas}/014-actionability-resolution.md +0 -0
@@ -10,6 +10,7 @@ async function run() {
10
10
  const originalTapHandler = (ToolsInteract as any).tapHandler
11
11
  const originalSwipeHandler = (ToolsInteract as any).swipeHandler
12
12
  const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
13
+ const originalGetInteractionService = (ToolsInteract as any).getInteractionService
13
14
 
14
15
  try {
15
16
  ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
@@ -298,6 +299,253 @@ async function run() {
298
299
  assert.strictEqual(disabledAdjust.success, false)
299
300
  assert.strictEqual(disabledAdjust.failure_code, 'ELEMENT_NOT_INTERACTABLE')
300
301
 
302
+ const bothTargetsAdjust = await ToolsInteract.adjustControlHandler({
303
+ selector: { text: 'Duration' },
304
+ element_id: wait.element.elementId,
305
+ property: 'value',
306
+ targetValue: 8,
307
+ platform: 'android'
308
+ })
309
+
310
+ assert.strictEqual(bothTargetsAdjust.success, false)
311
+ assert.strictEqual(bothTargetsAdjust.failure_code, 'ELEMENT_NOT_FOUND')
312
+
313
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
314
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
315
+ screen: '',
316
+ resolution: { width: 1080, height: 2400 },
317
+ elements: [
318
+ {
319
+ text: 'Plain button',
320
+ type: 'android.widget.Button',
321
+ contentDescription: null,
322
+ clickable: true,
323
+ enabled: true,
324
+ visible: true,
325
+ bounds: [0, 0, 200, 40],
326
+ resourceId: 'plain_button',
327
+ state: null
328
+ }
329
+ ]
330
+ })
331
+
332
+ const nonAdjustable = await ToolsInteract.adjustControlHandler({
333
+ selector: { text: 'Plain button' },
334
+ property: 'value',
335
+ targetValue: 1,
336
+ platform: 'android'
337
+ })
338
+
339
+ assert.strictEqual(nonAdjustable.success, false)
340
+ assert.strictEqual(nonAdjustable.failure_code, 'ELEMENT_NOT_INTERACTABLE')
341
+
342
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
343
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
344
+ screen: '',
345
+ resolution: { width: 1080, height: 2400 },
346
+ elements: [
347
+ {
348
+ text: 'Direct slider',
349
+ type: 'android.widget.SeekBar',
350
+ contentDescription: null,
351
+ clickable: true,
352
+ enabled: true,
353
+ visible: true,
354
+ bounds: [0, 0, 200, 40],
355
+ resourceId: 'seek_direct',
356
+ actions: ['ACTION_SET_PROGRESS'],
357
+ state: {
358
+ value: 10,
359
+ raw_value: 10,
360
+ value_range: { min: 0, max: 100 }
361
+ }
362
+ }
363
+ ]
364
+ })
365
+
366
+ let directCalls = 0
367
+ ;(ToolsInteract as any).getInteractionService = async () => ({
368
+ resolved: { id: 'mock-device' },
369
+ interact: {
370
+ setAdjustableValue: async ({ value }: any) => {
371
+ directCalls++
372
+ assert.strictEqual(value, 42)
373
+ return {
374
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
375
+ success: true
376
+ }
377
+ }
378
+ },
379
+ platform: 'android'
380
+ })
381
+
382
+ ;(ToolsInteract as any).expectStateHandler = async () => ({
383
+ success: true,
384
+ selector: { text: 'Direct slider' },
385
+ element_id: null,
386
+ expected_state: { property: 'value', expected: 42 },
387
+ observed_state: { property: 'value', value: 42, raw_value: 42 },
388
+ reason: 'value matches expected value'
389
+ })
390
+
391
+ const directAdjust = await ToolsInteract.adjustControlHandler({
392
+ selector: { text: 'Direct slider' },
393
+ property: 'value',
394
+ targetValue: 42,
395
+ platform: 'android'
396
+ })
397
+
398
+ assert.strictEqual(directAdjust.success, true)
399
+ assert.strictEqual(directAdjust.adjustment_mode, 'semantic')
400
+ assert.strictEqual(directAdjust.attempts, 1)
401
+ assert.strictEqual(directCalls, 1)
402
+
403
+ let directRefreshValue = 10
404
+ let directRefreshTreeFetches = 0
405
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
406
+ directRefreshTreeFetches++
407
+ return {
408
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
409
+ screen: '',
410
+ resolution: { width: 1080, height: 2400 },
411
+ elements: [
412
+ {
413
+ text: 'Retrying direct slider',
414
+ type: 'android.widget.SeekBar',
415
+ contentDescription: null,
416
+ clickable: true,
417
+ enabled: true,
418
+ visible: true,
419
+ bounds: [0, 0, 200, 40],
420
+ resourceId: 'seek_direct_retry',
421
+ actions: ['ACTION_SET_PROGRESS'],
422
+ state: {
423
+ value: directRefreshValue,
424
+ raw_value: directRefreshValue,
425
+ value_range: { min: 0, max: 100 }
426
+ }
427
+ }
428
+ ]
429
+ }
430
+ }
431
+
432
+ let retryingDirectCalls = 0
433
+ ;(ToolsInteract as any).getInteractionService = async () => ({
434
+ resolved: { id: 'mock-device' },
435
+ interact: {
436
+ setAdjustableValue: async () => {
437
+ retryingDirectCalls++
438
+ directRefreshValue = retryingDirectCalls === 1 ? 60 : 80
439
+ return {
440
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
441
+ success: true
442
+ }
443
+ }
444
+ },
445
+ platform: 'android'
446
+ })
447
+
448
+ ;(ToolsInteract as any).tapHandler = async () => {
449
+ throw new Error('coordinate fallback should not run before direct retry refreshes state')
450
+ }
451
+
452
+ ;(ToolsInteract as any).expectStateHandler = async () => ({
453
+ success: directRefreshValue === 80,
454
+ selector: { text: 'Retrying direct slider' },
455
+ element_id: null,
456
+ expected_state: { property: 'value', expected: 80 },
457
+ observed_state: { property: 'value', value: directRefreshValue, raw_value: directRefreshValue },
458
+ reason: directRefreshValue === 80 ? 'value matches expected value' : 'direct adjustment partially applied'
459
+ })
460
+
461
+ const retryingDirectAdjust = await ToolsInteract.adjustControlHandler({
462
+ selector: { text: 'Retrying direct slider' },
463
+ property: 'value',
464
+ targetValue: 80,
465
+ maxAttempts: 2,
466
+ platform: 'android'
467
+ })
468
+
469
+ assert.strictEqual(retryingDirectAdjust.success, true)
470
+ assert.strictEqual(retryingDirectAdjust.adjustment_mode, 'semantic')
471
+ assert.strictEqual(retryingDirectAdjust.attempts, 2)
472
+ assert.strictEqual(retryingDirectCalls, 2)
473
+ assert.ok(directRefreshTreeFetches >= 2, 'direct retry should refetch the UI tree instead of cascading into stale coordinate fallback')
474
+
475
+ let incrementalTreeValue = 1
476
+ let incrementalTreeFetches = 0
477
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
478
+ incrementalTreeFetches++
479
+ return {
480
+ device: { platform: 'ios', id: 'booted', osVersion: '17', model: 'Simulator', simulator: true },
481
+ screen: '',
482
+ resolution: { width: 390, height: 844 },
483
+ elements: [
484
+ {
485
+ text: 'Stepper',
486
+ type: 'XCUIElementTypeStepper',
487
+ contentDescription: null,
488
+ clickable: true,
489
+ enabled: true,
490
+ visible: true,
491
+ bounds: [0, 0, 120, 44],
492
+ resourceId: 'stepper',
493
+ actions: ['increment', 'decrement'],
494
+ state: {
495
+ value: incrementalTreeValue,
496
+ raw_value: incrementalTreeValue,
497
+ value_range: { min: 0, max: 5, step: 1 },
498
+ step: 1
499
+ }
500
+ }
501
+ ]
502
+ }
503
+ }
504
+
505
+ const directions: string[] = []
506
+ ;(ToolsInteract as any).getInteractionService = async () => ({
507
+ resolved: { id: 'booted' },
508
+ interact: {
509
+ adjustAccessibleValue: async ({ direction }: any) => {
510
+ directions.push(direction)
511
+ incrementalTreeValue += direction === 'increment' ? 1 : -1
512
+ return {
513
+ device: { platform: 'ios', id: 'booted', osVersion: '17', model: 'Simulator', simulator: true },
514
+ success: true
515
+ }
516
+ }
517
+ },
518
+ platform: 'ios'
519
+ })
520
+
521
+ ;(ToolsInteract as any).expectStateHandler = async () => {
522
+ return {
523
+ success: incrementalTreeValue === 3,
524
+ selector: { text: 'Stepper' },
525
+ element_id: null,
526
+ expected_state: { property: 'value', expected: 3 },
527
+ observed_state: { property: 'value', value: incrementalTreeValue, raw_value: incrementalTreeValue },
528
+ reason: incrementalTreeValue === 3 ? 'value matches expected value' : 'value still below target'
529
+ }
530
+ }
531
+
532
+ const incrementalAdjust = await ToolsInteract.adjustControlHandler({
533
+ selector: { text: 'Stepper' },
534
+ property: 'value',
535
+ targetValue: 3,
536
+ maxAttempts: 5,
537
+ platform: 'ios'
538
+ })
539
+
540
+ assert.strictEqual(incrementalAdjust.success, true)
541
+ assert.strictEqual(incrementalAdjust.adjustment_mode, 'semantic')
542
+ assert.strictEqual(incrementalAdjust.attempts, 2)
543
+ assert.deepStrictEqual(directions, ['increment', 'increment'])
544
+ assert.strictEqual(incrementalAdjust.target_state.tolerance, 0)
545
+ assert.ok(incrementalTreeFetches >= 2, 'incremental retry should refetch the UI tree instead of cascading into stale coordinate fallback')
546
+
547
+ ;(ToolsInteract as any).getInteractionService = originalGetInteractionService
548
+
301
549
  ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
302
550
  device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
303
551
  screen: '',
@@ -399,8 +647,9 @@ async function run() {
399
647
  tapCalls.push({ platform, x, y, deviceId })
400
648
  return {
401
649
  device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
402
- success: false,
403
- error: 'tap failed'
650
+ success: true,
651
+ x,
652
+ y
404
653
  }
405
654
  }
406
655
 
@@ -450,37 +699,43 @@ async function run() {
450
699
  assert.strictEqual(cachedResolveAdjust.success, true)
451
700
  assert.strictEqual(cachedResolveAdjust.converged, true)
452
701
  assert.strictEqual(cachedResolveAdjust.within_tolerance, true)
453
- assert.ok(cachedResolveAdjust.attempts >= 3)
454
- assert.strictEqual(treeFetches, 1, 'second attempt should reuse the resolved element instead of refetching the UI tree')
702
+ assert.strictEqual(cachedResolveAdjust.attempts, 2)
703
+ assert.ok(treeFetches >= 2, 'second attempt should refetch after a non-converged coordinate tap updates observed state')
455
704
 
456
705
  const probeTapStart = tapCalls.length
457
706
  const probeSwipeStart = swipeCalls.length
458
- let probeVerificationCount = 0
459
- ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
460
- device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
461
- screen: '',
462
- resolution: { width: 1080, height: 2400 },
463
- elements: [
464
- {
465
- text: 'Duration',
466
- type: 'android.widget.SeekBar',
467
- contentDescription: null,
468
- clickable: true,
469
- enabled: true,
470
- visible: true,
471
- bounds: [0, 0, 200, 40],
472
- resourceId: 'seek_duration',
473
- state: {
474
- value: 10,
475
- raw_value: 10,
476
- value_range: { min: 0, max: 20 }
707
+ const probeSwipeFetches: number[] = []
708
+ let probeTreeValue = 10
709
+ let probeTreeFetches = 0
710
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
711
+ probeTreeFetches++
712
+ return {
713
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
714
+ screen: '',
715
+ resolution: { width: 1080, height: 2400 },
716
+ elements: [
717
+ {
718
+ text: 'Duration',
719
+ type: 'android.widget.SeekBar',
720
+ contentDescription: null,
721
+ clickable: true,
722
+ enabled: true,
723
+ visible: true,
724
+ bounds: [0, 0, 200, 40],
725
+ resourceId: 'seek_duration',
726
+ state: {
727
+ value: probeTreeValue,
728
+ raw_value: probeTreeValue,
729
+ value_range: { min: 0, max: 20 }
730
+ }
477
731
  }
478
- }
479
- ]
480
- })
732
+ ]
733
+ }
734
+ }
481
735
 
482
736
  ;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
483
737
  tapCalls.push({ platform, x, y, deviceId })
738
+ probeTreeValue = probeTreeValue === 10 ? 11 : 12
484
739
  return {
485
740
  device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
486
741
  success: true,
@@ -488,10 +743,19 @@ async function run() {
488
743
  y
489
744
  }
490
745
  }
746
+ ;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
747
+ probeSwipeFetches.push(probeTreeFetches)
748
+ swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
749
+ return {
750
+ device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
751
+ success: true,
752
+ start: [x1, y1],
753
+ end: [x2, y2],
754
+ duration
755
+ }
756
+ }
491
757
 
492
758
  ;(ToolsInteract as any).expectStateHandler = async () => {
493
- probeVerificationCount++
494
- const value = probeVerificationCount === 1 ? 11 : 12
495
759
  return {
496
760
  success: true,
497
761
  selector: { text: 'Duration' },
@@ -505,10 +769,10 @@ async function run() {
505
769
  class: 'android.widget.SeekBar',
506
770
  bounds: [0, 0, 200, 40],
507
771
  index: 0,
508
- state: { value, raw_value: value, value_range: { min: 0, max: 20 } }
772
+ state: { value: probeTreeValue, raw_value: probeTreeValue, value_range: { min: 0, max: 20 } }
509
773
  },
510
- observed_state: { property: 'value', value, raw_value: value },
511
- reason: value === 12 ? 'value matches expected value' : 'value still below target'
774
+ observed_state: { property: 'value', value: probeTreeValue, raw_value: probeTreeValue },
775
+ reason: probeTreeValue === 12 ? 'value matches expected value' : 'value still below target'
512
776
  }
513
777
  }
514
778
 
@@ -525,9 +789,297 @@ async function run() {
525
789
  assert.strictEqual(probeAdjust.converged, true)
526
790
  assert.strictEqual(probeAdjust.within_tolerance, true)
527
791
  assert.strictEqual(probeAdjust.adjustment_mode, 'coordinate')
528
- assert.strictEqual(probeAdjust.attempts, 2)
792
+ assert.strictEqual(probeAdjust.attempts, 3)
529
793
  assert.strictEqual(tapCalls.length, probeTapStart + 2)
530
- assert.strictEqual(swipeCalls.length, probeSwipeStart)
794
+ assert.strictEqual(swipeCalls.length, probeSwipeStart + 1)
795
+ assert.ok(probeTreeFetches >= 2, 'coordinate retry should refetch the UI tree instead of cascading into stale swipe fallback')
796
+ assert.ok(probeSwipeFetches[0] >= 2, 'swipe fallback should run only after a fresh UI tree read')
797
+
798
+ let defaultToleranceVerificationCount = 0
799
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
800
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
801
+ screen: '',
802
+ resolution: { width: 1080, height: 2400 },
803
+ elements: [
804
+ {
805
+ text: 'Default tolerance',
806
+ type: 'android.widget.SeekBar',
807
+ contentDescription: null,
808
+ clickable: true,
809
+ enabled: true,
810
+ visible: true,
811
+ bounds: [0, 0, 200, 40],
812
+ resourceId: 'seek_default',
813
+ state: {
814
+ value: 10,
815
+ raw_value: 10,
816
+ value_range: { min: 0, max: 100 }
817
+ }
818
+ }
819
+ ]
820
+ })
821
+
822
+ ;(ToolsInteract as any).expectStateHandler = async () => {
823
+ defaultToleranceVerificationCount++
824
+ return {
825
+ success: false,
826
+ selector: { text: 'Default tolerance' },
827
+ element_id: null,
828
+ expected_state: { property: 'value', expected: 50 },
829
+ observed_state: { property: 'value', value: 50.6, raw_value: 50.6 },
830
+ reason: 'exact value mismatch'
831
+ }
832
+ }
833
+
834
+ const defaultToleranceAdjust = await ToolsInteract.adjustControlHandler({
835
+ selector: { text: 'Default tolerance' },
836
+ property: 'value',
837
+ targetValue: 50,
838
+ platform: 'android'
839
+ })
840
+
841
+ assert.strictEqual(defaultToleranceAdjust.success, true)
842
+ assert.strictEqual(defaultToleranceAdjust.target_state.tolerance, 1)
843
+ assert.strictEqual(defaultToleranceVerificationCount, 1)
844
+
845
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
846
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
847
+ screen: '',
848
+ resolution: { width: 1080, height: 2400 },
849
+ elements: [
850
+ {
851
+ text: 'Discrete slider',
852
+ type: 'android.widget.SeekBar',
853
+ contentDescription: null,
854
+ clickable: true,
855
+ enabled: true,
856
+ visible: true,
857
+ bounds: [0, 0, 200, 40],
858
+ resourceId: 'seek_discrete',
859
+ state: {
860
+ value: 0,
861
+ raw_value: 0,
862
+ value_range: { min: 0, max: 20, step: 5 },
863
+ step: 5
864
+ }
865
+ }
866
+ ]
867
+ })
868
+
869
+ ;(ToolsInteract as any).expectStateHandler = async () => ({
870
+ success: true,
871
+ selector: { text: 'Discrete slider' },
872
+ element_id: null,
873
+ expected_state: { property: 'value', expected: 10 },
874
+ observed_state: { property: 'value', value: 10, raw_value: 10 },
875
+ reason: 'value matches expected value'
876
+ })
877
+
878
+ const discreteAdjust = await ToolsInteract.adjustControlHandler({
879
+ selector: { text: 'Discrete slider' },
880
+ property: 'value',
881
+ targetValue: 12,
882
+ platform: 'android'
883
+ })
884
+
885
+ assert.strictEqual(discreteAdjust.success, true)
886
+ assert.strictEqual(discreteAdjust.target_state.target_value, 10)
887
+ assert.strictEqual(discreteAdjust.target_state.tolerance, 0)
888
+
889
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
890
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
891
+ screen: '',
892
+ resolution: { width: 1080, height: 2400 },
893
+ elements: [
894
+ {
895
+ text: 'Scientific step slider',
896
+ type: 'android.widget.SeekBar',
897
+ contentDescription: null,
898
+ clickable: true,
899
+ enabled: true,
900
+ visible: true,
901
+ bounds: [0, 0, 200, 40],
902
+ resourceId: 'seek_scientific_step',
903
+ state: {
904
+ value: 0,
905
+ raw_value: 0,
906
+ value_range: { min: 0, max: 0.000001, step: 1e-7 },
907
+ step: 1e-7
908
+ }
909
+ }
910
+ ]
911
+ })
912
+
913
+ ;(ToolsInteract as any).expectStateHandler = async () => ({
914
+ success: true,
915
+ selector: { text: 'Scientific step slider' },
916
+ element_id: null,
917
+ expected_state: { property: 'value', expected: 0.0000003 },
918
+ observed_state: { property: 'value', value: 0.0000003, raw_value: 0.0000003 },
919
+ reason: 'value matches expected value'
920
+ })
921
+
922
+ const scientificStepAdjust = await ToolsInteract.adjustControlHandler({
923
+ selector: { text: 'Scientific step slider' },
924
+ property: 'value',
925
+ targetValue: 0.00000034,
926
+ platform: 'android'
927
+ })
928
+
929
+ assert.strictEqual(scientificStepAdjust.success, true)
930
+ assert.strictEqual(scientificStepAdjust.target_state.target_value, 0.0000003)
931
+ assert.strictEqual(scientificStepAdjust.target_state.tolerance, 0)
932
+
933
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
934
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
935
+ screen: '',
936
+ resolution: { width: 1080, height: 2400 },
937
+ elements: [
938
+ {
939
+ text: 'Out of range',
940
+ type: 'android.widget.SeekBar',
941
+ contentDescription: null,
942
+ clickable: true,
943
+ enabled: true,
944
+ visible: true,
945
+ bounds: [0, 0, 200, 40],
946
+ resourceId: 'seek_range',
947
+ state: {
948
+ value: 10,
949
+ raw_value: 10,
950
+ value_range: { min: 0, max: 20 }
951
+ }
952
+ }
953
+ ]
954
+ })
955
+
956
+ const outOfRangeAdjust = await ToolsInteract.adjustControlHandler({
957
+ selector: { text: 'Out of range' },
958
+ property: 'value',
959
+ targetValue: 30,
960
+ platform: 'android'
961
+ })
962
+
963
+ assert.strictEqual(outOfRangeAdjust.success, false)
964
+ assert.strictEqual(outOfRangeAdjust.failure_code, 'CONTROL_CONVERGENCE_FAILED')
965
+
966
+ let inferredLabelTreeValue = 3
967
+ const inferredLabelTree = () => ({
968
+ device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
969
+ screen: '',
970
+ resolution: { width: 1280, height: 2856 },
971
+ elements: [
972
+ {
973
+ text: 'Duration',
974
+ type: 'android.widget.TextView',
975
+ contentDescription: null,
976
+ clickable: false,
977
+ enabled: true,
978
+ visible: true,
979
+ bounds: [132, 1855, 404, 1951],
980
+ resourceId: null,
981
+ state: { value: 'Duration' }
982
+ },
983
+ {
984
+ text: String(inferredLabelTreeValue),
985
+ type: 'android.widget.TextView',
986
+ contentDescription: null,
987
+ clickable: false,
988
+ enabled: true,
989
+ visible: true,
990
+ bounds: [987, 1827, 1063, 1985],
991
+ resourceId: null,
992
+ state: { value: String(inferredLabelTreeValue) }
993
+ },
994
+ {
995
+ text: 'min',
996
+ type: 'android.widget.TextView',
997
+ contentDescription: null,
998
+ clickable: false,
999
+ enabled: true,
1000
+ visible: true,
1001
+ bounds: [1075, 1891, 1148, 1951],
1002
+ resourceId: null,
1003
+ state: { value: 'min' }
1004
+ },
1005
+ {
1006
+ text: '1 min',
1007
+ type: 'android.widget.TextView',
1008
+ contentDescription: null,
1009
+ clickable: false,
1010
+ enabled: true,
1011
+ visible: true,
1012
+ bounds: [132, 2177, 228, 2225],
1013
+ resourceId: null,
1014
+ state: { value: '1 min' }
1015
+ },
1016
+ {
1017
+ text: '15 min',
1018
+ type: 'android.widget.TextView',
1019
+ contentDescription: null,
1020
+ clickable: false,
1021
+ enabled: true,
1022
+ visible: true,
1023
+ bounds: [1031, 2177, 1148, 2225],
1024
+ resourceId: null,
1025
+ state: { value: '15 min' }
1026
+ },
1027
+ {
1028
+ text: null,
1029
+ type: 'android.view.View',
1030
+ contentDescription: null,
1031
+ clickable: true,
1032
+ enabled: true,
1033
+ visible: true,
1034
+ bounds: [84, 2369, 1196, 2513],
1035
+ resourceId: null,
1036
+ state: { selected: true, enabled: true }
1037
+ }
1038
+ ]
1039
+ })
1040
+
1041
+ ;(Observe as any).ToolsObserve.getUITreeHandler = async () => inferredLabelTree()
1042
+ ;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
1043
+ tapCalls.push({ platform, x, y, deviceId })
1044
+ return {
1045
+ device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
1046
+ success: true,
1047
+ x,
1048
+ y
1049
+ }
1050
+ }
1051
+ ;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
1052
+ swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
1053
+ inferredLabelTreeValue = 5
1054
+ return {
1055
+ device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
1056
+ success: true,
1057
+ start: [x1, y1],
1058
+ end: [x2, y2],
1059
+ duration
1060
+ }
1061
+ }
1062
+
1063
+ const inferredTapStart = tapCalls.length
1064
+ const inferredSwipeStart = swipeCalls.length
1065
+ const inferredLabelAdjust = await ToolsInteract.adjustControlHandler({
1066
+ selector: { text: 'Duration' },
1067
+ property: 'value',
1068
+ targetValue: 5,
1069
+ tolerance: 0.5,
1070
+ platform: 'android'
1071
+ })
1072
+
1073
+ assert.strictEqual(inferredLabelAdjust.success, true)
1074
+ assert.strictEqual(inferredLabelAdjust.adjustment_mode, 'gesture')
1075
+ assert.strictEqual(inferredLabelAdjust.target_state.target_value, 5)
1076
+ assert.strictEqual(inferredLabelAdjust.actual_state?.value, 5)
1077
+ assert.strictEqual(tapCalls.length, inferredTapStart + 1)
1078
+ assert.strictEqual(swipeCalls.length, inferredSwipeStart + 1)
1079
+ assert.strictEqual(swipeCalls[swipeCalls.length - 1].duration, 520)
1080
+ assert.ok(tapCalls[tapCalls.length - 1].y < 2369, 'inferred slider tap should avoid the clickable view below the min/max labels')
1081
+ assert.ok(swipeCalls[swipeCalls.length - 1].y1 < 2369, 'inferred slider drag should start above the clickable view below the min/max labels')
1082
+ assert.ok(swipeCalls[swipeCalls.length - 1].y2 < 2369, 'inferred slider drag should end above the clickable view below the min/max labels')
531
1083
 
532
1084
  console.log('adjust_control unit tests passed')
533
1085
  } finally {
@@ -536,6 +1088,7 @@ async function run() {
536
1088
  ;(ToolsInteract as any).tapHandler = originalTapHandler
537
1089
  ;(ToolsInteract as any).swipeHandler = originalSwipeHandler
538
1090
  ;(ToolsInteract as any).expectStateHandler = originalExpectStateHandler
1091
+ ;(ToolsInteract as any).getInteractionService = originalGetInteractionService
539
1092
  }
540
1093
  }
541
1094
 
@@ -70,6 +70,10 @@ async function run() {
70
70
  const adjustControl = toolDefinitions.find((tool) => tool.name === 'adjust_control')
71
71
  assert(adjustControl, 'adjust_control should be registered')
72
72
  assert.deepStrictEqual((adjustControl as any).inputSchema.required, ['targetValue'])
73
+ assert.deepStrictEqual((adjustControl as any).inputSchema.oneOf, [
74
+ { required: ['selector'] },
75
+ { required: ['element_id'] }
76
+ ])
73
77
  assert.strictEqual((adjustControl as any).inputSchema.properties.targetValue.type, 'number')
74
78
  assert.match((adjustControl as any).description, /numeric control value/i)
75
79
  assert.match((adjustControl as any).description, /expect_state/i)