mobile-debug-mcp 0.26.3 → 0.26.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +456 -0
- package/dist/observe/ios.js +1 -1
- package/dist/server/tool-definitions.js +56 -0
- package/dist/server/tool-handlers.js +25 -0
- package/dist/server-core.js +1 -1
- package/dist/utils/android/utils.js +2 -2
- package/docs/CHANGELOG.md +3 -0
- package/docs/ROADMAP.md +64 -9
- package/docs/rfcs/008-adjustable-control-support-and-semantic-value-manipulation.md +273 -0
- package/docs/specs/mcp-tooling-spec-v1.md +1 -1
- package/docs/tools/interact.md +21 -0
- package/package.json +1 -1
- package/src/interact/index.ts +585 -0
- package/src/observe/ios.ts +1 -1
- package/src/server/tool-definitions.ts +56 -0
- package/src/server/tool-handlers.ts +26 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +17 -0
- package/src/utils/android/utils.ts +2 -2
- package/test/unit/interact/adjust_control.test.ts +365 -0
- package/test/unit/observe/state_extraction.test.ts +24 -0
- package/test/unit/server/contract.test.ts +8 -0
- package/test/unit/server/response_shapes.test.ts +39 -0
package/dist/interact/index.js
CHANGED
|
@@ -219,6 +219,80 @@ export class ToolsInteract {
|
|
|
219
219
|
reason: candidate.reason
|
|
220
220
|
};
|
|
221
221
|
}
|
|
222
|
+
static _isAdjustableControl(el) {
|
|
223
|
+
if (!el)
|
|
224
|
+
return false;
|
|
225
|
+
const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
|
|
226
|
+
const role = ToolsInteract._normalize(el.role ?? '');
|
|
227
|
+
return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
|
|
228
|
+
}
|
|
229
|
+
static _readNumericControlValue(el, property) {
|
|
230
|
+
if (!el?.state)
|
|
231
|
+
return null;
|
|
232
|
+
const stateValue = el.state[property];
|
|
233
|
+
if (typeof stateValue === 'number' && Number.isFinite(stateValue))
|
|
234
|
+
return stateValue;
|
|
235
|
+
if (property === 'value' || property === 'raw_value') {
|
|
236
|
+
const fallback = el.state.raw_value ?? el.state.value;
|
|
237
|
+
if (typeof fallback === 'number' && Number.isFinite(fallback))
|
|
238
|
+
return fallback;
|
|
239
|
+
}
|
|
240
|
+
return null;
|
|
241
|
+
}
|
|
242
|
+
static _buildControlPoint(bounds, ratio, axis) {
|
|
243
|
+
const clampedRatio = Math.max(0, Math.min(1, ratio));
|
|
244
|
+
const [left, top, right, bottom] = bounds;
|
|
245
|
+
const width = Math.max(1, right - left);
|
|
246
|
+
const height = Math.max(1, bottom - top);
|
|
247
|
+
const insetX = Math.max(8, Math.floor(width * 0.08));
|
|
248
|
+
const insetY = Math.max(8, Math.floor(height * 0.08));
|
|
249
|
+
if (axis === 'vertical') {
|
|
250
|
+
const usableHeight = Math.max(1, height - (insetY * 2));
|
|
251
|
+
return {
|
|
252
|
+
x: Math.floor((left + right) / 2),
|
|
253
|
+
y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
const usableWidth = Math.max(1, width - (insetX * 2));
|
|
257
|
+
return {
|
|
258
|
+
x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
|
|
259
|
+
y: Math.floor((top + bottom) / 2)
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
static _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis) {
|
|
263
|
+
const range = Math.max(1, max - min);
|
|
264
|
+
const targetRatio = (targetValue - min) / range;
|
|
265
|
+
const stepRatio = 1 / range;
|
|
266
|
+
const centerBias = stepRatio / 2;
|
|
267
|
+
const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue);
|
|
268
|
+
const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0]);
|
|
269
|
+
const edgeWindow = Math.max(3, Math.floor(range * 0.1));
|
|
270
|
+
const isNearLowEdge = targetValue - min <= edgeWindow;
|
|
271
|
+
const isNearHighEdge = max - targetValue <= edgeWindow;
|
|
272
|
+
const directionBias = direction > 0
|
|
273
|
+
? -stepRatio * 0.15
|
|
274
|
+
: direction < 0
|
|
275
|
+
? stepRatio * 0.65
|
|
276
|
+
: 0;
|
|
277
|
+
const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx));
|
|
278
|
+
const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin);
|
|
279
|
+
const edgeBias = isNearLowEdge
|
|
280
|
+
? endpointMargin
|
|
281
|
+
: isNearHighEdge
|
|
282
|
+
? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
|
|
283
|
+
: 0;
|
|
284
|
+
const safeRatio = Math.min(1 - (endpointMargin * 0.25), Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias));
|
|
285
|
+
return ToolsInteract._buildControlPoint(bounds, safeRatio, axis);
|
|
286
|
+
}
|
|
287
|
+
static _controlAxis(el, bounds) {
|
|
288
|
+
const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
|
|
289
|
+
const role = ToolsInteract._normalize(el.role ?? '');
|
|
290
|
+
if (/vertical/.test(type) || /vertical/.test(role))
|
|
291
|
+
return 'vertical';
|
|
292
|
+
if (/horizontal/.test(type) || /horizontal/.test(role))
|
|
293
|
+
return 'horizontal';
|
|
294
|
+
return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal';
|
|
295
|
+
}
|
|
222
296
|
static _actionFailure(actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter, sourceModule = 'interact') {
|
|
223
297
|
return buildActionExecutionResult({
|
|
224
298
|
actionType,
|
|
@@ -410,6 +484,388 @@ export class ToolsInteract {
|
|
|
410
484
|
sourceModule: 'interact'
|
|
411
485
|
});
|
|
412
486
|
}
|
|
487
|
+
static async adjustControlHandler({ selector, element_id, property = 'value', targetValue, tolerance = 0, maxAttempts = 3, platform, deviceId }) {
|
|
488
|
+
const actionType = 'adjust_control';
|
|
489
|
+
const targetSelector = selector ?? (element_id ? { elementId: element_id } : null);
|
|
490
|
+
const normalizedTolerance = Number.isFinite(tolerance) ? Math.max(0, tolerance) : 0;
|
|
491
|
+
const attemptsLimit = Math.max(1, Math.floor(Number(maxAttempts) || 1));
|
|
492
|
+
const sourcePlatform = platform || 'android';
|
|
493
|
+
let resolvedPlatform = sourcePlatform;
|
|
494
|
+
let resolvedDeviceId = deviceId;
|
|
495
|
+
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
496
|
+
let semanticFallbackElement = null;
|
|
497
|
+
const buildFailure = (failureCode, reason, resolved, device, actualState, attempts, adjustmentMode = 'gesture', retryable = false, uiFingerprintAfter = null) => {
|
|
498
|
+
const base = buildActionExecutionResult({
|
|
499
|
+
actionType,
|
|
500
|
+
sourceModule: 'interact',
|
|
501
|
+
device,
|
|
502
|
+
selector: targetSelector,
|
|
503
|
+
resolved,
|
|
504
|
+
success: false,
|
|
505
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
506
|
+
uiFingerprintAfter,
|
|
507
|
+
failure: { failureCode, retryable },
|
|
508
|
+
details: {
|
|
509
|
+
target_value: targetValue,
|
|
510
|
+
tolerance: normalizedTolerance,
|
|
511
|
+
property,
|
|
512
|
+
attempts,
|
|
513
|
+
adjustment_mode: adjustmentMode,
|
|
514
|
+
actual_state: actualState,
|
|
515
|
+
converged: false,
|
|
516
|
+
within_tolerance: false,
|
|
517
|
+
reason
|
|
518
|
+
}
|
|
519
|
+
});
|
|
520
|
+
return {
|
|
521
|
+
...base,
|
|
522
|
+
target_state: {
|
|
523
|
+
property,
|
|
524
|
+
target_value: targetValue,
|
|
525
|
+
tolerance: normalizedTolerance
|
|
526
|
+
},
|
|
527
|
+
actual_state: actualState,
|
|
528
|
+
within_tolerance: false,
|
|
529
|
+
converged: false,
|
|
530
|
+
attempts,
|
|
531
|
+
adjustment_mode: adjustmentMode
|
|
532
|
+
};
|
|
533
|
+
};
|
|
534
|
+
const resolveCurrentMatch = async () => {
|
|
535
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
|
|
536
|
+
resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform;
|
|
537
|
+
resolvedDeviceId = tree?.device?.id || resolvedDeviceId;
|
|
538
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
539
|
+
if (element_id) {
|
|
540
|
+
const stored = ToolsInteract._resolvedUiElements.get(element_id);
|
|
541
|
+
if (!stored) {
|
|
542
|
+
return null;
|
|
543
|
+
}
|
|
544
|
+
const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored);
|
|
545
|
+
if (!current) {
|
|
546
|
+
return null;
|
|
547
|
+
}
|
|
548
|
+
return {
|
|
549
|
+
tree,
|
|
550
|
+
device: tree?.device,
|
|
551
|
+
match: { el: current.el, idx: current.index },
|
|
552
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index), current.el, current.index)
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
if (semanticFallbackElement) {
|
|
556
|
+
const fallbackBounds = ToolsInteract._normalizeBounds(Array.isArray(semanticFallbackElement.bounds)
|
|
557
|
+
? semanticFallbackElement.bounds
|
|
558
|
+
: semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
|
|
559
|
+
? [
|
|
560
|
+
Number(semanticFallbackElement.bounds.left),
|
|
561
|
+
Number(semanticFallbackElement.bounds.top),
|
|
562
|
+
Number(semanticFallbackElement.bounds.right),
|
|
563
|
+
Number(semanticFallbackElement.bounds.bottom)
|
|
564
|
+
]
|
|
565
|
+
: null);
|
|
566
|
+
let matchedIndex = -1;
|
|
567
|
+
if (fallbackBounds) {
|
|
568
|
+
matchedIndex = elements.findIndex((el) => {
|
|
569
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
570
|
+
return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3];
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
if (matchedIndex === -1 && fallbackBounds) {
|
|
574
|
+
const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2);
|
|
575
|
+
const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2);
|
|
576
|
+
let bestDistance = Infinity;
|
|
577
|
+
for (let i = 0; i < elements.length; i++) {
|
|
578
|
+
const el = elements[i];
|
|
579
|
+
if (!ToolsInteract._isAdjustableControl(el))
|
|
580
|
+
continue;
|
|
581
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
582
|
+
if (!bounds)
|
|
583
|
+
continue;
|
|
584
|
+
const centerX = Math.floor((bounds[0] + bounds[2]) / 2);
|
|
585
|
+
const centerY = Math.floor((bounds[1] + bounds[3]) / 2);
|
|
586
|
+
const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY);
|
|
587
|
+
if (distance < bestDistance) {
|
|
588
|
+
bestDistance = distance;
|
|
589
|
+
matchedIndex = i;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
if (matchedIndex >= 0 && elements[matchedIndex]) {
|
|
594
|
+
const matched = { el: elements[matchedIndex], idx: matchedIndex };
|
|
595
|
+
return {
|
|
596
|
+
tree,
|
|
597
|
+
device: tree?.device,
|
|
598
|
+
match: matched,
|
|
599
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
if (selector) {
|
|
604
|
+
const matched = ToolsInteract._findFirstMatchingElement(elements, selector);
|
|
605
|
+
if (!matched) {
|
|
606
|
+
return null;
|
|
607
|
+
}
|
|
608
|
+
return {
|
|
609
|
+
tree,
|
|
610
|
+
device: tree?.device,
|
|
611
|
+
match: matched,
|
|
612
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
return null;
|
|
616
|
+
};
|
|
617
|
+
if (!selector && !element_id) {
|
|
618
|
+
return buildFailure('ELEMENT_NOT_FOUND', 'selector or element_id is required', null, undefined, null, 0, 'gesture', false);
|
|
619
|
+
}
|
|
620
|
+
if (selector && !element_id) {
|
|
621
|
+
const waitResult = await ToolsInteract.waitForUIHandler({
|
|
622
|
+
selector,
|
|
623
|
+
condition: 'clickable',
|
|
624
|
+
timeout_ms: 5000,
|
|
625
|
+
poll_interval_ms: 300,
|
|
626
|
+
platform: resolvedPlatform,
|
|
627
|
+
deviceId: resolvedDeviceId
|
|
628
|
+
});
|
|
629
|
+
if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
|
|
630
|
+
const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? '';
|
|
631
|
+
if (!semanticQuery) {
|
|
632
|
+
return buildFailure(waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', waitResult?.error?.code === 'ELEMENT_NOT_FOUND');
|
|
633
|
+
}
|
|
634
|
+
const fallback = await ToolsInteract.findElementHandler({
|
|
635
|
+
query: semanticQuery,
|
|
636
|
+
exact: false,
|
|
637
|
+
timeoutMs: 3000,
|
|
638
|
+
platform: resolvedPlatform,
|
|
639
|
+
deviceId: resolvedDeviceId
|
|
640
|
+
});
|
|
641
|
+
if (!fallback.found || !fallback.element) {
|
|
642
|
+
return buildFailure('ELEMENT_NOT_FOUND', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', true);
|
|
643
|
+
}
|
|
644
|
+
semanticFallbackElement = fallback.element;
|
|
645
|
+
}
|
|
646
|
+
else {
|
|
647
|
+
element_id = waitResult.element.elementId;
|
|
648
|
+
semanticFallbackElement = null;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
let lastObservedState = null;
|
|
652
|
+
let lastAdjustmentMode = 'gesture';
|
|
653
|
+
let resolvedTarget = null;
|
|
654
|
+
let currentDevice = undefined;
|
|
655
|
+
let attemptCount = 0;
|
|
656
|
+
let cachedResolvedMatch = null;
|
|
657
|
+
for (let attempt = 0; attempt < attemptsLimit; attempt++) {
|
|
658
|
+
const resolved = cachedResolvedMatch
|
|
659
|
+
? {
|
|
660
|
+
tree: null,
|
|
661
|
+
device: currentDevice,
|
|
662
|
+
match: cachedResolvedMatch,
|
|
663
|
+
resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx), cachedResolvedMatch.el, cachedResolvedMatch.idx)
|
|
664
|
+
}
|
|
665
|
+
: await resolveCurrentMatch();
|
|
666
|
+
if (!resolved || !resolved.match || !resolved.resolvedTarget) {
|
|
667
|
+
return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
|
|
668
|
+
}
|
|
669
|
+
currentDevice = resolved.device;
|
|
670
|
+
resolvedTarget = resolved.resolvedTarget;
|
|
671
|
+
const currentEl = resolved.match.el;
|
|
672
|
+
cachedResolvedMatch = resolved.match;
|
|
673
|
+
const bounds = ToolsInteract._normalizeBounds(currentEl.bounds);
|
|
674
|
+
const valueRange = currentEl.state?.value_range ?? null;
|
|
675
|
+
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property);
|
|
676
|
+
const actualState = currentValue !== null
|
|
677
|
+
? { property, value: currentValue, raw_value: typeof currentEl.state?.raw_value === 'number' ? currentEl.state.raw_value : undefined }
|
|
678
|
+
: null;
|
|
679
|
+
lastObservedState = actualState;
|
|
680
|
+
if (property !== 'value' && property !== 'raw_value') {
|
|
681
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
682
|
+
}
|
|
683
|
+
if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
|
|
684
|
+
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
685
|
+
const base = buildActionExecutionResult({
|
|
686
|
+
actionType,
|
|
687
|
+
sourceModule: 'interact',
|
|
688
|
+
device: currentDevice,
|
|
689
|
+
selector: targetSelector,
|
|
690
|
+
resolved: resolvedTarget,
|
|
691
|
+
success: true,
|
|
692
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
693
|
+
uiFingerprintAfter,
|
|
694
|
+
details: {
|
|
695
|
+
target_value: targetValue,
|
|
696
|
+
tolerance: normalizedTolerance,
|
|
697
|
+
property,
|
|
698
|
+
attempts: attemptCount,
|
|
699
|
+
adjustment_mode: 'semantic',
|
|
700
|
+
actual_state: actualState,
|
|
701
|
+
converged: true,
|
|
702
|
+
within_tolerance: true,
|
|
703
|
+
reason: 'control already within tolerance'
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
return {
|
|
707
|
+
...base,
|
|
708
|
+
target_state: {
|
|
709
|
+
property,
|
|
710
|
+
target_value: targetValue,
|
|
711
|
+
tolerance: normalizedTolerance
|
|
712
|
+
},
|
|
713
|
+
actual_state: actualState,
|
|
714
|
+
within_tolerance: true,
|
|
715
|
+
converged: true,
|
|
716
|
+
attempts: attemptCount,
|
|
717
|
+
adjustment_mode: 'semantic'
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
if (!ToolsInteract._isAdjustableControl(currentEl)) {
|
|
721
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
722
|
+
}
|
|
723
|
+
if (!bounds) {
|
|
724
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
725
|
+
}
|
|
726
|
+
const min = typeof valueRange?.min === 'number' ? valueRange.min : null;
|
|
727
|
+
const max = typeof valueRange?.max === 'number' ? valueRange.max : null;
|
|
728
|
+
if (min === null || max === null || !Number.isFinite(min) || !Number.isFinite(max) || max <= min) {
|
|
729
|
+
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
730
|
+
}
|
|
731
|
+
if (targetValue < min || targetValue > max) {
|
|
732
|
+
return buildFailure('UNKNOWN', `targetValue ${targetValue} is outside the control range ${min}..${max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
|
|
733
|
+
}
|
|
734
|
+
const axis = ToolsInteract._controlAxis(currentEl, bounds);
|
|
735
|
+
const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis);
|
|
736
|
+
const currentPoint = currentValue !== null
|
|
737
|
+
? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
|
|
738
|
+
: ToolsInteract._buildControlPoint(bounds, 0.5, axis);
|
|
739
|
+
const runVerification = async () => {
|
|
740
|
+
const verification = await ToolsInteract.expectStateHandler({
|
|
741
|
+
element_id: resolvedTarget?.elementId ?? element_id,
|
|
742
|
+
selector: selector ?? undefined,
|
|
743
|
+
property,
|
|
744
|
+
expected: targetValue,
|
|
745
|
+
platform: resolvedPlatform,
|
|
746
|
+
deviceId: resolvedDeviceId
|
|
747
|
+
});
|
|
748
|
+
const observedValue = typeof verification?.observed_state?.value === 'number'
|
|
749
|
+
? verification.observed_state.value
|
|
750
|
+
: typeof verification?.observed_state?.raw_value === 'number'
|
|
751
|
+
? verification.observed_state.raw_value
|
|
752
|
+
: null;
|
|
753
|
+
const observedState = observedValue !== null
|
|
754
|
+
? {
|
|
755
|
+
property,
|
|
756
|
+
value: observedValue,
|
|
757
|
+
raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
|
|
758
|
+
}
|
|
759
|
+
: actualState;
|
|
760
|
+
return {
|
|
761
|
+
verification,
|
|
762
|
+
observedState,
|
|
763
|
+
withinTolerance: observedValue !== null && Math.abs(observedValue - targetValue) <= normalizedTolerance
|
|
764
|
+
};
|
|
765
|
+
};
|
|
766
|
+
lastAdjustmentMode = 'coordinate';
|
|
767
|
+
const primaryActionResult = await ToolsInteract.tapHandler({
|
|
768
|
+
platform: resolvedPlatform,
|
|
769
|
+
x: targetPoint.x,
|
|
770
|
+
y: targetPoint.y,
|
|
771
|
+
deviceId: resolvedDeviceId
|
|
772
|
+
});
|
|
773
|
+
let actionDevice = primaryActionResult.device ?? currentDevice;
|
|
774
|
+
attemptCount++;
|
|
775
|
+
if (!primaryActionResult.success) {
|
|
776
|
+
lastAdjustmentMode = 'gesture';
|
|
777
|
+
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
778
|
+
platform: resolvedPlatform,
|
|
779
|
+
x1: currentPoint.x,
|
|
780
|
+
y1: currentPoint.y,
|
|
781
|
+
x2: targetPoint.x,
|
|
782
|
+
y2: targetPoint.y,
|
|
783
|
+
duration: 220,
|
|
784
|
+
deviceId: resolvedDeviceId
|
|
785
|
+
});
|
|
786
|
+
attemptCount++;
|
|
787
|
+
if (!fallbackActionResult.success) {
|
|
788
|
+
return buildFailure('UNKNOWN', fallbackActionResult.error ?? primaryActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? primaryActionResult.device, actualState, attemptCount, lastAdjustmentMode, false);
|
|
789
|
+
}
|
|
790
|
+
actionDevice = fallbackActionResult.device ?? actionDevice;
|
|
791
|
+
}
|
|
792
|
+
let verificationResult = await runVerification();
|
|
793
|
+
let observedState = verificationResult.observedState;
|
|
794
|
+
lastObservedState = observedState;
|
|
795
|
+
if (!verificationResult.withinTolerance && currentValue !== null) {
|
|
796
|
+
lastAdjustmentMode = 'gesture';
|
|
797
|
+
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
798
|
+
platform: resolvedPlatform,
|
|
799
|
+
x1: currentPoint.x,
|
|
800
|
+
y1: currentPoint.y,
|
|
801
|
+
x2: targetPoint.x,
|
|
802
|
+
y2: targetPoint.y,
|
|
803
|
+
duration: 220,
|
|
804
|
+
deviceId: resolvedDeviceId
|
|
805
|
+
});
|
|
806
|
+
attemptCount++;
|
|
807
|
+
if (!fallbackActionResult.success) {
|
|
808
|
+
return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device, observedState ?? actualState, attemptCount, lastAdjustmentMode, false);
|
|
809
|
+
}
|
|
810
|
+
verificationResult = await runVerification();
|
|
811
|
+
observedState = verificationResult.observedState;
|
|
812
|
+
}
|
|
813
|
+
const verification = verificationResult.verification;
|
|
814
|
+
lastObservedState = observedState;
|
|
815
|
+
if (verificationResult.withinTolerance) {
|
|
816
|
+
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
817
|
+
const base = buildActionExecutionResult({
|
|
818
|
+
actionType,
|
|
819
|
+
sourceModule: 'interact',
|
|
820
|
+
device: actionDevice ?? currentDevice,
|
|
821
|
+
selector: targetSelector,
|
|
822
|
+
resolved: resolvedTarget,
|
|
823
|
+
success: true,
|
|
824
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
825
|
+
uiFingerprintAfter,
|
|
826
|
+
details: {
|
|
827
|
+
target_value: targetValue,
|
|
828
|
+
tolerance: normalizedTolerance,
|
|
829
|
+
property,
|
|
830
|
+
attempts: attemptCount,
|
|
831
|
+
adjustment_mode: lastAdjustmentMode,
|
|
832
|
+
actual_state: observedState,
|
|
833
|
+
converged: true,
|
|
834
|
+
within_tolerance: true,
|
|
835
|
+
reason: verification?.reason ?? 'control converged to target value'
|
|
836
|
+
}
|
|
837
|
+
});
|
|
838
|
+
return {
|
|
839
|
+
...base,
|
|
840
|
+
target_state: {
|
|
841
|
+
property,
|
|
842
|
+
target_value: targetValue,
|
|
843
|
+
tolerance: normalizedTolerance
|
|
844
|
+
},
|
|
845
|
+
actual_state: observedState,
|
|
846
|
+
within_tolerance: true,
|
|
847
|
+
converged: true,
|
|
848
|
+
attempts: attemptCount,
|
|
849
|
+
adjustment_mode: lastAdjustmentMode
|
|
850
|
+
};
|
|
851
|
+
}
|
|
852
|
+
cachedResolvedMatch = {
|
|
853
|
+
el: {
|
|
854
|
+
...currentEl,
|
|
855
|
+
state: {
|
|
856
|
+
...(currentEl.state ?? null),
|
|
857
|
+
...(observedState ? {
|
|
858
|
+
[observedState.property]: observedState.value,
|
|
859
|
+
raw_value: observedState.raw_value ?? observedState.value
|
|
860
|
+
} : {})
|
|
861
|
+
}
|
|
862
|
+
},
|
|
863
|
+
idx: resolved.match.idx
|
|
864
|
+
};
|
|
865
|
+
}
|
|
866
|
+
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
867
|
+
return buildFailure('TIMEOUT', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter);
|
|
868
|
+
}
|
|
413
869
|
static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
|
|
414
870
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
415
871
|
return await interact.swipe(x1, y1, x2, y2, duration, resolved.id);
|
package/dist/observe/ios.js
CHANGED
|
@@ -106,7 +106,7 @@ function buildIOSSemantic(type, traits) {
|
|
|
106
106
|
};
|
|
107
107
|
}
|
|
108
108
|
function isIOSAdjustable(node, type, traits) {
|
|
109
|
-
return /slider|adjustable|stepper
|
|
109
|
+
return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait));
|
|
110
110
|
}
|
|
111
111
|
function extractIOSState(node, type, label, value, traits) {
|
|
112
112
|
const state = {};
|
|
@@ -555,6 +555,62 @@ Failure Handling:
|
|
|
555
555
|
required: ['property', 'expected']
|
|
556
556
|
}
|
|
557
557
|
},
|
|
558
|
+
{
|
|
559
|
+
name: 'adjust_control',
|
|
560
|
+
description: `Purpose:
|
|
561
|
+
Adjust a numeric control value with verification.
|
|
562
|
+
|
|
563
|
+
This is the initial adjustable-control surface for slider-like controls and other controls that expose a numeric value or value_range.
|
|
564
|
+
|
|
565
|
+
Inputs:
|
|
566
|
+
- selector or element_id
|
|
567
|
+
- property (defaults to "value")
|
|
568
|
+
- targetValue
|
|
569
|
+
- tolerance (optional)
|
|
570
|
+
- maxAttempts (optional)
|
|
571
|
+
- platform/deviceId (optional)
|
|
572
|
+
|
|
573
|
+
Output Structure:
|
|
574
|
+
- action_id, timestamp (ISO 8601), action_type
|
|
575
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
576
|
+
- source_module: runtime source of the action envelope
|
|
577
|
+
- target_state / actual_state / within_tolerance / converged / attempts / adjustment_mode
|
|
578
|
+
- target.selector = original selector or element handle
|
|
579
|
+
- success = true when the control converges within tolerance
|
|
580
|
+
|
|
581
|
+
Verification Guidance:
|
|
582
|
+
- Prefer direct target placement when value_range is available; fall back to a drag only if the direct tap does not converge
|
|
583
|
+
- Use expect_state for the control value readback
|
|
584
|
+
- Treat coordinate fallback as degraded mode
|
|
585
|
+
|
|
586
|
+
Failure Handling:
|
|
587
|
+
- ELEMENT_NOT_FOUND → re-resolve the control
|
|
588
|
+
- ELEMENT_NOT_INTERACTABLE → the control cannot be adjusted through the current runtime
|
|
589
|
+
- TIMEOUT → the control did not converge within bounded retries
|
|
590
|
+
- UNKNOWN → capture a snapshot and stop`,
|
|
591
|
+
inputSchema: {
|
|
592
|
+
type: 'object',
|
|
593
|
+
properties: {
|
|
594
|
+
selector: {
|
|
595
|
+
type: 'object',
|
|
596
|
+
properties: {
|
|
597
|
+
text: { type: 'string' },
|
|
598
|
+
resource_id: { type: 'string' },
|
|
599
|
+
accessibility_id: { type: 'string' },
|
|
600
|
+
contains: { type: 'boolean', default: false }
|
|
601
|
+
}
|
|
602
|
+
},
|
|
603
|
+
element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
|
|
604
|
+
property: { type: 'string', description: 'Readable numeric state property to adjust.', default: 'value' },
|
|
605
|
+
targetValue: { type: 'number', description: 'Target numeric value.' },
|
|
606
|
+
tolerance: { type: 'number', description: 'Accepted numeric tolerance around the target value.', default: 0 },
|
|
607
|
+
maxAttempts: { type: 'number', description: 'Maximum adjustment attempts.', default: 3 },
|
|
608
|
+
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
|
|
609
|
+
deviceId: { type: 'string', description: 'Optional device serial/udid' }
|
|
610
|
+
},
|
|
611
|
+
required: ['targetValue']
|
|
612
|
+
}
|
|
613
|
+
},
|
|
558
614
|
{
|
|
559
615
|
name: 'wait_for_ui',
|
|
560
616
|
description: `Purpose:
|
|
@@ -226,6 +226,30 @@ async function handleExpectState(args) {
|
|
|
226
226
|
const res = await ToolsInteract.expectStateHandler({ selector: selector ?? undefined, element_id: element_id ?? undefined, property, expected, platform, deviceId });
|
|
227
227
|
return wrapResponse(res);
|
|
228
228
|
}
|
|
229
|
+
async function handleAdjustControl(args) {
|
|
230
|
+
const selector = getObjectArg(args, 'selector');
|
|
231
|
+
const element_id = getStringArg(args, 'element_id');
|
|
232
|
+
const property = getStringArg(args, 'property') ?? 'value';
|
|
233
|
+
const targetValue = requireNumberArg(args, 'targetValue');
|
|
234
|
+
const tolerance = getNumberArg(args, 'tolerance') ?? 0;
|
|
235
|
+
const maxAttempts = getNumberArg(args, 'maxAttempts') ?? 3;
|
|
236
|
+
const platform = getStringArg(args, 'platform');
|
|
237
|
+
const deviceId = getStringArg(args, 'deviceId');
|
|
238
|
+
if (!selector && !element_id) {
|
|
239
|
+
throw new Error('Missing selector or element_id argument');
|
|
240
|
+
}
|
|
241
|
+
const res = await ToolsInteract.adjustControlHandler({
|
|
242
|
+
selector: selector ?? undefined,
|
|
243
|
+
element_id: element_id ?? undefined,
|
|
244
|
+
property,
|
|
245
|
+
targetValue,
|
|
246
|
+
tolerance,
|
|
247
|
+
maxAttempts,
|
|
248
|
+
platform,
|
|
249
|
+
deviceId
|
|
250
|
+
});
|
|
251
|
+
return wrapResponse(res);
|
|
252
|
+
}
|
|
229
253
|
async function handleWaitForUI(args) {
|
|
230
254
|
const selector = getObjectArg(args, 'selector');
|
|
231
255
|
const condition = getStringArg(args, 'condition') ?? 'exists';
|
|
@@ -431,6 +455,7 @@ export const toolHandlers = {
|
|
|
431
455
|
expect_screen: handleExpectScreen,
|
|
432
456
|
expect_element_visible: handleExpectElementVisible,
|
|
433
457
|
expect_state: handleExpectState,
|
|
458
|
+
adjust_control: handleAdjustControl,
|
|
434
459
|
wait_for_ui: handleWaitForUI,
|
|
435
460
|
find_element: handleFindElement,
|
|
436
461
|
tap: handleTap,
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.26.
|
|
9
|
+
version: '0.26.4'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|
|
@@ -360,7 +360,7 @@ function normalizeClassName(value) {
|
|
|
360
360
|
return typeof value === 'string' ? value.trim().toLowerCase() : '';
|
|
361
361
|
}
|
|
362
362
|
function inferAndroidRole(className) {
|
|
363
|
-
if (/seekbar|slider
|
|
363
|
+
if (/seekbar|slider/.test(className))
|
|
364
364
|
return 'slider';
|
|
365
365
|
if (/switch|toggle/.test(className))
|
|
366
366
|
return 'switch';
|
|
@@ -411,7 +411,7 @@ function buildAndroidSemantic(clickable, className) {
|
|
|
411
411
|
}
|
|
412
412
|
function isSliderLikeAndroid(node) {
|
|
413
413
|
const className = String(node['@_class'] || '').toLowerCase();
|
|
414
|
-
return /seekbar|slider|range
|
|
414
|
+
return /seekbar|slider|range/i.test(className);
|
|
415
415
|
}
|
|
416
416
|
function extractAndroidState(node) {
|
|
417
417
|
const checked = parseBooleanAttr(node['@_checked']);
|
package/docs/CHANGELOG.md
CHANGED