mobile-debug-mcp 0.24.7 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/interact/index.js +302 -6
- package/dist/observe/ios.js +56 -2
- package/dist/server/common.js +2 -1
- package/dist/server/tool-definitions.js +55 -0
- package/dist/server/tool-handlers.js +17 -0
- package/dist/server-core.js +1 -1
- package/dist/utils/android/utils.js +67 -1
- package/docs/CHANGELOG.md +6 -0
- package/docs/ROADMAP.md +388 -0
- package/docs/rfcs/001-state-verification.md +452 -0
- package/docs/specs/mcp-tooling-spec-v1.md +4 -0
- package/docs/tools/interact.md +25 -0
- package/docs/tools/observe.md +2 -1
- package/package.json +1 -1
- package/src/interact/index.ts +352 -7
- package/src/observe/ios.ts +62 -3
- package/src/server/common.ts +2 -1
- package/src/server/tool-definitions.ts +55 -0
- package/src/server/tool-handlers.ts +18 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +41 -0
- package/src/utils/android/utils.ts +78 -14
- package/test/unit/observe/find_element.test.ts +64 -5
- package/test/unit/observe/state_extraction.test.ts +43 -0
- package/test/unit/server/response_shapes.test.ts +40 -2
package/README.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# Mobile Debug Tools
|
|
2
2
|
|
|
3
|
-
A minimal, secure MCP server for AI-assisted mobile development. Build, install, and inspect Android/iOS apps from an MCP-compatible client.
|
|
3
|
+
A minimal, secure MCP server for AI-assisted mobile development. Build, install, interact and inspect Android/iOS apps from an MCP-compatible client.
|
|
4
4
|
|
|
5
5
|
> **Support:**
|
|
6
|
-
> *
|
|
7
|
-
> *
|
|
8
|
-
> *
|
|
9
|
-
> * Flutter
|
|
10
|
-
> * React native not tested
|
|
6
|
+
> * KMP
|
|
7
|
+
> * Android
|
|
8
|
+
> * iOS
|
|
9
|
+
> * Flutter - not tested
|
|
10
|
+
> * React native - not tested
|
|
11
11
|
|
|
12
12
|
## Requirements
|
|
13
13
|
|
package/dist/interact/index.js
CHANGED
|
@@ -7,6 +7,14 @@ import { ToolsObserve } from '../observe/index.js';
|
|
|
7
7
|
import { nextActionId } from '../server/common.js';
|
|
8
8
|
export class ToolsInteract {
|
|
9
9
|
static _maxResolvedUiElements = 256;
|
|
10
|
+
static _sliderSearchLookahead = 8;
|
|
11
|
+
static _sliderNegativeGapTolerancePx = 32;
|
|
12
|
+
static _sliderPositiveGapLimitPx = 640;
|
|
13
|
+
static _sliderTrackMinLengthPx = 220;
|
|
14
|
+
static _sliderTrackMaxThicknessPx = 180;
|
|
15
|
+
static _sliderTrackLengthRatio = 0.18;
|
|
16
|
+
static _sliderTrackThicknessRatio = 0.08;
|
|
17
|
+
static _sliderLabelWidthRatio = 1.5;
|
|
10
18
|
static _resolvedUiElements = new Map();
|
|
11
19
|
static _normalize(s) {
|
|
12
20
|
if (s === null || s === undefined)
|
|
@@ -26,6 +34,43 @@ export class ToolsInteract {
|
|
|
26
34
|
return null;
|
|
27
35
|
return normalized;
|
|
28
36
|
}
|
|
37
|
+
static _matchesSelector(el, selector) {
|
|
38
|
+
if (!selector)
|
|
39
|
+
return false;
|
|
40
|
+
const normalize = ToolsInteract._normalize;
|
|
41
|
+
const containsFlag = !!selector.contains;
|
|
42
|
+
const text = normalize(el.text ?? el.label ?? el.value ?? '');
|
|
43
|
+
const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
|
|
44
|
+
const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '');
|
|
45
|
+
if (selector.text !== undefined && selector.text !== null) {
|
|
46
|
+
const q = normalize(selector.text);
|
|
47
|
+
if (containsFlag ? !text.includes(q) : text !== q)
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
if (selector.resource_id !== undefined && selector.resource_id !== null) {
|
|
51
|
+
const q = normalize(selector.resource_id);
|
|
52
|
+
if (containsFlag ? !resourceId.includes(q) : resourceId !== q)
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
|
|
56
|
+
const q = normalize(selector.accessibility_id);
|
|
57
|
+
if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q)
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
static _findFirstMatchingElement(elements, selector) {
|
|
63
|
+
if (!selector)
|
|
64
|
+
return null;
|
|
65
|
+
for (let i = 0; i < elements.length; i++) {
|
|
66
|
+
const el = elements[i];
|
|
67
|
+
if (!el)
|
|
68
|
+
continue;
|
|
69
|
+
if (ToolsInteract._matchesSelector(el, selector))
|
|
70
|
+
return { el, idx: i };
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
29
74
|
static _isVisibleElement(el) {
|
|
30
75
|
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
31
76
|
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
|
|
@@ -92,7 +137,8 @@ export class ToolsInteract {
|
|
|
92
137
|
accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
|
|
93
138
|
class: element.type ?? element.class ?? null,
|
|
94
139
|
bounds: ToolsInteract._normalizeBounds(element.bounds),
|
|
95
|
-
index
|
|
140
|
+
index,
|
|
141
|
+
state: element.state ?? null
|
|
96
142
|
};
|
|
97
143
|
}
|
|
98
144
|
static _actionFailure(actionId, timestamp, actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter) {
|
|
@@ -174,6 +220,63 @@ export class ToolsInteract {
|
|
|
174
220
|
}
|
|
175
221
|
return best;
|
|
176
222
|
}
|
|
223
|
+
static _resolveNearbyActionableControl(elements, chosen, screen) {
|
|
224
|
+
if (!chosen)
|
|
225
|
+
return null;
|
|
226
|
+
const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
|
|
227
|
+
if (!labelBounds)
|
|
228
|
+
return null;
|
|
229
|
+
const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds;
|
|
230
|
+
const labelWidth = labelRight - labelLeft;
|
|
231
|
+
const labelHeight = labelBottom - labelTop;
|
|
232
|
+
const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0;
|
|
233
|
+
const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0;
|
|
234
|
+
const minTrackLengthPx = Math.max(ToolsInteract._sliderTrackMinLengthPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0);
|
|
235
|
+
const maxTrackThicknessPx = Math.max(ToolsInteract._sliderTrackMaxThicknessPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0);
|
|
236
|
+
let best = null;
|
|
237
|
+
let bestScore = Infinity;
|
|
238
|
+
for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
|
|
239
|
+
const candidate = elements[i];
|
|
240
|
+
if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false)
|
|
241
|
+
continue;
|
|
242
|
+
const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds);
|
|
243
|
+
if (!candidateBounds)
|
|
244
|
+
continue;
|
|
245
|
+
const [left, top, right] = candidateBounds;
|
|
246
|
+
const width = right - left;
|
|
247
|
+
const height = candidateBounds[3] - top;
|
|
248
|
+
const verticalGap = top - labelBottom;
|
|
249
|
+
if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx)
|
|
250
|
+
continue;
|
|
251
|
+
const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left);
|
|
252
|
+
if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx)
|
|
253
|
+
continue;
|
|
254
|
+
const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '');
|
|
255
|
+
const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '');
|
|
256
|
+
const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '');
|
|
257
|
+
let score = verticalGap;
|
|
258
|
+
const horizontalTrackLike = width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
|
|
259
|
+
height <= maxTrackThicknessPx;
|
|
260
|
+
const verticalTrackLike = height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
|
|
261
|
+
width <= maxTrackThicknessPx;
|
|
262
|
+
const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike;
|
|
263
|
+
if (!candidateText && !candidateContent)
|
|
264
|
+
score -= 18;
|
|
265
|
+
if (trackLike)
|
|
266
|
+
score -= 30;
|
|
267
|
+
if (/view|layout|group|frame/i.test(candidateClass))
|
|
268
|
+
score -= 10;
|
|
269
|
+
if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio)
|
|
270
|
+
score -= 8;
|
|
271
|
+
if (candidateText || candidateContent)
|
|
272
|
+
score += 20;
|
|
273
|
+
if (score < bestScore) {
|
|
274
|
+
bestScore = score;
|
|
275
|
+
best = { el: candidate, idx: i, sliderLike: trackLike };
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return best;
|
|
279
|
+
}
|
|
177
280
|
static async getInteractionService(platform, deviceId) {
|
|
178
281
|
const effectivePlatform = platform || 'android';
|
|
179
282
|
const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
|
|
@@ -262,6 +365,7 @@ export class ToolsInteract {
|
|
|
262
365
|
return { found: false, error: 'Empty query' };
|
|
263
366
|
let best = null;
|
|
264
367
|
let bestScore = 0;
|
|
368
|
+
let lastTree = null;
|
|
265
369
|
const scoreElement = (el) => {
|
|
266
370
|
if (!el || !el.visible)
|
|
267
371
|
return 0;
|
|
@@ -305,6 +409,7 @@ export class ToolsInteract {
|
|
|
305
409
|
while (Date.now() <= deadline) {
|
|
306
410
|
try {
|
|
307
411
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
412
|
+
lastTree = tree;
|
|
308
413
|
if (tree && Array.isArray(tree.elements)) {
|
|
309
414
|
const elements = tree.elements;
|
|
310
415
|
for (let i = 0; i < elements.length; i++) {
|
|
@@ -342,8 +447,8 @@ export class ToolsInteract {
|
|
|
342
447
|
return { found: false, error: 'Element not found' };
|
|
343
448
|
// If the best match is not interactable, try to resolve an actionable ancestor.
|
|
344
449
|
try {
|
|
345
|
-
const
|
|
346
|
-
const
|
|
450
|
+
const elements = (lastTree && Array.isArray(lastTree.elements)) ? lastTree.elements : [];
|
|
451
|
+
const screen = lastTree?.resolution && typeof lastTree.resolution === 'object' ? lastTree.resolution : null;
|
|
347
452
|
let chosen = best;
|
|
348
453
|
const childBounds = Array.isArray(chosen?.bounds) ? chosen.bounds : null;
|
|
349
454
|
// Strategy 1: if parentId references an index, climb that chain
|
|
@@ -412,6 +517,15 @@ export class ToolsInteract {
|
|
|
412
517
|
// small score bump to reflect actionability
|
|
413
518
|
bestScore = Math.min(1, bestScore + 0.02);
|
|
414
519
|
}
|
|
520
|
+
if (best && !(best.clickable || best.focusable)) {
|
|
521
|
+
const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best, idx: best._index ?? elements.indexOf(best) }, screen);
|
|
522
|
+
if (nearbyActionable) {
|
|
523
|
+
best = nearbyActionable.el;
|
|
524
|
+
best._index = nearbyActionable.idx;
|
|
525
|
+
best._interactable = true;
|
|
526
|
+
best._sliderLike = nearbyActionable.sliderLike;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
415
529
|
}
|
|
416
530
|
catch (e) {
|
|
417
531
|
console.error('Error resolving ancestor:', e);
|
|
@@ -431,9 +545,19 @@ export class ToolsInteract {
|
|
|
431
545
|
tapCoordinates,
|
|
432
546
|
telemetry: {
|
|
433
547
|
matchedIndex: best?._index ?? null,
|
|
434
|
-
matchedInteractable: !!best?._interactable
|
|
548
|
+
matchedInteractable: !!best?._interactable,
|
|
549
|
+
sliderLike: !!best?._sliderLike
|
|
435
550
|
}
|
|
436
551
|
};
|
|
552
|
+
if (best?._sliderLike) {
|
|
553
|
+
const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left);
|
|
554
|
+
const interactionHint = {
|
|
555
|
+
kind: 'slider',
|
|
556
|
+
axis: isVertical ? 'vertical' : 'horizontal',
|
|
557
|
+
trackBounds: boundsObj
|
|
558
|
+
};
|
|
559
|
+
outEl.interactionHint = interactionHint;
|
|
560
|
+
}
|
|
437
561
|
const scoreVal = Math.min(1, Number(bestScore.toFixed(3)));
|
|
438
562
|
return { found: true, element: outEl, score: scoreVal, confidence: scoreVal };
|
|
439
563
|
}
|
|
@@ -815,7 +939,8 @@ export class ToolsInteract {
|
|
|
815
939
|
accessibility_id: result.element.accessibility_id ?? null,
|
|
816
940
|
class: result.element.class ?? null,
|
|
817
941
|
bounds: result.element.bounds ?? null,
|
|
818
|
-
index: typeof result.element.index === 'number' ? result.element.index : null
|
|
942
|
+
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
943
|
+
state: result.element.state ?? null
|
|
819
944
|
},
|
|
820
945
|
observed: {
|
|
821
946
|
status: result.status,
|
|
@@ -829,7 +954,8 @@ export class ToolsInteract {
|
|
|
829
954
|
accessibility_id: result.element.accessibility_id ?? null,
|
|
830
955
|
class: result.element.class ?? null,
|
|
831
956
|
bounds: result.element.bounds ?? null,
|
|
832
|
-
index: typeof result.element.index === 'number' ? result.element.index : null
|
|
957
|
+
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
958
|
+
state: result.element.state ?? null
|
|
833
959
|
}
|
|
834
960
|
},
|
|
835
961
|
reason: 'selector is visible'
|
|
@@ -853,6 +979,176 @@ export class ToolsInteract {
|
|
|
853
979
|
retryable: errorCode === 'TIMEOUT'
|
|
854
980
|
};
|
|
855
981
|
}
|
|
982
|
+
static async expectStateHandler({ selector, element_id, property, expected, platform, deviceId }) {
|
|
983
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
984
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
985
|
+
const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
|
|
986
|
+
const treeDeviceId = tree?.device?.id || deviceId;
|
|
987
|
+
let matched = null;
|
|
988
|
+
if (element_id) {
|
|
989
|
+
const resolved = ToolsInteract._resolvedUiElements.get(element_id);
|
|
990
|
+
if (resolved) {
|
|
991
|
+
const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
|
|
992
|
+
if (current)
|
|
993
|
+
matched = { el: current.el, idx: current.index };
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
if (!matched && selector) {
|
|
997
|
+
matched = ToolsInteract._findFirstMatchingElement(elements, selector);
|
|
998
|
+
}
|
|
999
|
+
if (!matched) {
|
|
1000
|
+
return {
|
|
1001
|
+
success: false,
|
|
1002
|
+
selector,
|
|
1003
|
+
element_id: element_id ?? null,
|
|
1004
|
+
expected_state: { property, expected },
|
|
1005
|
+
reason: 'element not found',
|
|
1006
|
+
failure_code: 'ELEMENT_NOT_FOUND',
|
|
1007
|
+
retryable: true
|
|
1008
|
+
};
|
|
1009
|
+
}
|
|
1010
|
+
const resolvedElement = ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx), matched.el, matched.idx);
|
|
1011
|
+
const observedState = matched.el.state ?? null;
|
|
1012
|
+
const actual = observedState?.[property] ?? null;
|
|
1013
|
+
const compareBoolean = (value) => typeof value === 'boolean' ? value : null;
|
|
1014
|
+
const compareString = (value) => typeof value === 'string' ? value : null;
|
|
1015
|
+
const compareNumber = (value) => typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
1016
|
+
let success = false;
|
|
1017
|
+
let reason = '';
|
|
1018
|
+
let rawValue = null;
|
|
1019
|
+
let observedValue = actual;
|
|
1020
|
+
switch (property) {
|
|
1021
|
+
case 'checked':
|
|
1022
|
+
case 'focused':
|
|
1023
|
+
case 'expanded':
|
|
1024
|
+
case 'enabled': {
|
|
1025
|
+
const expectedBool = compareBoolean(expected);
|
|
1026
|
+
const actualBool = compareBoolean(actual);
|
|
1027
|
+
if (expectedBool === null) {
|
|
1028
|
+
reason = `expected ${property} must be boolean`;
|
|
1029
|
+
}
|
|
1030
|
+
else if (actualBool === null) {
|
|
1031
|
+
reason = `${property} state unavailable`;
|
|
1032
|
+
}
|
|
1033
|
+
else {
|
|
1034
|
+
rawValue = actualBool;
|
|
1035
|
+
success = actualBool === expectedBool;
|
|
1036
|
+
reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`;
|
|
1037
|
+
}
|
|
1038
|
+
observedValue = actualBool;
|
|
1039
|
+
break;
|
|
1040
|
+
}
|
|
1041
|
+
case 'value':
|
|
1042
|
+
case 'raw_value': {
|
|
1043
|
+
const expectedNumber = compareNumber(expected);
|
|
1044
|
+
const actualNumber = compareNumber(actual);
|
|
1045
|
+
if (expectedNumber !== null && actualNumber !== null) {
|
|
1046
|
+
success = actualNumber === expectedNumber;
|
|
1047
|
+
rawValue = actualNumber;
|
|
1048
|
+
observedValue = actualNumber;
|
|
1049
|
+
reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`;
|
|
1050
|
+
break;
|
|
1051
|
+
}
|
|
1052
|
+
const expectedString = typeof expected === 'string' ? expected : null;
|
|
1053
|
+
const actualString = compareString(actual);
|
|
1054
|
+
if (expectedString !== null && actualString !== null) {
|
|
1055
|
+
success = actualString === expectedString;
|
|
1056
|
+
rawValue = actualString;
|
|
1057
|
+
observedValue = actualString;
|
|
1058
|
+
reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`;
|
|
1059
|
+
}
|
|
1060
|
+
else {
|
|
1061
|
+
reason = 'value state unavailable';
|
|
1062
|
+
}
|
|
1063
|
+
break;
|
|
1064
|
+
}
|
|
1065
|
+
case 'selected': {
|
|
1066
|
+
const expectedBool = typeof expected === 'boolean' ? expected : null;
|
|
1067
|
+
const expectedString = typeof expected === 'string'
|
|
1068
|
+
? expected
|
|
1069
|
+
: expected && typeof expected === 'object'
|
|
1070
|
+
? String(expected.id ?? expected.label ?? '')
|
|
1071
|
+
: null;
|
|
1072
|
+
if (!observedState || observedState.selected === undefined || observedState.selected === null) {
|
|
1073
|
+
reason = 'selected state unavailable';
|
|
1074
|
+
break;
|
|
1075
|
+
}
|
|
1076
|
+
if (expectedBool !== null) {
|
|
1077
|
+
const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null;
|
|
1078
|
+
if (actualBool === null) {
|
|
1079
|
+
reason = 'selected state is not boolean';
|
|
1080
|
+
break;
|
|
1081
|
+
}
|
|
1082
|
+
rawValue = actualBool;
|
|
1083
|
+
observedValue = actualBool;
|
|
1084
|
+
success = actualBool === expectedBool;
|
|
1085
|
+
reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`;
|
|
1086
|
+
break;
|
|
1087
|
+
}
|
|
1088
|
+
const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
|
|
1089
|
+
? String(observedState.selected.id ?? observedState.selected.label ?? '')
|
|
1090
|
+
: String(observedState.selected);
|
|
1091
|
+
const actualString = actualSelected.trim();
|
|
1092
|
+
if (!expectedString) {
|
|
1093
|
+
reason = 'expected selected must be boolean, string, or object with id/label';
|
|
1094
|
+
break;
|
|
1095
|
+
}
|
|
1096
|
+
rawValue = actualString;
|
|
1097
|
+
observedValue = actualString;
|
|
1098
|
+
success = actualString === expectedString;
|
|
1099
|
+
reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`;
|
|
1100
|
+
break;
|
|
1101
|
+
}
|
|
1102
|
+
case 'text_value': {
|
|
1103
|
+
const expectedString = typeof expected === 'string' ? expected : null;
|
|
1104
|
+
const actualString = compareString(actual);
|
|
1105
|
+
if (!expectedString) {
|
|
1106
|
+
reason = 'expected text_value must be string';
|
|
1107
|
+
}
|
|
1108
|
+
else if (!actualString) {
|
|
1109
|
+
reason = 'text_value state unavailable';
|
|
1110
|
+
}
|
|
1111
|
+
else {
|
|
1112
|
+
success = actualString === expectedString;
|
|
1113
|
+
rawValue = actualString;
|
|
1114
|
+
observedValue = actualString;
|
|
1115
|
+
reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`;
|
|
1116
|
+
}
|
|
1117
|
+
break;
|
|
1118
|
+
}
|
|
1119
|
+
default: {
|
|
1120
|
+
if (actual !== null && actual !== undefined) {
|
|
1121
|
+
success = actual === expected;
|
|
1122
|
+
observedValue = actual;
|
|
1123
|
+
rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null;
|
|
1124
|
+
reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`;
|
|
1125
|
+
}
|
|
1126
|
+
else {
|
|
1127
|
+
reason = `unsupported or unavailable state property: ${property}`;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
if (!success && !reason) {
|
|
1132
|
+
reason = `${property} did not match expected value`;
|
|
1133
|
+
}
|
|
1134
|
+
return {
|
|
1135
|
+
success,
|
|
1136
|
+
selector,
|
|
1137
|
+
element_id: element_id ?? resolvedElement.elementId,
|
|
1138
|
+
expected_state: { property, expected },
|
|
1139
|
+
element: {
|
|
1140
|
+
...resolvedElement,
|
|
1141
|
+
state: observedState
|
|
1142
|
+
},
|
|
1143
|
+
observed_state: {
|
|
1144
|
+
property,
|
|
1145
|
+
value: observedValue,
|
|
1146
|
+
...(rawValue !== null ? { raw_value: rawValue } : {})
|
|
1147
|
+
},
|
|
1148
|
+
reason,
|
|
1149
|
+
...(success ? {} : { failure_code: 'UNKNOWN', retryable: false })
|
|
1150
|
+
};
|
|
1151
|
+
}
|
|
856
1152
|
static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
|
|
857
1153
|
const start = Date.now();
|
|
858
1154
|
const deadline = start + (timeoutMs || 0);
|
package/dist/observe/ios.js
CHANGED
|
@@ -37,7 +37,59 @@ function getCenter(bounds) {
|
|
|
37
37
|
const [x1, y1, x2, y2] = bounds;
|
|
38
38
|
return [Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)];
|
|
39
39
|
}
|
|
40
|
-
function
|
|
40
|
+
function parseIOSNumber(value) {
|
|
41
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
42
|
+
return value;
|
|
43
|
+
if (typeof value !== 'string')
|
|
44
|
+
return null;
|
|
45
|
+
const parsed = Number(value);
|
|
46
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
47
|
+
}
|
|
48
|
+
function isIOSAdjustable(node, type, traits) {
|
|
49
|
+
return /slider|adjustable|stepper|progress/i.test(type) || traits.some((trait) => /adjustable|slider|progress/i.test(trait));
|
|
50
|
+
}
|
|
51
|
+
function extractIOSState(node, type, label, value, traits) {
|
|
52
|
+
const state = {};
|
|
53
|
+
const normalizedTraits = traits.map((trait) => String(trait).toLowerCase());
|
|
54
|
+
if (normalizedTraits.some((trait) => /selected/.test(trait))) {
|
|
55
|
+
state.selected = label || value || true;
|
|
56
|
+
}
|
|
57
|
+
if (normalizedTraits.some((trait) => /focused/.test(trait))) {
|
|
58
|
+
state.focused = true;
|
|
59
|
+
}
|
|
60
|
+
if (normalizedTraits.some((trait) => /enabled/.test(trait))) {
|
|
61
|
+
state.enabled = true;
|
|
62
|
+
}
|
|
63
|
+
if (normalizedTraits.some((trait) => /disabled/.test(trait))) {
|
|
64
|
+
state.enabled = false;
|
|
65
|
+
}
|
|
66
|
+
if (value && /textfield|search|text/i.test(type)) {
|
|
67
|
+
state.text_value = value;
|
|
68
|
+
}
|
|
69
|
+
if (isIOSAdjustable(node, type, traits)) {
|
|
70
|
+
const rawValue = parseIOSNumber(value);
|
|
71
|
+
if (rawValue !== null) {
|
|
72
|
+
state.raw_value = rawValue;
|
|
73
|
+
state.value = rawValue >= 0 && rawValue <= 1 ? Math.round(rawValue * 100) : rawValue;
|
|
74
|
+
}
|
|
75
|
+
else if (value) {
|
|
76
|
+
state.raw_value = value;
|
|
77
|
+
state.value = value;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
else if (value) {
|
|
81
|
+
const numericValue = parseIOSNumber(value);
|
|
82
|
+
if (numericValue !== null) {
|
|
83
|
+
state.value = numericValue;
|
|
84
|
+
state.raw_value = numericValue;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
state.value = value;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return Object.keys(state).length > 0 ? state : null;
|
|
91
|
+
}
|
|
92
|
+
export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
|
|
41
93
|
if (!node)
|
|
42
94
|
return -1;
|
|
43
95
|
let currentIndex = -1;
|
|
@@ -46,6 +98,7 @@ function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
|
|
|
46
98
|
const value = node.AXValue || null;
|
|
47
99
|
const frame = node.AXFrame || node.frame;
|
|
48
100
|
const traits = node.AXTraits || [];
|
|
101
|
+
const state = extractIOSState(node, type, label, value, traits);
|
|
49
102
|
const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
|
|
50
103
|
const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
|
|
51
104
|
if (isUseful) {
|
|
@@ -60,7 +113,8 @@ function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
|
|
|
60
113
|
visible: true,
|
|
61
114
|
bounds: bounds,
|
|
62
115
|
center: getCenter(bounds),
|
|
63
|
-
depth: depth
|
|
116
|
+
depth: depth,
|
|
117
|
+
state
|
|
64
118
|
};
|
|
65
119
|
if (parentIndex !== -1) {
|
|
66
120
|
element.parentId = parentIndex;
|
package/dist/server/common.js
CHANGED
|
@@ -79,7 +79,8 @@ export function normalizeResolvedTarget(value = null) {
|
|
|
79
79
|
accessibility_id: value.accessibility_id ?? null,
|
|
80
80
|
class: value.class ?? null,
|
|
81
81
|
bounds: value.bounds ?? null,
|
|
82
|
-
index: value.index ?? null
|
|
82
|
+
index: value.index ?? null,
|
|
83
|
+
state: value.state ?? null
|
|
83
84
|
};
|
|
84
85
|
}
|
|
85
86
|
export function inferGenericFailure(message) {
|
|
@@ -468,6 +468,61 @@ Failure Handling:
|
|
|
468
468
|
required: ['selector']
|
|
469
469
|
}
|
|
470
470
|
},
|
|
471
|
+
{
|
|
472
|
+
name: 'expect_state',
|
|
473
|
+
description: `Purpose:
|
|
474
|
+
Verify a readable UI state property on the currently visible element.
|
|
475
|
+
|
|
476
|
+
Inputs:
|
|
477
|
+
- selector or element_id
|
|
478
|
+
- property
|
|
479
|
+
- expected
|
|
480
|
+
- platform/deviceId (optional)
|
|
481
|
+
|
|
482
|
+
Supported properties:
|
|
483
|
+
- checked, selected, focused, expanded, enabled, text_value, value, raw_value
|
|
484
|
+
|
|
485
|
+
Verification Guidance:
|
|
486
|
+
- Use this when the UI element is visible but its state must also be confirmed
|
|
487
|
+
- Prefer the canonical property names above
|
|
488
|
+
- The tool compares the normalized readable state and returns the observed value when available
|
|
489
|
+
|
|
490
|
+
Constraints:
|
|
491
|
+
- Returns structured success/failure only
|
|
492
|
+
- Does not infer a state when the property is unavailable
|
|
493
|
+
|
|
494
|
+
Failure Handling:
|
|
495
|
+
- ELEMENT_NOT_FOUND → re-resolve the element or wait for UI stabilization
|
|
496
|
+
- UNKNOWN → capture a snapshot and stop`,
|
|
497
|
+
inputSchema: {
|
|
498
|
+
type: 'object',
|
|
499
|
+
properties: {
|
|
500
|
+
selector: {
|
|
501
|
+
type: 'object',
|
|
502
|
+
properties: {
|
|
503
|
+
text: { type: 'string' },
|
|
504
|
+
resource_id: { type: 'string' },
|
|
505
|
+
accessibility_id: { type: 'string' },
|
|
506
|
+
contains: { type: 'boolean', default: false }
|
|
507
|
+
}
|
|
508
|
+
},
|
|
509
|
+
element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
|
|
510
|
+
property: { type: 'string', description: 'Readable state property to verify.' },
|
|
511
|
+
expected: {
|
|
512
|
+
description: 'Expected normalized state value.',
|
|
513
|
+
oneOf: [
|
|
514
|
+
{ type: 'boolean' },
|
|
515
|
+
{ type: 'number' },
|
|
516
|
+
{ type: 'string' },
|
|
517
|
+
{ type: 'object' }
|
|
518
|
+
]
|
|
519
|
+
},
|
|
520
|
+
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
|
|
521
|
+
deviceId: { type: 'string', description: 'Optional device serial/udid' }
|
|
522
|
+
},
|
|
523
|
+
required: ['property', 'expected']
|
|
524
|
+
}
|
|
525
|
+
},
|
|
471
526
|
{
|
|
472
527
|
name: 'wait_for_ui',
|
|
473
528
|
description: `Purpose:
|
|
@@ -208,6 +208,22 @@ async function handleExpectElementVisible(args) {
|
|
|
208
208
|
const res = await ToolsInteract.expectElementVisibleHandler({ selector, element_id, timeout_ms, poll_interval_ms, platform, deviceId });
|
|
209
209
|
return wrapResponse(res);
|
|
210
210
|
}
|
|
211
|
+
async function handleExpectState(args) {
|
|
212
|
+
const selector = getObjectArg(args, 'selector');
|
|
213
|
+
const element_id = getStringArg(args, 'element_id');
|
|
214
|
+
const property = requireStringArg(args, 'property');
|
|
215
|
+
const platform = getStringArg(args, 'platform');
|
|
216
|
+
const deviceId = getStringArg(args, 'deviceId');
|
|
217
|
+
if (!selector && !element_id) {
|
|
218
|
+
throw new Error('Missing selector or element_id argument');
|
|
219
|
+
}
|
|
220
|
+
if (!Object.prototype.hasOwnProperty.call(args, 'expected')) {
|
|
221
|
+
throw new Error('Missing expected argument');
|
|
222
|
+
}
|
|
223
|
+
const expected = args.expected;
|
|
224
|
+
const res = await ToolsInteract.expectStateHandler({ selector: selector ?? undefined, element_id: element_id ?? undefined, property, expected, platform, deviceId });
|
|
225
|
+
return wrapResponse(res);
|
|
226
|
+
}
|
|
211
227
|
async function handleWaitForUI(args) {
|
|
212
228
|
const selector = getObjectArg(args, 'selector');
|
|
213
229
|
const condition = getStringArg(args, 'condition') ?? 'exists';
|
|
@@ -395,6 +411,7 @@ export const toolHandlers = {
|
|
|
395
411
|
wait_for_screen_change: handleWaitForScreenChange,
|
|
396
412
|
expect_screen: handleExpectScreen,
|
|
397
413
|
expect_element_visible: handleExpectElementVisible,
|
|
414
|
+
expect_state: handleExpectState,
|
|
398
415
|
wait_for_ui: handleWaitForUI,
|
|
399
416
|
find_element: handleFindElement,
|
|
400
417
|
tap: handleTap,
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.
|
|
9
|
+
version: '0.25.0'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|