mobile-debug-mcp 0.21.0 → 0.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/android.js +0 -27
- package/dist/interact/index.js +23 -38
- package/dist/interact/ios.js +0 -26
- package/dist/observe/android.js +85 -21
- package/dist/observe/ios.js +50 -1
- package/dist/server.js +16 -39
- package/dist/utils/image.js +18 -0
- package/dist/utils/resolve-device.js +5 -0
- package/docs/CHANGELOG.md +8 -2
- package/docs/tools/interact.md +7 -27
- package/package.json +4 -3
- package/src/interact/android.ts +1 -32
- package/src/interact/index.ts +38 -46
- package/src/interact/ios.ts +1 -31
- package/src/observe/android.ts +84 -20
- package/src/observe/ios.ts +54 -3
- package/src/server.ts +17 -39
- package/src/types.ts +6 -0
- package/src/utils/image.ts +14 -0
- package/src/utils/resolve-device.ts +6 -0
- package/test/interact/device/run-real-test.ts +3 -19
- package/test/interact/unit/{observe_until.test.ts → wait_for_ui.test.ts} +6 -6
- package/test/observe/device/wait_for_element_real.ts +3 -80
- package/test/observe/unit/wait_for_element_mock.ts +2 -104
- package/test/observe/unit/{observe_until_edge_cases.test.ts → wait_for_ui_edge_cases.test.ts} +5 -5
- package/test/observe/unit/{observe_until_stability.test.ts → wait_for_ui_stability.test.ts} +3 -3
- package/test/unit/index.ts +27 -15
package/dist/interact/android.js
CHANGED
|
@@ -3,33 +3,6 @@ import { AndroidObserve } from "../observe/index.js";
|
|
|
3
3
|
import { scrollToElementShared } from "../utils/ui/index.js";
|
|
4
4
|
export class AndroidInteract {
|
|
5
5
|
observe = new AndroidObserve();
|
|
6
|
-
async waitForElement(text, timeout, deviceId) {
|
|
7
|
-
const metadata = await getAndroidDeviceMetadata("", deviceId);
|
|
8
|
-
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
|
|
9
|
-
const startTime = Date.now();
|
|
10
|
-
while (Date.now() - startTime < timeout) {
|
|
11
|
-
try {
|
|
12
|
-
const tree = await this.observe.getUITree(deviceId);
|
|
13
|
-
if (tree.error) {
|
|
14
|
-
return { device: deviceInfo, found: false, error: tree.error };
|
|
15
|
-
}
|
|
16
|
-
const element = tree.elements.find(e => e.text === text);
|
|
17
|
-
if (element) {
|
|
18
|
-
return { device: deviceInfo, found: true, element };
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
catch (e) {
|
|
22
|
-
// Ignore errors during polling and retry
|
|
23
|
-
console.error("Error polling UI tree:", e);
|
|
24
|
-
}
|
|
25
|
-
const elapsed = Date.now() - startTime;
|
|
26
|
-
const remaining = timeout - elapsed;
|
|
27
|
-
if (remaining <= 0)
|
|
28
|
-
break;
|
|
29
|
-
await new Promise(resolve => setTimeout(resolve, Math.min(500, remaining)));
|
|
30
|
-
}
|
|
31
|
-
return { device: deviceInfo, found: false };
|
|
32
|
-
}
|
|
33
6
|
async tap(x, y, deviceId) {
|
|
34
7
|
const metadata = await getAndroidDeviceMetadata("", deviceId);
|
|
35
8
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
|
package/dist/interact/index.js
CHANGED
|
@@ -10,11 +10,6 @@ export class ToolsInteract {
|
|
|
10
10
|
const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract();
|
|
11
11
|
return { interact: interact, resolved, platform: effectivePlatform };
|
|
12
12
|
}
|
|
13
|
-
static async waitForElementHandler({ platform, text, timeout, deviceId }) {
|
|
14
|
-
const effectiveTimeout = timeout ?? 10000;
|
|
15
|
-
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
16
|
-
return await interact.waitForElement(text, effectiveTimeout, resolved.id);
|
|
17
|
-
}
|
|
18
13
|
static async tapHandler({ platform, x, y, deviceId }) {
|
|
19
14
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
20
15
|
return await interact.tap(x, y, resolved.id);
|
|
@@ -220,6 +215,10 @@ export class ToolsInteract {
|
|
|
220
215
|
const scoreVal = Math.min(1, Number(bestScore.toFixed(3)));
|
|
221
216
|
return { found: true, element: outEl, score: scoreVal, confidence: scoreVal };
|
|
222
217
|
}
|
|
218
|
+
static async waitForUIHandler({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
|
|
219
|
+
// Backwards-compatible wrapper that delegates to the core waitForUICore implementation
|
|
220
|
+
return await ToolsInteract.waitForUICore({ type, query, timeoutMs, pollIntervalMs, includeSnapshotOnFailure, match, stability_ms, observationDelayMs, platform, deviceId });
|
|
221
|
+
}
|
|
223
222
|
static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }) {
|
|
224
223
|
const start = Date.now();
|
|
225
224
|
let lastFingerprint = null;
|
|
@@ -258,7 +257,7 @@ export class ToolsInteract {
|
|
|
258
257
|
}
|
|
259
258
|
return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start };
|
|
260
259
|
}
|
|
261
|
-
static async
|
|
260
|
+
static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
|
|
262
261
|
const start = Date.now();
|
|
263
262
|
const deadline = start + (timeoutMs || 0);
|
|
264
263
|
const q = (query === null || query === undefined) ? '' : String(query);
|
|
@@ -281,7 +280,7 @@ export class ToolsInteract {
|
|
|
281
280
|
}
|
|
282
281
|
catch (err) {
|
|
283
282
|
try {
|
|
284
|
-
console.warn('
|
|
283
|
+
console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err));
|
|
285
284
|
}
|
|
286
285
|
catch { }
|
|
287
286
|
}
|
|
@@ -292,7 +291,7 @@ export class ToolsInteract {
|
|
|
292
291
|
// Optional initial observation delay requested by caller
|
|
293
292
|
if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
|
|
294
293
|
try {
|
|
295
|
-
console.log(`
|
|
294
|
+
console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`);
|
|
296
295
|
}
|
|
297
296
|
catch { }
|
|
298
297
|
await sleep(observationDelayMs);
|
|
@@ -309,29 +308,11 @@ export class ToolsInteract {
|
|
|
309
308
|
// Evaluate condition per type
|
|
310
309
|
if (type === 'ui') {
|
|
311
310
|
try {
|
|
312
|
-
//
|
|
311
|
+
// Prefer using the public findElementHandler which tests can override. This avoids relying
|
|
312
|
+
// on resolveObserve/getUITree for unit tests which may not have devices available.
|
|
313
313
|
try {
|
|
314
|
-
|
|
315
|
-
const
|
|
316
|
-
const tree = await withTimeout(ToolsObserve.getUITreeHandler({ platform, deviceId }), Math.min(pollInterval, 500));
|
|
317
|
-
const elems = Array.isArray(tree && tree.elements) ? tree.elements : [];
|
|
318
|
-
const qnorm = q.toLowerCase();
|
|
319
|
-
let matched = null;
|
|
320
|
-
for (const el of elems) {
|
|
321
|
-
try {
|
|
322
|
-
const txt = ((el && (el.text || el.label || el.value || el.contentDescription || el.accessibilityLabel)) || '');
|
|
323
|
-
if (!txt)
|
|
324
|
-
continue;
|
|
325
|
-
if (String(txt).toLowerCase().includes(qnorm)) {
|
|
326
|
-
matched = el;
|
|
327
|
-
break;
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
catch {
|
|
331
|
-
continue;
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
const isPresent = !!matched;
|
|
314
|
+
const findRes = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId });
|
|
315
|
+
const isPresent = !!(findRes && findRes.found);
|
|
335
316
|
const conditionTrue = (match === 'present') ? isPresent : !isPresent;
|
|
336
317
|
if (conditionTrue) {
|
|
337
318
|
if (matchedAt === null)
|
|
@@ -339,8 +320,8 @@ export class ToolsInteract {
|
|
|
339
320
|
stableDuration = Date.now() - matchedAt;
|
|
340
321
|
lastObservedState = true;
|
|
341
322
|
if (stableDuration >= stability_ms) {
|
|
342
|
-
matchSource = 'ui-
|
|
343
|
-
const element = isPresent ?
|
|
323
|
+
matchSource = 'ui-find';
|
|
324
|
+
const element = isPresent ? findRes.element : null;
|
|
344
325
|
const now2 = Date.now();
|
|
345
326
|
return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null };
|
|
346
327
|
}
|
|
@@ -352,11 +333,11 @@ export class ToolsInteract {
|
|
|
352
333
|
}
|
|
353
334
|
}
|
|
354
335
|
catch (err) {
|
|
355
|
-
console.error('
|
|
336
|
+
console.error('waitForUI(ui) find error:', err);
|
|
356
337
|
}
|
|
357
338
|
}
|
|
358
339
|
catch (err) {
|
|
359
|
-
console.error('
|
|
340
|
+
console.error('waitForUI(ui) outer error:', err);
|
|
360
341
|
}
|
|
361
342
|
}
|
|
362
343
|
else if (type === 'log') {
|
|
@@ -387,7 +368,7 @@ export class ToolsInteract {
|
|
|
387
368
|
}
|
|
388
369
|
}
|
|
389
370
|
catch (err) {
|
|
390
|
-
console.error('
|
|
371
|
+
console.error('waitForUI(log) error:', err);
|
|
391
372
|
}
|
|
392
373
|
}
|
|
393
374
|
else if (type === 'screen') {
|
|
@@ -415,7 +396,7 @@ export class ToolsInteract {
|
|
|
415
396
|
}
|
|
416
397
|
}
|
|
417
398
|
catch (err) {
|
|
418
|
-
console.error('
|
|
399
|
+
console.error('waitForUI(screen) error:', err);
|
|
419
400
|
}
|
|
420
401
|
}
|
|
421
402
|
else if (type === 'idle') {
|
|
@@ -439,7 +420,7 @@ export class ToolsInteract {
|
|
|
439
420
|
}
|
|
440
421
|
}
|
|
441
422
|
catch (err) {
|
|
442
|
-
console.error('
|
|
423
|
+
console.error('waitForUI(idle) error:', err);
|
|
443
424
|
}
|
|
444
425
|
}
|
|
445
426
|
// Respect poll interval and avoid tight loop
|
|
@@ -449,7 +430,11 @@ export class ToolsInteract {
|
|
|
449
430
|
let snapshot = null;
|
|
450
431
|
if (includeSnapshotOnFailure) {
|
|
451
432
|
try {
|
|
452
|
-
|
|
433
|
+
// Use dynamic import to avoid circular-initialization issues where the ToolsObserve
|
|
434
|
+
// binding captured earlier may not reflect test-time overrides. Importing at call
|
|
435
|
+
// time ensures the latest exported ToolsObserve object is used.
|
|
436
|
+
const Obs = await import('../observe/index.js');
|
|
437
|
+
snapshot = await Obs.ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId });
|
|
453
438
|
}
|
|
454
439
|
catch (err) {
|
|
455
440
|
snapshot = { error: err instanceof Error ? err.message : String(err) };
|
package/dist/interact/ios.js
CHANGED
|
@@ -4,32 +4,6 @@ import { iOSObserve } from "../observe/index.js";
|
|
|
4
4
|
import { scrollToElementShared } from "../utils/ui/index.js";
|
|
5
5
|
export class iOSInteract {
|
|
6
6
|
observe = new iOSObserve();
|
|
7
|
-
async waitForElement(text, timeout, deviceId = "booted") {
|
|
8
|
-
const device = await getIOSDeviceMetadata(deviceId);
|
|
9
|
-
const startTime = Date.now();
|
|
10
|
-
while (Date.now() - startTime < timeout) {
|
|
11
|
-
try {
|
|
12
|
-
const tree = await this.observe.getUITree(deviceId);
|
|
13
|
-
if (tree.error) {
|
|
14
|
-
return { device, found: false, error: tree.error };
|
|
15
|
-
}
|
|
16
|
-
const element = tree.elements.find(e => e.text === text);
|
|
17
|
-
if (element) {
|
|
18
|
-
return { device, found: true, element };
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
catch (e) {
|
|
22
|
-
// Ignore errors during polling and retry
|
|
23
|
-
console.error("Error polling UI tree:", e);
|
|
24
|
-
}
|
|
25
|
-
const elapsed = Date.now() - startTime;
|
|
26
|
-
const remaining = timeout - elapsed;
|
|
27
|
-
if (remaining <= 0)
|
|
28
|
-
break;
|
|
29
|
-
await new Promise(resolve => setTimeout(resolve, Math.min(500, remaining)));
|
|
30
|
-
}
|
|
31
|
-
return { device, found: false };
|
|
32
|
-
}
|
|
33
7
|
async tap(x, y, deviceId = "booted") {
|
|
34
8
|
const device = await getIOSDeviceMetadata(deviceId);
|
|
35
9
|
// Use shared helper to detect idb
|
package/dist/observe/android.js
CHANGED
|
@@ -5,6 +5,7 @@ import { createWriteStream } from "fs";
|
|
|
5
5
|
import { promises as fsPromises } from "fs";
|
|
6
6
|
import path from "path";
|
|
7
7
|
import { computeScreenFingerprint } from "../utils/ui/index.js";
|
|
8
|
+
import { parsePngSize } from "../utils/image.js";
|
|
8
9
|
const activeLogStreams = new Map();
|
|
9
10
|
export class AndroidObserve {
|
|
10
11
|
async getDeviceMetadata(appId, deviceId) {
|
|
@@ -120,7 +121,7 @@ export class AndroidObserve {
|
|
|
120
121
|
child.kill();
|
|
121
122
|
reject(new Error(`ADB screencap timed out after 10s`));
|
|
122
123
|
}, 10000);
|
|
123
|
-
child.on('close', (code) => {
|
|
124
|
+
child.on('close', async (code) => {
|
|
124
125
|
clearTimeout(timeout);
|
|
125
126
|
if (code !== 0) {
|
|
126
127
|
reject(new Error(stderr.trim() || `Screencap failed with code ${code}`));
|
|
@@ -128,28 +129,91 @@ export class AndroidObserve {
|
|
|
128
129
|
}
|
|
129
130
|
const screenshotBuffer = Buffer.concat(chunks);
|
|
130
131
|
const screenshotBase64 = screenshotBuffer.toString('base64');
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
132
|
+
const parsed = parsePngSize(screenshotBuffer);
|
|
133
|
+
if (parsed.width > 0 && parsed.height > 0) {
|
|
134
|
+
// Attempt to convert to WebP (preferred) and provide JPEG fallback (awaited to avoid race)
|
|
135
|
+
try {
|
|
136
|
+
const sharpModule = await import('sharp');
|
|
137
|
+
const sharp = sharpModule && sharpModule.default ? sharpModule.default : sharpModule;
|
|
138
|
+
const buf = screenshotBuffer;
|
|
139
|
+
const img = sharp(buf);
|
|
140
|
+
const meta = await img.metadata().catch((err) => { console.error('sharp.metadata failed (Android):', err); return {}; });
|
|
141
|
+
const hasAlpha = !!meta.hasAlpha || (meta.channels && meta.channels > 3);
|
|
142
|
+
let webpBuf = null;
|
|
143
|
+
let jpegBuf = null;
|
|
144
|
+
try {
|
|
145
|
+
webpBuf = await img.webp({ quality: 80 }).toBuffer();
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
console.error('WebP conversion failed (Android):', err instanceof Error ? err.message : String(err));
|
|
149
|
+
webpBuf = null;
|
|
150
|
+
}
|
|
151
|
+
try {
|
|
152
|
+
jpegBuf = await img.jpeg({ quality: 80 }).toBuffer();
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
console.error('JPEG conversion failed (Android):', err instanceof Error ? err.message : String(err));
|
|
156
|
+
jpegBuf = null;
|
|
157
|
+
}
|
|
158
|
+
if (hasAlpha) {
|
|
159
|
+
if (webpBuf) {
|
|
160
|
+
const webpB64 = webpBuf.toString('base64');
|
|
161
|
+
const jpegB64 = jpegBuf ? jpegBuf.toString('base64') : null;
|
|
162
|
+
resolve({ device: deviceInfo, screenshot: webpB64, screenshot_mime: 'image/webp', screenshot_fallback: jpegB64, screenshot_fallback_mime: jpegB64 ? 'image/jpeg' : undefined, resolution: { width: parsed.width, height: parsed.height } });
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
const pngB64 = buf.toString('base64');
|
|
166
|
+
resolve({ device: deviceInfo, screenshot: pngB64, screenshot_mime: 'image/png', resolution: { width: parsed.width, height: parsed.height } });
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
if (webpBuf) {
|
|
170
|
+
const webpB64 = webpBuf.toString('base64');
|
|
171
|
+
const jpegB64 = jpegBuf ? jpegBuf.toString('base64') : null;
|
|
172
|
+
resolve({ device: deviceInfo, screenshot: webpB64, screenshot_mime: 'image/webp', screenshot_fallback: jpegB64, screenshot_fallback_mime: jpegB64 ? 'image/jpeg' : undefined, resolution: { width: parsed.width, height: parsed.height } });
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
if (jpegBuf) {
|
|
176
|
+
resolve({ device: deviceInfo, screenshot: jpegBuf.toString('base64'), screenshot_mime: 'image/jpeg', resolution: { width: parsed.width, height: parsed.height } });
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
// No conversions succeeded; return original PNG
|
|
180
|
+
resolve({ device: deviceInfo, screenshot: screenshotBase64, screenshot_mime: 'image/png', resolution: { width: parsed.width, height: parsed.height } });
|
|
181
|
+
return;
|
|
139
182
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
resolution: { width, height }
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
183
|
+
catch (err) {
|
|
184
|
+
console.error('Screenshot conversion pipeline failed (Android):', err instanceof Error ? err.message : String(err));
|
|
185
|
+
// Conversion failed - fall back to original PNG with parsed resolution
|
|
186
|
+
resolve({ device: deviceInfo, screenshot: screenshotBase64, screenshot_mime: 'image/png', resolution: { width: parsed.width, height: parsed.height } });
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
// Fallback to querying wm size if parsing failed
|
|
192
|
+
execAdb(['shell', 'wm', 'size'], deviceId)
|
|
193
|
+
.then(sizeStdout => {
|
|
194
|
+
let width = 0;
|
|
195
|
+
let height = 0;
|
|
196
|
+
const match = sizeStdout.match(/Physical size: (\d+)x(\d+)/);
|
|
197
|
+
if (match) {
|
|
198
|
+
width = parseInt(match[1], 10);
|
|
199
|
+
height = parseInt(match[2], 10);
|
|
200
|
+
}
|
|
201
|
+
resolve({
|
|
202
|
+
device: deviceInfo,
|
|
203
|
+
screenshot: screenshotBase64,
|
|
204
|
+
screenshot_mime: 'image/png',
|
|
205
|
+
resolution: { width, height }
|
|
206
|
+
});
|
|
207
|
+
})
|
|
208
|
+
.catch(() => {
|
|
209
|
+
resolve({
|
|
210
|
+
device: deviceInfo,
|
|
211
|
+
screenshot: screenshotBase64,
|
|
212
|
+
screenshot_mime: 'image/png',
|
|
213
|
+
resolution: { width: 0, height: 0 }
|
|
214
|
+
});
|
|
151
215
|
});
|
|
152
|
-
}
|
|
216
|
+
}
|
|
153
217
|
});
|
|
154
218
|
child.on('error', (err) => {
|
|
155
219
|
clearTimeout(timeout);
|
package/dist/observe/ios.js
CHANGED
|
@@ -5,6 +5,7 @@ import { createWriteStream, promises as fsPromises } from 'fs';
|
|
|
5
5
|
import path from 'path';
|
|
6
6
|
import { parseLogLine } from '../utils/android/utils.js';
|
|
7
7
|
import { computeScreenFingerprint } from '../utils/ui/index.js';
|
|
8
|
+
import { parsePngSize } from '../utils/image.js';
|
|
8
9
|
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
9
10
|
function parseIDBFrame(frame) {
|
|
10
11
|
if (!frame)
|
|
@@ -109,11 +110,59 @@ export class iOSObserve {
|
|
|
109
110
|
await execCommand(['simctl', 'io', deviceId, 'screenshot', tmpFile], deviceId);
|
|
110
111
|
const buffer = await fs.readFile(tmpFile);
|
|
111
112
|
const base64 = buffer.toString('base64');
|
|
113
|
+
const dims = parsePngSize(buffer);
|
|
114
|
+
// Try to generate WebP (preferred) and JPEG fallback using sharp (in-process, cross-platform)
|
|
115
|
+
try {
|
|
116
|
+
const sharpModule = await import('sharp');
|
|
117
|
+
const sharp = sharpModule && sharpModule.default ? sharpModule.default : sharpModule;
|
|
118
|
+
const img = sharp(buffer);
|
|
119
|
+
const meta = await img.metadata().catch((err) => { console.error('sharp.metadata failed:', err); return {}; });
|
|
120
|
+
// If image has alpha channel, prefer lossless PNG to preserve transparency
|
|
121
|
+
const hasAlpha = !!meta.hasAlpha || (meta.channels && meta.channels > 3);
|
|
122
|
+
// Generate WebP and JPEG buffers; log failures
|
|
123
|
+
let webpBuf = null;
|
|
124
|
+
let jpegBuf = null;
|
|
125
|
+
try {
|
|
126
|
+
webpBuf = await img.webp({ quality: 80 }).toBuffer();
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
console.error('WebP conversion failed (iOS):', err instanceof Error ? err.message : String(err));
|
|
130
|
+
webpBuf = null;
|
|
131
|
+
}
|
|
132
|
+
try {
|
|
133
|
+
jpegBuf = await img.jpeg({ quality: 80 }).toBuffer();
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
console.error('JPEG conversion failed (iOS):', err instanceof Error ? err.message : String(err));
|
|
137
|
+
jpegBuf = null;
|
|
138
|
+
}
|
|
139
|
+
await fs.rm(tmpFile).catch(() => { });
|
|
140
|
+
if (hasAlpha) {
|
|
141
|
+
// preserve alpha: return PNG if WebP not available
|
|
142
|
+
if (webpBuf) {
|
|
143
|
+
return { device, screenshot: webpBuf.toString('base64'), screenshot_mime: 'image/webp', screenshot_fallback: base64, screenshot_fallback_mime: 'image/png', resolution: { width: dims.width, height: dims.height } };
|
|
144
|
+
}
|
|
145
|
+
// if webp unavailable, return original PNG
|
|
146
|
+
return { device, screenshot: base64, screenshot_mime: 'image/png', resolution: { width: dims.width, height: dims.height } };
|
|
147
|
+
}
|
|
148
|
+
// No alpha: prefer webp, fall back to jpeg
|
|
149
|
+
if (webpBuf) {
|
|
150
|
+
return { device, screenshot: webpBuf.toString('base64'), screenshot_mime: 'image/webp', screenshot_fallback: jpegBuf ? jpegBuf.toString('base64') : undefined, screenshot_fallback_mime: jpegBuf ? 'image/jpeg' : undefined, resolution: { width: dims.width, height: dims.height } };
|
|
151
|
+
}
|
|
152
|
+
if (jpegBuf) {
|
|
153
|
+
return { device, screenshot: jpegBuf.toString('base64'), screenshot_mime: 'image/jpeg', resolution: { width: dims.width, height: dims.height } };
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
console.error('Screenshot conversion pipeline failed (iOS):', err instanceof Error ? err.message : String(err));
|
|
158
|
+
// fall through to png fallback
|
|
159
|
+
}
|
|
112
160
|
await fs.rm(tmpFile).catch(() => { });
|
|
113
161
|
return {
|
|
114
162
|
device,
|
|
115
163
|
screenshot: base64,
|
|
116
|
-
|
|
164
|
+
screenshot_mime: 'image/png',
|
|
165
|
+
resolution: { width: dims.width, height: dims.height },
|
|
117
166
|
};
|
|
118
167
|
}
|
|
119
168
|
catch (e) {
|
package/dist/server.js
CHANGED
|
@@ -315,8 +315,8 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
|
315
315
|
}
|
|
316
316
|
},
|
|
317
317
|
{
|
|
318
|
-
name: "
|
|
319
|
-
description: "Wait for a UI condition
|
|
318
|
+
name: "wait_for_ui",
|
|
319
|
+
description: "Wait for a UI/log/screen/idle condition with a stability window before returning success.",
|
|
320
320
|
inputSchema: {
|
|
321
321
|
type: "object",
|
|
322
322
|
properties: {
|
|
@@ -332,34 +332,6 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
|
332
332
|
}
|
|
333
333
|
}
|
|
334
334
|
},
|
|
335
|
-
{
|
|
336
|
-
name: "wait_for_element",
|
|
337
|
-
description: "Wait until a UI element with matching text appears on screen or timeout is reached.",
|
|
338
|
-
inputSchema: {
|
|
339
|
-
type: "object",
|
|
340
|
-
properties: {
|
|
341
|
-
platform: {
|
|
342
|
-
type: "string",
|
|
343
|
-
enum: ["android", "ios"],
|
|
344
|
-
description: "Platform to check"
|
|
345
|
-
},
|
|
346
|
-
text: {
|
|
347
|
-
type: "string",
|
|
348
|
-
description: "Text content of the element to wait for"
|
|
349
|
-
},
|
|
350
|
-
timeout: {
|
|
351
|
-
type: "number",
|
|
352
|
-
description: "Max wait time in ms (default 10000)",
|
|
353
|
-
default: 10000
|
|
354
|
-
},
|
|
355
|
-
deviceId: {
|
|
356
|
-
type: "string",
|
|
357
|
-
description: "Device Serial/UDID. Defaults to connected/booted device."
|
|
358
|
-
}
|
|
359
|
-
},
|
|
360
|
-
required: ["platform", "text"]
|
|
361
|
-
}
|
|
362
|
-
},
|
|
363
335
|
{
|
|
364
336
|
name: "find_element",
|
|
365
337
|
description: "Find a UI element by semantic query (text, content-desc, resource-id, class). Returns best match.",
|
|
@@ -612,12 +584,17 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
612
584
|
if (name === "capture_screenshot") {
|
|
613
585
|
const { platform, deviceId } = args;
|
|
614
586
|
const res = await ToolsObserve.captureScreenshotHandler({ platform, deviceId });
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
587
|
+
const mime = res.screenshot_mime || 'image/png';
|
|
588
|
+
const content = [
|
|
589
|
+
{ type: 'text', text: JSON.stringify({ device: res.device, result: { resolution: res.resolution, mimeType: mime } }, null, 2) },
|
|
590
|
+
{ type: 'image', data: res.screenshot, mimeType: mime }
|
|
591
|
+
];
|
|
592
|
+
// If a jpeg fallback is available, include a small note and the fallback as an additional image block for compatibility
|
|
593
|
+
if (res.screenshot_fallback) {
|
|
594
|
+
content.push({ type: 'text', text: JSON.stringify({ note: 'JPEG fallback included for compatibility', mimeType: res.screenshot_fallback_mime || 'image/jpeg' }) });
|
|
595
|
+
content.push({ type: 'image', data: res.screenshot_fallback, mimeType: res.screenshot_fallback_mime || 'image/jpeg' });
|
|
596
|
+
}
|
|
597
|
+
return { content };
|
|
621
598
|
}
|
|
622
599
|
if (name === "capture_debug_snapshot") {
|
|
623
600
|
const { reason, includeLogs, logLines, platform, appId, deviceId, sessionId } = args;
|
|
@@ -644,9 +621,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
644
621
|
const res = await ToolsInteract.waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs, pollIntervalMs, deviceId });
|
|
645
622
|
return wrapResponse(res);
|
|
646
623
|
}
|
|
647
|
-
if (name === "
|
|
648
|
-
const {
|
|
649
|
-
const res = await ToolsInteract.
|
|
624
|
+
if (name === "wait_for_ui") {
|
|
625
|
+
const { type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId } = (args || {});
|
|
626
|
+
const res = await ToolsInteract.waitForUIHandler({ type, query, timeoutMs, pollIntervalMs, includeSnapshotOnFailure, match, stability_ms, observationDelayMs, platform, deviceId });
|
|
650
627
|
return wrapResponse(res);
|
|
651
628
|
}
|
|
652
629
|
if (name === "find_element") {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export function parsePngSize(buf) {
|
|
2
|
+
try {
|
|
3
|
+
if (!buf || buf.length < 24)
|
|
4
|
+
return { width: 0, height: 0 };
|
|
5
|
+
// PNG signature + IHDR checks
|
|
6
|
+
if (buf.readUInt32BE(0) !== 0x89504e47 || buf.readUInt32BE(4) !== 0x0d0a1a0a)
|
|
7
|
+
return { width: 0, height: 0 };
|
|
8
|
+
const ihdr = buf.toString('ascii', 12, 16);
|
|
9
|
+
if (ihdr !== 'IHDR')
|
|
10
|
+
return { width: 0, height: 0 };
|
|
11
|
+
const width = buf.readUInt32BE(16);
|
|
12
|
+
const height = buf.readUInt32BE(20);
|
|
13
|
+
return { width, height };
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return { width: 0, height: 0 };
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -23,6 +23,11 @@ export async function listDevices(platform, appId) {
|
|
|
23
23
|
export async function resolveTargetDevice(opts) {
|
|
24
24
|
const { platform, appId, prefer, deviceId } = opts;
|
|
25
25
|
const devices = await listDevices(platform, appId);
|
|
26
|
+
// During unit tests (no adb/xcrun available), provide a lightweight mock device so
|
|
27
|
+
// the observe/interact unit tests can run without real devices.
|
|
28
|
+
if ((!devices || devices.length === 0) && (process.env.NODE_ENV === 'test' || process.env.MCP_TEST_MOCK_DEVICES === '1')) {
|
|
29
|
+
return { id: 'mock', platform: platform || 'android', osVersion: '12', model: 'Pixel', simulator: true };
|
|
30
|
+
}
|
|
26
31
|
if (deviceId) {
|
|
27
32
|
const found = devices.find(d => d.id === deviceId);
|
|
28
33
|
if (!found)
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -2,8 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the **Mobile Debug MCP** project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.21.2]
|
|
6
|
+
- Fixed screenshots not working, imnproved tool
|
|
7
|
+
|
|
8
|
+
## [0.21.1]
|
|
9
|
+
- Removed wait_for_element and renamed observe_until to wait_for_ui (obsolete references removed)
|
|
10
|
+
|
|
5
11
|
## [0.21.0]
|
|
6
|
-
- Added `
|
|
12
|
+
- Added `wait_for_ui` as a tool for agents to wait for things like API requests
|
|
7
13
|
|
|
8
14
|
## [0.20.1]
|
|
9
15
|
- Fixes gradle home issue for android
|
|
@@ -24,7 +30,7 @@ All notable changes to the **Mobile Debug MCP** project will be documented in th
|
|
|
24
30
|
|
|
25
31
|
## [0.19.0]
|
|
26
32
|
|
|
27
|
-
- Added `
|
|
33
|
+
- Added `wait_for_ui` interaction tool: waits for UI, log, screen fingerprint or idle conditions with configurable polling and timeout. Returns rich details on match (element info, log line, new fingerprint).
|
|
28
34
|
|
|
29
35
|
|
|
30
36
|
## [0.18.0]
|
package/docs/tools/interact.md
CHANGED
|
@@ -2,26 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
Tools that perform UI interactions: tap, swipe, type_text, press_back, and waiting for elements.
|
|
4
4
|
|
|
5
|
-
## wait_for_element
|
|
6
|
-
Wait until a UI element with matching text appears on screen or timeout is reached.
|
|
7
|
-
|
|
8
|
-
Input:
|
|
9
|
-
|
|
10
|
-
```
|
|
11
|
-
{ "platform": "android", "text": "Home", "timeout": 5000, "deviceId": "emulator-5554" }
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
Response:
|
|
15
|
-
|
|
16
|
-
```
|
|
17
|
-
{ "device": { "platform": "android", "id": "emulator-5554" }, "found": true, "element": { "text": "Home", "resourceId": "com.example:id/home" } }
|
|
18
|
-
```
|
|
19
|
-
|
|
20
|
-
Notes:
|
|
21
|
-
- Polls get_ui_tree until timeout or element found. Returns an `error` field if system failures occur.
|
|
22
|
-
|
|
23
|
-
---
|
|
24
|
-
|
|
25
5
|
## tap / swipe / type_text / press_back
|
|
26
6
|
|
|
27
7
|
Tap input example:
|
|
@@ -153,7 +133,7 @@ Notes:
|
|
|
153
133
|
|
|
154
134
|
---
|
|
155
135
|
|
|
156
|
-
##
|
|
136
|
+
## wait_for_ui
|
|
157
137
|
|
|
158
138
|
Purpose:
|
|
159
139
|
- Wait for a condition to occur on the device: UI element appearance, a log line, a screen fingerprint change, or an idle/stable screen state.
|
|
@@ -164,7 +144,7 @@ Supported types and behavior:
|
|
|
164
144
|
- screen: Compares screen fingerprints (visual checks) against an initial baseline and returns when fingerprint changes. If `query` is provided it will attempt a `find_element` on the new screen to validate the expected content.
|
|
165
145
|
- idle: Waits until the screen fingerprint remains stable for a short stability window (default 1000ms).
|
|
166
146
|
|
|
167
|
-
Input (ToolsInteract.
|
|
147
|
+
Input (ToolsInteract.waitForUIHandler):
|
|
168
148
|
```
|
|
169
149
|
{ "type": "ui|log|screen|idle", "query": "optional string", "timeoutMs": 5000, "pollIntervalMs": 200, "platform": "android|ios", "deviceId": "optional device id" }
|
|
170
150
|
```
|
|
@@ -190,16 +170,16 @@ Notes & tips:
|
|
|
190
170
|
- For UI-sensitive flows prefer type='ui' rather than relying solely on visual fingerprint changes, as some UI updates don't alter the fingerprint.
|
|
191
171
|
|
|
192
172
|
Tests:
|
|
193
|
-
- Unit: `test/interact/unit/
|
|
194
|
-
- Device runner: `test/interact/device/
|
|
173
|
+
- Unit: `test/interact/unit/wait_for_ui.test.ts`
|
|
174
|
+
- Device runner: `test/interact/device/wait_for_ui_device.ts` (requires devices/emulators and adb/xcrun in PATH)
|
|
195
175
|
|
|
196
176
|
Example:
|
|
197
177
|
```
|
|
198
178
|
// Wait up to 5s for a button labeled "Generate Session" on Android
|
|
199
|
-
ToolsInteract.
|
|
179
|
+
ToolsInteract.waitForUIHandler({ type: 'ui', query: 'Generate Session', timeoutMs: 5000, platform: 'android' })
|
|
200
180
|
```
|
|
201
181
|
|
|
202
182
|
Troubleshooting:
|
|
203
|
-
- If
|
|
204
|
-
- If
|
|
183
|
+
- If wait_for_ui(log) never matches, ensure log streaming is started for the target package and baseline logs captured correctly.
|
|
184
|
+
- If wait_for_ui(screen) times out despite visible UI change, try type='ui' to validate content-level changes.
|
|
205
185
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobile-debug-mcp",
|
|
3
|
-
"version": "0.21.
|
|
3
|
+
"version": "0.21.2",
|
|
4
4
|
"description": "MCP server for mobile app debugging (Android + iOS), with focus on security and reliability",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"healthcheck": "tsx ./src/cli/idb/check-idb.ts",
|
|
14
14
|
"install-idb": "tsx ./src/cli/idb/install-idb.ts",
|
|
15
15
|
"preflight-ios": "tsx ./src/cli/ios/preflight-ios.ts",
|
|
16
|
-
"test:unit": "tsx test/unit/index.ts",
|
|
16
|
+
"test:unit": "SKIP_DEVICE_TESTS=1 tsx test/unit/index.ts",
|
|
17
17
|
"test:integration": "npm run build && tsx test/device/index.ts",
|
|
18
18
|
"test:device": "npm run build && tsx test/device/index.ts",
|
|
19
19
|
"test": "npm run test:unit",
|
|
@@ -27,7 +27,8 @@
|
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"@modelcontextprotocol/sdk": "^1.0.0",
|
|
29
29
|
"fast-xml-parser": "^5.5.1",
|
|
30
|
-
"zod": "^3.22.4"
|
|
30
|
+
"zod": "^3.22.4",
|
|
31
|
+
"sharp": "^0.32.1"
|
|
31
32
|
},
|
|
32
33
|
"devDependencies": {
|
|
33
34
|
"@types/node": "^25.4.0",
|