mobile-debug-mcp 0.20.1 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/android.js +0 -27
- package/dist/interact/index.js +145 -124
- package/dist/interact/ios.js +0 -26
- package/dist/scripts/capture_ui_after_tap.mjs +43 -0
- package/dist/scripts/check_play_observe.mjs +18 -0
- package/dist/scripts/check_play_substring.mjs +38 -0
- package/dist/scripts/dump_ui_tree.mjs +20 -0
- package/dist/scripts/observe-test.mjs +32 -0
- package/dist/scripts/press_and_observe.mjs +90 -0
- package/dist/scripts/press_and_wait_ui.mjs +85 -0
- package/dist/scripts/test_generate_and_wait.mjs +123 -0
- package/dist/server.js +15 -25
- package/dist/system/gradle.js +4 -4
- package/dist/utils/android/utils.js +2 -2
- package/dist/utils/resolve-device.js +5 -0
- package/docs/CHANGELOG.md +7 -1
- package/docs/tools/interact.md +7 -27
- package/package.json +2 -2
- package/src/interact/android.ts +1 -32
- package/src/interact/index.ts +98 -78
- package/src/interact/ios.ts +1 -31
- package/src/server.ts +18 -25
- package/src/system/gradle.ts +4 -4
- package/src/utils/android/utils.ts +2 -2
- package/src/utils/resolve-device.ts +6 -0
- package/test/interact/device/run-real-test.ts +3 -19
- package/test/interact/unit/{observe_until.test.ts → wait_for_ui.test.ts} +6 -6
- package/test/observe/device/wait_for_element_real.ts +3 -80
- package/test/observe/unit/wait_for_element_mock.ts +2 -104
- package/test/observe/unit/wait_for_ui_edge_cases.test.ts +41 -0
- package/test/observe/unit/wait_for_ui_stability.test.ts +30 -0
- package/test/unit/index.ts +27 -15
- package/test/interact/device/observe_until_device.ts +0 -24
package/dist/interact/android.js
CHANGED
|
@@ -3,33 +3,6 @@ import { AndroidObserve } from "../observe/index.js";
|
|
|
3
3
|
import { scrollToElementShared } from "../utils/ui/index.js";
|
|
4
4
|
export class AndroidInteract {
|
|
5
5
|
observe = new AndroidObserve();
|
|
6
|
-
async waitForElement(text, timeout, deviceId) {
|
|
7
|
-
const metadata = await getAndroidDeviceMetadata("", deviceId);
|
|
8
|
-
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
|
|
9
|
-
const startTime = Date.now();
|
|
10
|
-
while (Date.now() - startTime < timeout) {
|
|
11
|
-
try {
|
|
12
|
-
const tree = await this.observe.getUITree(deviceId);
|
|
13
|
-
if (tree.error) {
|
|
14
|
-
return { device: deviceInfo, found: false, error: tree.error };
|
|
15
|
-
}
|
|
16
|
-
const element = tree.elements.find(e => e.text === text);
|
|
17
|
-
if (element) {
|
|
18
|
-
return { device: deviceInfo, found: true, element };
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
catch (e) {
|
|
22
|
-
// Ignore errors during polling and retry
|
|
23
|
-
console.error("Error polling UI tree:", e);
|
|
24
|
-
}
|
|
25
|
-
const elapsed = Date.now() - startTime;
|
|
26
|
-
const remaining = timeout - elapsed;
|
|
27
|
-
if (remaining <= 0)
|
|
28
|
-
break;
|
|
29
|
-
await new Promise(resolve => setTimeout(resolve, Math.min(500, remaining)));
|
|
30
|
-
}
|
|
31
|
-
return { device: deviceInfo, found: false };
|
|
32
|
-
}
|
|
33
6
|
async tap(x, y, deviceId) {
|
|
34
7
|
const metadata = await getAndroidDeviceMetadata("", deviceId);
|
|
35
8
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
|
package/dist/interact/index.js
CHANGED
|
@@ -3,7 +3,6 @@ import { iOSInteract } from './ios.js';
|
|
|
3
3
|
export { AndroidInteract, iOSInteract };
|
|
4
4
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
5
5
|
import { ToolsObserve } from '../observe/index.js';
|
|
6
|
-
const STABLE_IDLE_MS = 1000;
|
|
7
6
|
export class ToolsInteract {
|
|
8
7
|
static async getInteractionService(platform, deviceId) {
|
|
9
8
|
const effectivePlatform = platform || 'android';
|
|
@@ -11,11 +10,6 @@ export class ToolsInteract {
|
|
|
11
10
|
const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract();
|
|
12
11
|
return { interact: interact, resolved, platform: effectivePlatform };
|
|
13
12
|
}
|
|
14
|
-
static async waitForElementHandler({ platform, text, timeout, deviceId }) {
|
|
15
|
-
const effectiveTimeout = timeout ?? 10000;
|
|
16
|
-
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
17
|
-
return await interact.waitForElement(text, effectiveTimeout, resolved.id);
|
|
18
|
-
}
|
|
19
13
|
static async tapHandler({ platform, x, y, deviceId }) {
|
|
20
14
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
21
15
|
return await interact.tap(x, y, resolved.id);
|
|
@@ -221,6 +215,10 @@ export class ToolsInteract {
|
|
|
221
215
|
const scoreVal = Math.min(1, Number(bestScore.toFixed(3)));
|
|
222
216
|
return { found: true, element: outEl, score: scoreVal, confidence: scoreVal };
|
|
223
217
|
}
|
|
218
|
+
static async waitForUIHandler({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
|
|
219
|
+
// Backwards-compatible wrapper that delegates to the core waitForUICore implementation
|
|
220
|
+
return await ToolsInteract.waitForUICore({ type, query, timeoutMs, pollIntervalMs, includeSnapshotOnFailure, match, stability_ms, observationDelayMs, platform, deviceId });
|
|
221
|
+
}
|
|
224
222
|
static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }) {
|
|
225
223
|
const start = Date.now();
|
|
226
224
|
let lastFingerprint = null;
|
|
@@ -259,167 +257,190 @@ export class ToolsInteract {
|
|
|
259
257
|
}
|
|
260
258
|
return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start };
|
|
261
259
|
}
|
|
262
|
-
static async
|
|
260
|
+
static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
|
|
263
261
|
const start = Date.now();
|
|
264
262
|
const deadline = start + (timeoutMs || 0);
|
|
265
263
|
const q = (query === null || query === undefined) ? '' : String(query);
|
|
266
|
-
//
|
|
264
|
+
// Clamp polling interval to 250-500ms for consistent behavior
|
|
265
|
+
const pollInterval = Math.max(250, Math.min(pollIntervalMs || 300, 500));
|
|
266
|
+
// Baseline state (fetch in parallel but bound to short timeouts so observation starts promptly)
|
|
267
267
|
let initialFingerprint = null;
|
|
268
|
-
try {
|
|
269
|
-
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
270
|
-
initialFingerprint = fpRes?.fingerprint ?? null;
|
|
271
|
-
}
|
|
272
|
-
catch (err) {
|
|
273
|
-
console.error('observeUntil: error getting initial fingerprint', err);
|
|
274
|
-
initialFingerprint = null;
|
|
275
|
-
}
|
|
276
|
-
// For logs, capture a baseline snapshot (count or last line) to avoid matching historical lines
|
|
277
268
|
let baselineLastLine = null;
|
|
278
269
|
try {
|
|
279
|
-
const
|
|
280
|
-
const
|
|
281
|
-
|
|
270
|
+
const fpPromise = ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
271
|
+
const logsPromise = ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
|
|
272
|
+
const withTimeout = (p, ms) => Promise.race([p, new Promise(resolve => setTimeout(() => resolve(null), ms))]);
|
|
273
|
+
const [fpRes, gl] = await Promise.all([withTimeout(fpPromise, 300), withTimeout(logsPromise, 500)]);
|
|
274
|
+
if (fpRes && typeof fpRes === 'object')
|
|
275
|
+
initialFingerprint = fpRes.fingerprint ?? null;
|
|
276
|
+
if (gl) {
|
|
277
|
+
const logsArr = Array.isArray(gl.logs) ? gl.logs : [];
|
|
278
|
+
baselineLastLine = logsArr.length ? logsArr[logsArr.length - 1] : null;
|
|
279
|
+
}
|
|
282
280
|
}
|
|
283
281
|
catch (err) {
|
|
284
|
-
// non-fatal but surface warning to aid debugging
|
|
285
282
|
try {
|
|
286
|
-
console.warn('
|
|
283
|
+
console.warn('waitForUI: failed to get baseline data (non-fatal):', err instanceof Error ? err.message : String(err));
|
|
287
284
|
}
|
|
288
285
|
catch { }
|
|
289
286
|
}
|
|
287
|
+
// Network-based waiting removed. Rely on UI and screen fingerprints for determinism.
|
|
290
288
|
let lastChangeAt = Date.now();
|
|
291
289
|
let prevFingerprint = initialFingerprint;
|
|
292
290
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
291
|
+
// Optional initial observation delay requested by caller
|
|
292
|
+
if (typeof observationDelayMs === 'number' && observationDelayMs > 0) {
|
|
293
|
+
try {
|
|
294
|
+
console.log(`waitForUI: delaying observation for ${observationDelayMs}ms`);
|
|
295
|
+
}
|
|
296
|
+
catch { }
|
|
297
|
+
await sleep(observationDelayMs);
|
|
298
|
+
}
|
|
293
299
|
// Telemetry
|
|
294
300
|
let pollCount = 0;
|
|
295
|
-
let
|
|
301
|
+
let matchedAt = null;
|
|
302
|
+
let lastObservedState = null;
|
|
303
|
+
let stableDuration = 0;
|
|
296
304
|
let matchSource = null;
|
|
297
305
|
while (Date.now() <= deadline) {
|
|
298
306
|
pollCount++;
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
timeToMatch = Date.now() - start;
|
|
306
|
-
// determine matchSource heuristics
|
|
307
|
-
const el = found.element || {};
|
|
308
|
-
if (el && el.resourceId && String(el.resourceId).toLowerCase().includes(q.toLowerCase()))
|
|
309
|
-
matchSource = 'ui-resourceId';
|
|
310
|
-
else if (el && el.text && String(el.text).toLowerCase() === q.toLowerCase())
|
|
311
|
-
matchSource = 'ui-exact';
|
|
312
|
-
else
|
|
313
|
-
matchSource = 'ui-partial';
|
|
314
|
-
return { success: true, type: 'ui', matched: true, details: `UI element matched '${q}'`, timestamp: Date.now(), element: found.element, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
catch (err) {
|
|
318
|
-
console.error('observeUntil(ui) find error:', err);
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
else if (type === 'log') {
|
|
307
|
+
const now = Date.now();
|
|
308
|
+
// Evaluate condition per type
|
|
309
|
+
if (type === 'ui') {
|
|
310
|
+
try {
|
|
311
|
+
// Prefer using the public findElementHandler which tests can override. This avoids relying
|
|
312
|
+
// on resolveObserve/getUITree for unit tests which may not have devices available.
|
|
322
313
|
try {
|
|
323
|
-
|
|
324
|
-
const
|
|
325
|
-
const
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
314
|
+
const findRes = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, pollInterval), platform, deviceId });
|
|
315
|
+
const isPresent = !!(findRes && findRes.found);
|
|
316
|
+
const conditionTrue = (match === 'present') ? isPresent : !isPresent;
|
|
317
|
+
if (conditionTrue) {
|
|
318
|
+
if (matchedAt === null)
|
|
319
|
+
matchedAt = Date.now();
|
|
320
|
+
stableDuration = Date.now() - matchedAt;
|
|
321
|
+
lastObservedState = true;
|
|
322
|
+
if (stableDuration >= stability_ms) {
|
|
323
|
+
matchSource = 'ui-find';
|
|
324
|
+
const element = isPresent ? findRes.element : null;
|
|
325
|
+
const now2 = Date.now();
|
|
326
|
+
return { success: true, condition: match, query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, matchedElement: element, matchSource, timestamp: now2, type: 'ui', observed_state: lastObservedState ?? null };
|
|
332
327
|
}
|
|
333
328
|
}
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
let startIndex = 0;
|
|
339
|
-
if (baselineLastLine) {
|
|
340
|
-
const idx = logsArr.lastIndexOf(baselineLastLine);
|
|
341
|
-
startIndex = idx >= 0 ? idx + 1 : 0;
|
|
342
|
-
}
|
|
343
|
-
for (let i = startIndex; i < logsArr.length; i++) {
|
|
344
|
-
const line = logsArr[i];
|
|
345
|
-
if (q && String(line).includes(q)) {
|
|
346
|
-
timeToMatch = Date.now() - start;
|
|
347
|
-
matchSource = 'log-snapshot';
|
|
348
|
-
return { success: true, type: 'log', matched: true, details: `Log matched '${q}'`, timestamp: Date.now(), log: { message: line }, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
349
|
-
}
|
|
329
|
+
else {
|
|
330
|
+
matchedAt = null;
|
|
331
|
+
stableDuration = 0;
|
|
332
|
+
lastObservedState = false;
|
|
350
333
|
}
|
|
351
334
|
}
|
|
352
335
|
catch (err) {
|
|
353
|
-
console.error('
|
|
336
|
+
console.error('waitForUI(ui) find error:', err);
|
|
354
337
|
}
|
|
355
338
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
}
|
|
371
|
-
catch (err) {
|
|
372
|
-
console.error('observeUntil(screen) find error:', err);
|
|
373
|
-
}
|
|
374
|
-
// If query provided but not matched yet, continue polling until timeout
|
|
375
|
-
}
|
|
376
|
-
else {
|
|
377
|
-
timeToMatch = Date.now() - start;
|
|
378
|
-
matchSource = 'screen-fingerprint';
|
|
379
|
-
return { success: true, type: 'screen', matched: true, details: 'Screen fingerprint changed', timestamp: Date.now(), newFingerprint: fp, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
380
|
-
}
|
|
339
|
+
catch (err) {
|
|
340
|
+
console.error('waitForUI(ui) outer error:', err);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
else if (type === 'log') {
|
|
344
|
+
try {
|
|
345
|
+
// Logs: presence semantics only (match 'present'). Stability not applicable (immediate)
|
|
346
|
+
const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 });
|
|
347
|
+
const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : [];
|
|
348
|
+
for (const ent of entries) {
|
|
349
|
+
const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : '';
|
|
350
|
+
if (q && String(msg).includes(q)) {
|
|
351
|
+
const now2 = Date.now();
|
|
352
|
+
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: msg, raw: ent }, matchSource: 'log-stream', timestamp: now2, type: 'log', observed_state: true };
|
|
381
353
|
}
|
|
382
354
|
}
|
|
383
|
-
|
|
384
|
-
|
|
355
|
+
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
|
|
356
|
+
const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : [];
|
|
357
|
+
let startIndex = 0;
|
|
358
|
+
if (baselineLastLine) {
|
|
359
|
+
const idx = logsArr.lastIndexOf(baselineLastLine);
|
|
360
|
+
startIndex = idx >= 0 ? idx + 1 : 0;
|
|
361
|
+
}
|
|
362
|
+
for (let i = startIndex; i < logsArr.length; i++) {
|
|
363
|
+
const line = logsArr[i];
|
|
364
|
+
if (q && String(line).includes(q)) {
|
|
365
|
+
const now2 = Date.now();
|
|
366
|
+
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: 0, matchedLog: { message: line }, matchSource: 'log-snapshot', timestamp: now2, type: 'log', observed_state: true };
|
|
367
|
+
}
|
|
385
368
|
}
|
|
386
369
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
370
|
+
catch (err) {
|
|
371
|
+
console.error('waitForUI(log) error:', err);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
else if (type === 'screen') {
|
|
375
|
+
try {
|
|
376
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
377
|
+
const fp = fpRes?.fingerprint ?? null;
|
|
378
|
+
if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
|
|
379
|
+
// when screen changed, require stability_ms where fingerprint remains the same
|
|
380
|
+
if (matchedAt === null)
|
|
381
|
+
matchedAt = now;
|
|
382
|
+
const confirmFp = (await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }))?.fingerprint ?? null;
|
|
383
|
+
if (confirmFp === fp) {
|
|
384
|
+
stableDuration = Date.now() - matchedAt;
|
|
385
|
+
lastObservedState = true;
|
|
386
|
+
if (stableDuration >= stability_ms) {
|
|
387
|
+
const now2 = Date.now();
|
|
388
|
+
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: stableDuration, newFingerprint: fp, matchSource: 'screen-fingerprint', timestamp: now2, type: 'screen', observed_state: lastObservedState ?? null };
|
|
389
|
+
}
|
|
394
390
|
}
|
|
395
391
|
else {
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
return { success: true, type: 'idle', matched: true, details: `UI stable for ${STABLE_IDLE_MS}ms`, timestamp: Date.now(), fingerprint: fp, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
400
|
-
}
|
|
392
|
+
matchedAt = null;
|
|
393
|
+
stableDuration = 0;
|
|
394
|
+
lastObservedState = false;
|
|
401
395
|
}
|
|
402
396
|
}
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
397
|
+
}
|
|
398
|
+
catch (err) {
|
|
399
|
+
console.error('waitForUI(screen) error:', err);
|
|
406
400
|
}
|
|
407
401
|
}
|
|
408
|
-
|
|
409
|
-
|
|
402
|
+
else if (type === 'idle') {
|
|
403
|
+
try {
|
|
404
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
405
|
+
const fp = fpRes?.fingerprint ?? null;
|
|
406
|
+
if (fp !== prevFingerprint) {
|
|
407
|
+
prevFingerprint = fp;
|
|
408
|
+
lastChangeAt = Date.now();
|
|
409
|
+
matchedAt = null;
|
|
410
|
+
stableDuration = 0;
|
|
411
|
+
lastObservedState = false;
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
const idleMs = Date.now() - lastChangeAt;
|
|
415
|
+
lastObservedState = true;
|
|
416
|
+
if (idleMs >= stability_ms) {
|
|
417
|
+
const now2 = Date.now();
|
|
418
|
+
return { success: true, condition: 'present', query: q, poll_count: pollCount, duration_ms: now2 - start, stable_duration_ms: idleMs, matchSource: 'idle-stable', timestamp: now2, type: 'idle', observed_state: lastObservedState ?? null };
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
catch (err) {
|
|
423
|
+
console.error('waitForUI(idle) error:', err);
|
|
424
|
+
}
|
|
410
425
|
}
|
|
411
426
|
// Respect poll interval and avoid tight loop
|
|
412
|
-
await sleep(
|
|
427
|
+
await sleep(pollInterval);
|
|
413
428
|
}
|
|
414
|
-
// On timeout, capture a failure snapshot to aid debugging (best-effort)
|
|
429
|
+
// On timeout, optionally capture a failure snapshot to aid debugging (best-effort)
|
|
415
430
|
let snapshot = null;
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
431
|
+
if (includeSnapshotOnFailure) {
|
|
432
|
+
try {
|
|
433
|
+
// Use dynamic import to avoid circular-initialization issues where the ToolsObserve
|
|
434
|
+
// binding captured earlier may not reflect test-time overrides. Importing at call
|
|
435
|
+
// time ensures the latest exported ToolsObserve object is used.
|
|
436
|
+
const Obs = await import('../observe/index.js');
|
|
437
|
+
snapshot = await Obs.ToolsObserve.captureDebugSnapshotHandler({ reason: `wait_for_ui timeout for ${type}`, includeLogs: true, platform, deviceId });
|
|
438
|
+
}
|
|
439
|
+
catch (err) {
|
|
440
|
+
snapshot = { error: err instanceof Error ? err.message : String(err) };
|
|
441
|
+
}
|
|
421
442
|
}
|
|
422
443
|
const elapsed = Date.now() - start;
|
|
423
|
-
return { success: false,
|
|
444
|
+
return { success: false, condition: match, query: q, poll_count: pollCount, duration_ms: elapsed, stable_duration_ms: stableDuration, error: 'Timeout waiting for condition', snapshot, observed_state: lastObservedState ?? null };
|
|
424
445
|
}
|
|
425
446
|
}
|
package/dist/interact/ios.js
CHANGED
|
@@ -4,32 +4,6 @@ import { iOSObserve } from "../observe/index.js";
|
|
|
4
4
|
import { scrollToElementShared } from "../utils/ui/index.js";
|
|
5
5
|
export class iOSInteract {
|
|
6
6
|
observe = new iOSObserve();
|
|
7
|
-
async waitForElement(text, timeout, deviceId = "booted") {
|
|
8
|
-
const device = await getIOSDeviceMetadata(deviceId);
|
|
9
|
-
const startTime = Date.now();
|
|
10
|
-
while (Date.now() - startTime < timeout) {
|
|
11
|
-
try {
|
|
12
|
-
const tree = await this.observe.getUITree(deviceId);
|
|
13
|
-
if (tree.error) {
|
|
14
|
-
return { device, found: false, error: tree.error };
|
|
15
|
-
}
|
|
16
|
-
const element = tree.elements.find(e => e.text === text);
|
|
17
|
-
if (element) {
|
|
18
|
-
return { device, found: true, element };
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
catch (e) {
|
|
22
|
-
// Ignore errors during polling and retry
|
|
23
|
-
console.error("Error polling UI tree:", e);
|
|
24
|
-
}
|
|
25
|
-
const elapsed = Date.now() - startTime;
|
|
26
|
-
const remaining = timeout - elapsed;
|
|
27
|
-
if (remaining <= 0)
|
|
28
|
-
break;
|
|
29
|
-
await new Promise(resolve => setTimeout(resolve, Math.min(500, remaining)));
|
|
30
|
-
}
|
|
31
|
-
return { device, found: false };
|
|
32
|
-
}
|
|
33
7
|
async tap(x, y, deviceId = "booted") {
|
|
34
8
|
const device = await getIOSDeviceMetadata(deviceId);
|
|
35
9
|
// Use shared helper to detect idb
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { ToolsObserve } from '../observe/index.js'
|
|
2
|
+
import { ToolsInteract } from '../interact/index.js'
|
|
3
|
+
|
|
4
|
+
async function main() {
|
|
5
|
+
const platform = process.argv.includes('--platform') ? process.argv[process.argv.indexOf('--platform')+1] : 'android'
|
|
6
|
+
const device = process.argv.includes('--device') ? process.argv[process.argv.indexOf('--device')+1] : undefined
|
|
7
|
+
|
|
8
|
+
console.log('Tap then capture UI tree snapshots')
|
|
9
|
+
try {
|
|
10
|
+
// Tap generate
|
|
11
|
+
const found = await (await import('../interact/index.js')).ToolsInteract.findElementHandler({ query: 'Generate Session', platform, timeoutMs: 3000, exact: true, deviceId: device })
|
|
12
|
+
console.log('Found generate?', !!(found && found.found), JSON.stringify(found && found.element || {}))
|
|
13
|
+
if (found && found.found && found.element && found.element.tapCoordinates) {
|
|
14
|
+
await (await import('../interact/index.js')).ToolsInteract.tapHandler({ platform, x: found.element.tapCoordinates.x, y: found.element.tapCoordinates.y, deviceId: device })
|
|
15
|
+
console.log('Tapped Generate at', found.element.tapCoordinates)
|
|
16
|
+
} else {
|
|
17
|
+
console.log('Could not find generate element — aborting capture')
|
|
18
|
+
process.exit(2)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Capture UI trees and screenshot repeatedly for 20s
|
|
22
|
+
const start = Date.now()
|
|
23
|
+
const outDir = '/tmp/mcp_ui_captures'
|
|
24
|
+
try { await import('fs').then(fs=>fs.promises.mkdir(outDir,{recursive:true})) } catch(e){}
|
|
25
|
+
let i = 0
|
|
26
|
+
while (Date.now() - start < 20000) {
|
|
27
|
+
const snap = await ToolsObserve.getUITreeHandler({ platform, deviceId: device })
|
|
28
|
+
const fname = `${outDir}/ui_${Date.now()}_${i}.json`
|
|
29
|
+
await import('fs').then(fs=>fs.promises.writeFile(fname, JSON.stringify(snap,null,2)))
|
|
30
|
+
console.log('Saved', fname)
|
|
31
|
+
i++
|
|
32
|
+
await new Promise(r=>setTimeout(r,500))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
console.log('Capture complete')
|
|
36
|
+
process.exit(0)
|
|
37
|
+
} catch (e) {
|
|
38
|
+
console.error('Error during capture:', e)
|
|
39
|
+
process.exit(1)
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
main()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { ToolsInteract } from '../interact/index.js'
|
|
2
|
+
|
|
3
|
+
async function main() {
|
|
4
|
+
const platform = process.argv.includes('--platform') ? process.argv[process.argv.indexOf('--platform')+1] : 'android'
|
|
5
|
+
const device = process.argv.includes('--device') ? process.argv[process.argv.indexOf('--device')+1] : undefined
|
|
6
|
+
const timeout = parseInt(process.argv.includes('--timeout') ? process.argv[process.argv.indexOf('--timeout')+1] : '120000')
|
|
7
|
+
console.log('Running observeUntil for Play session, platform=', platform, 'device=', device)
|
|
8
|
+
try {
|
|
9
|
+
const res = await ToolsInteract.observeUntilHandler({ type: 'ui', query: 'Play session', match: 'present', timeoutMs: timeout, pollIntervalMs: 300, stability_ms: 700, platform, deviceId: device })
|
|
10
|
+
console.log('observeUntil result:', JSON.stringify(res, null, 2))
|
|
11
|
+
process.exit(res && res.success ? 0 : 2)
|
|
12
|
+
} catch (e) {
|
|
13
|
+
console.error('Error running observeUntil:', e)
|
|
14
|
+
process.exit(1)
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
main()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { ToolsObserve } from '../observe/index.js'
|
|
2
|
+
|
|
3
|
+
async function main() {
|
|
4
|
+
const platform = process.argv.includes('--platform') ? process.argv[process.argv.indexOf('--platform')+1] : 'android'
|
|
5
|
+
const device = process.argv.includes('--device') ? process.argv[process.argv.indexOf('--device')+1] : undefined
|
|
6
|
+
const timeout = parseInt(process.argv.includes('--timeout') ? process.argv[process.argv.indexOf('--timeout')+1] : '120000')
|
|
7
|
+
const start = Date.now()
|
|
8
|
+
const out = []
|
|
9
|
+
console.log('Searching UI trees for elements containing "play" (case-insensitive)')
|
|
10
|
+
while (Date.now() - start < timeout) {
|
|
11
|
+
try {
|
|
12
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId: device })
|
|
13
|
+
const elements = Array.isArray((tree && tree.elements)) ? tree.elements : []
|
|
14
|
+
const matches = []
|
|
15
|
+
const norm = (s)=> s ? String(s).toLowerCase() : ''
|
|
16
|
+
for (const el of elements) {
|
|
17
|
+
const text = norm(el.text)
|
|
18
|
+
const content = norm(el.contentDescription || el.contentDesc || el.accessibilityLabel)
|
|
19
|
+
const rid = norm(el.resourceId || el.id)
|
|
20
|
+
const cls = norm(el.type || el.class)
|
|
21
|
+
if (text.includes('play') || content.includes('play') || rid.includes('play') || cls.includes('play') || text.includes('session') || content.includes('session')) {
|
|
22
|
+
matches.push({ el: el, foundIn: { text: text.includes('play')? 'text': text.includes('session')? 'text-session': null } })
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
if (matches.length) {
|
|
26
|
+
console.log('Matches found:', JSON.stringify(matches, null, 2))
|
|
27
|
+
out.push({ ts: Date.now(), matches })
|
|
28
|
+
break
|
|
29
|
+
}
|
|
30
|
+
} catch (e) { console.error('UITree fetch err:', e) }
|
|
31
|
+
await new Promise(r=>setTimeout(r, 500))
|
|
32
|
+
}
|
|
33
|
+
console.log('Search complete. Results count=', out.length)
|
|
34
|
+
if (out.length > 0) process.exit(0)
|
|
35
|
+
process.exit(2)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
main()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { ToolsObserve } from '../observe/index.js'
|
|
2
|
+
|
|
3
|
+
async function main() {
|
|
4
|
+
const argv = process.argv.slice(2)
|
|
5
|
+
const opts = {}
|
|
6
|
+
for (let i = 0; i < argv.length; i++) {
|
|
7
|
+
if (argv[i] === '--platform') opts.platform = argv[++i]
|
|
8
|
+
else if (argv[i] === '--deviceId') opts.deviceId = argv[++i]
|
|
9
|
+
}
|
|
10
|
+
try {
|
|
11
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform: opts.platform || 'android', deviceId: opts.deviceId })
|
|
12
|
+
console.log(JSON.stringify(tree, null, 2))
|
|
13
|
+
process.exit(0)
|
|
14
|
+
} catch (e) {
|
|
15
|
+
console.error('failed to get ui tree:', e instanceof Error ? e.message : String(e))
|
|
16
|
+
process.exit(2)
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
main()
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { ToolsObserve } from '../observe/index.js'
|
|
2
|
+
import { ToolsInteract } from '../interact/index.js'
|
|
3
|
+
|
|
4
|
+
async function main() {
|
|
5
|
+
console.log('Starting log stream for com.ideamechanics.modul8')
|
|
6
|
+
try {
|
|
7
|
+
const start = await ToolsObserve.startLogStreamHandler({ platform: 'ios', packageName: 'com.ideamechanics.modul8', sessionId: 'test-session' })
|
|
8
|
+
console.log('startLogStream result:', JSON.stringify(start))
|
|
9
|
+
} catch (e) {
|
|
10
|
+
console.error('startLogStream failed:', e instanceof Error ? e.message : String(e))
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
console.log('\nPlease press the Generate Session button in the app now. Observing for network idle (15s timeout)...')
|
|
14
|
+
|
|
15
|
+
try {
|
|
16
|
+
const res = await ToolsInteract.observeUntilHandler({ type: 'network_idle', timeoutMs: 15000, pollIntervalMs: 300, platform: 'ios' })
|
|
17
|
+
console.log('observeUntil result:')
|
|
18
|
+
console.log(JSON.stringify(res, null, 2))
|
|
19
|
+
} catch (e) {
|
|
20
|
+
console.error('observeUntil failed:', e instanceof Error ? e.message : String(e))
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
console.log('\nStopping log stream (best-effort)')
|
|
24
|
+
try {
|
|
25
|
+
const stop = await ToolsObserve.stopLogStreamHandler({ platform: 'ios', sessionId: 'test-session' })
|
|
26
|
+
console.log('stopLogStream result:', JSON.stringify(stop))
|
|
27
|
+
} catch (e) { console.error('stopLogStream failed:', e) }
|
|
28
|
+
|
|
29
|
+
process.exit(0)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
main().catch(err => { console.error(err); process.exit(1) })
|