mobile-debug-mcp 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +164 -0
- package/docs/CHANGELOG.md +5 -0
- package/docs/tools/interact.md +52 -0
- package/package.json +1 -1
- package/src/interact/index.ts +145 -0
- package/test/interact/device/observe_until_device.ts +24 -0
- package/test/interact/unit/observe_until.test.ts +76 -0
- package/test/unit/index.ts +1 -0
package/dist/interact/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { iOSInteract } from './ios.js';
|
|
|
3
3
|
export { AndroidInteract, iOSInteract };
|
|
4
4
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
5
5
|
import { ToolsObserve } from '../observe/index.js';
|
|
6
|
+
const STABLE_IDLE_MS = 1000;
|
|
6
7
|
export class ToolsInteract {
|
|
7
8
|
static async getInteractionService(platform, deviceId) {
|
|
8
9
|
const effectivePlatform = platform || 'android';
|
|
@@ -258,4 +259,167 @@ export class ToolsInteract {
|
|
|
258
259
|
}
|
|
259
260
|
return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start };
|
|
260
261
|
}
|
|
262
|
+
static async observeUntilHandler({ type, query, timeoutMs = 5000, pollIntervalMs = 200, platform, deviceId }) {
|
|
263
|
+
const start = Date.now();
|
|
264
|
+
const deadline = start + (timeoutMs || 0);
|
|
265
|
+
const q = (query === null || query === undefined) ? '' : String(query);
|
|
266
|
+
// Baseline state
|
|
267
|
+
let initialFingerprint = null;
|
|
268
|
+
try {
|
|
269
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
270
|
+
initialFingerprint = fpRes?.fingerprint ?? null;
|
|
271
|
+
}
|
|
272
|
+
catch (err) {
|
|
273
|
+
console.error('observeUntil: error getting initial fingerprint', err);
|
|
274
|
+
initialFingerprint = null;
|
|
275
|
+
}
|
|
276
|
+
// For logs, capture a baseline snapshot (count or last line) to avoid matching historical lines
|
|
277
|
+
let baselineLastLine = null;
|
|
278
|
+
try {
|
|
279
|
+
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
|
|
280
|
+
const logsArr = Array.isArray(gl.logs) ? gl.logs : [];
|
|
281
|
+
baselineLastLine = logsArr.length ? logsArr[logsArr.length - 1] : null;
|
|
282
|
+
}
|
|
283
|
+
catch (err) {
|
|
284
|
+
// non-fatal but surface warning to aid debugging
|
|
285
|
+
try {
|
|
286
|
+
console.warn('observeUntil: failed to get baseline logs (non-fatal):', err instanceof Error ? err.message : String(err));
|
|
287
|
+
}
|
|
288
|
+
catch { }
|
|
289
|
+
}
|
|
290
|
+
let lastChangeAt = Date.now();
|
|
291
|
+
let prevFingerprint = initialFingerprint;
|
|
292
|
+
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
293
|
+
// Telemetry
|
|
294
|
+
let pollCount = 0;
|
|
295
|
+
let timeToMatch = null;
|
|
296
|
+
let matchSource = null;
|
|
297
|
+
while (Date.now() <= deadline) {
|
|
298
|
+
pollCount++;
|
|
299
|
+
try {
|
|
300
|
+
if (type === 'ui') {
|
|
301
|
+
// fast findElement with short timeout to avoid blocking
|
|
302
|
+
try {
|
|
303
|
+
const found = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, timeoutMs || 500), platform, deviceId });
|
|
304
|
+
if (found && found.found) {
|
|
305
|
+
timeToMatch = Date.now() - start;
|
|
306
|
+
// determine matchSource heuristics
|
|
307
|
+
const el = found.element || {};
|
|
308
|
+
if (el && el.resourceId && String(el.resourceId).toLowerCase().includes(q.toLowerCase()))
|
|
309
|
+
matchSource = 'ui-resourceId';
|
|
310
|
+
else if (el && el.text && String(el.text).toLowerCase() === q.toLowerCase())
|
|
311
|
+
matchSource = 'ui-exact';
|
|
312
|
+
else
|
|
313
|
+
matchSource = 'ui-partial';
|
|
314
|
+
return { success: true, type: 'ui', matched: true, details: `UI element matched '${q}'`, timestamp: Date.now(), element: found.element, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
catch (err) {
|
|
318
|
+
console.error('observeUntil(ui) find error:', err);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
else if (type === 'log') {
|
|
322
|
+
try {
|
|
323
|
+
// Try reading from active stream first
|
|
324
|
+
const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 });
|
|
325
|
+
const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : [];
|
|
326
|
+
for (const ent of entries) {
|
|
327
|
+
const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : '';
|
|
328
|
+
if (q && String(msg).includes(q)) {
|
|
329
|
+
timeToMatch = Date.now() - start;
|
|
330
|
+
matchSource = 'log-stream';
|
|
331
|
+
return { success: true, type: 'log', matched: true, details: `Log matched '${q}'`, timestamp: Date.now(), log: { message: msg, raw: ent }, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
// Fallback to snapshot logs
|
|
335
|
+
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 });
|
|
336
|
+
const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : [];
|
|
337
|
+
// Only consider new lines after baselineLastLine when possible
|
|
338
|
+
let startIndex = 0;
|
|
339
|
+
if (baselineLastLine) {
|
|
340
|
+
const idx = logsArr.lastIndexOf(baselineLastLine);
|
|
341
|
+
startIndex = idx >= 0 ? idx + 1 : 0;
|
|
342
|
+
}
|
|
343
|
+
for (let i = startIndex; i < logsArr.length; i++) {
|
|
344
|
+
const line = logsArr[i];
|
|
345
|
+
if (q && String(line).includes(q)) {
|
|
346
|
+
timeToMatch = Date.now() - start;
|
|
347
|
+
matchSource = 'log-snapshot';
|
|
348
|
+
return { success: true, type: 'log', matched: true, details: `Log matched '${q}'`, timestamp: Date.now(), log: { message: line }, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
catch (err) {
|
|
353
|
+
console.error('observeUntil(log) error:', err);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
else if (type === 'screen') {
|
|
357
|
+
try {
|
|
358
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
359
|
+
const fp = fpRes?.fingerprint ?? null;
|
|
360
|
+
if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
|
|
361
|
+
if (q) {
|
|
362
|
+
// optionally validate query against new screen context
|
|
363
|
+
try {
|
|
364
|
+
const found = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, timeoutMs || 500), platform, deviceId });
|
|
365
|
+
if (found && found.found) {
|
|
366
|
+
timeToMatch = Date.now() - start;
|
|
367
|
+
matchSource = 'screen-validated-ui';
|
|
368
|
+
return { success: true, type: 'screen', matched: true, details: `Screen changed and query matched on new screen`, timestamp: Date.now(), newFingerprint: fp, element: found.element, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
catch (err) {
|
|
372
|
+
console.error('observeUntil(screen) find error:', err);
|
|
373
|
+
}
|
|
374
|
+
// If query provided but not matched yet, continue polling until timeout
|
|
375
|
+
}
|
|
376
|
+
else {
|
|
377
|
+
timeToMatch = Date.now() - start;
|
|
378
|
+
matchSource = 'screen-fingerprint';
|
|
379
|
+
return { success: true, type: 'screen', matched: true, details: 'Screen fingerprint changed', timestamp: Date.now(), newFingerprint: fp, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
catch (err) {
|
|
384
|
+
console.error('observeUntil(screen) error:', err);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
else if (type === 'idle') {
|
|
388
|
+
try {
|
|
389
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
390
|
+
const fp = fpRes?.fingerprint ?? null;
|
|
391
|
+
if (fp !== prevFingerprint) {
|
|
392
|
+
prevFingerprint = fp;
|
|
393
|
+
lastChangeAt = Date.now();
|
|
394
|
+
}
|
|
395
|
+
else {
|
|
396
|
+
if (Date.now() - lastChangeAt >= STABLE_IDLE_MS) {
|
|
397
|
+
timeToMatch = Date.now() - start;
|
|
398
|
+
matchSource = 'idle-stable';
|
|
399
|
+
return { success: true, type: 'idle', matched: true, details: `UI stable for ${STABLE_IDLE_MS}ms`, timestamp: Date.now(), fingerprint: fp, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } };
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
catch (err) {
|
|
404
|
+
console.error('observeUntil(idle) error:', err);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
catch (err) {
|
|
409
|
+
console.error('observeUntil: unexpected error', err);
|
|
410
|
+
}
|
|
411
|
+
// Respect poll interval and avoid tight loop
|
|
412
|
+
await sleep(pollIntervalMs || 200);
|
|
413
|
+
}
|
|
414
|
+
// On timeout, capture a failure snapshot to aid debugging (best-effort)
|
|
415
|
+
let snapshot = null;
|
|
416
|
+
try {
|
|
417
|
+
snapshot = await ToolsObserve.captureDebugSnapshotHandler({ reason: `observe_until timeout for ${type}`, includeLogs: true, platform, deviceId });
|
|
418
|
+
}
|
|
419
|
+
catch (err) {
|
|
420
|
+
snapshot = { error: err instanceof Error ? err.message : String(err) };
|
|
421
|
+
}
|
|
422
|
+
const elapsed = Date.now() - start;
|
|
423
|
+
return { success: false, error: 'Timeout waiting for condition', type, timeoutMs, telemetry: { pollCount, elapsedMs: elapsed, matchSource: null }, snapshot };
|
|
424
|
+
}
|
|
261
425
|
}
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the **Mobile Debug MCP** project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.19.0]
|
|
6
|
+
|
|
7
|
+
- Added `observe_until` interaction tool: waits for UI, log, screen fingerprint or idle conditions with configurable polling and timeout. Returns rich details on match (element info, log line, new fingerprint).
|
|
8
|
+
|
|
9
|
+
|
|
5
10
|
## [0.18.0]
|
|
6
11
|
- Added `find_element` interact tool: semantic UI element search with actionable tap coordinates and lightweight telemetry. The tool searches the UI tree for the best match by text, content description, resource-id, and class, scores candidates (exact, partial, resource-id), and returns the most relevant visible element. When a matching node is non-interactable (e.g., Compose Text child), the tool locates a clickable ancestor (parent or containing element) and returns actionable tapCoordinates (x,y). The handler also returns a `confidence` value and `telemetry` metadata (matchedIndex, matchedInteractable) to aid agent decision-making and logging. Implemented as `ToolsInteract.findElementHandler` and covered by unit tests.
|
|
7
12
|
|
package/docs/tools/interact.md
CHANGED
|
@@ -151,3 +151,55 @@ Notes:
|
|
|
151
151
|
- The tool favours actionable (clickable/focusable) targets; when a matching node is not directly actionable, it finds the smallest containing clickable ancestor.
|
|
152
152
|
- Unit tests for edge cases (parent-clickable child-text, resource-id matches, fuzzy matching) are under `test/observe/unit/find_element.test.ts`.
|
|
153
153
|
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## observe_until
|
|
157
|
+
|
|
158
|
+
Purpose:
|
|
159
|
+
- Wait for a condition to occur on the device: UI element appearance, a log line, a screen fingerprint change, or an idle/stable screen state.
|
|
160
|
+
|
|
161
|
+
Supported types and behavior:
|
|
162
|
+
- ui: Delegates to `find_element` to perform a semantic search of the UI tree. Returns the matched element descriptor (including tapCoordinates) when found.
|
|
163
|
+
- log: Reads the active log stream (via `start_log_stream`/`readLogStreamHandler`) and falls back to a snapshot of recent logs (`getLogsHandler`). Matches when the query substring appears in a new log line after a captured baseline.
|
|
164
|
+
- screen: Compares screen fingerprints (visual checks) against an initial baseline and returns when fingerprint changes. If `query` is provided it will attempt a `find_element` on the new screen to validate the expected content.
|
|
165
|
+
- idle: Waits until the screen fingerprint remains stable for a short stability window (default 1000ms).
|
|
166
|
+
|
|
167
|
+
Input (ToolsInteract.observeUntilHandler):
|
|
168
|
+
```
|
|
169
|
+
{ "type": "ui|log|screen|idle", "query": "optional string", "timeoutMs": 5000, "pollIntervalMs": 200, "platform": "android|ios", "deviceId": "optional device id" }
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Success response highlights:
|
|
173
|
+
- success: true
|
|
174
|
+
- type: requested type
|
|
175
|
+
- matched: true
|
|
176
|
+
- details: human-friendly explanation
|
|
177
|
+
- timestamp: epoch ms
|
|
178
|
+
- element: (for ui/screen when matched) actionable element metadata with tapCoordinates
|
|
179
|
+
- log: (for log) matched log message and raw entry
|
|
180
|
+
- newFingerprint: (for screen) new fingerprint value
|
|
181
|
+
|
|
182
|
+
Failure/timeout response:
|
|
183
|
+
- success: false
|
|
184
|
+
- error or reason: explanation
|
|
185
|
+
- type: requested type
|
|
186
|
+
- timeoutMs: value used
|
|
187
|
+
|
|
188
|
+
Notes & tips:
|
|
189
|
+
- Defaults (timeoutMs=5000, pollIntervalMs=200) balance responsiveness with device query overhead; adjust in tests or scripts as needed.
|
|
190
|
+
- For UI-sensitive flows prefer type='ui' rather than relying solely on visual fingerprint changes, as some UI updates don't alter the fingerprint.
|
|
191
|
+
|
|
192
|
+
Tests:
|
|
193
|
+
- Unit: `test/interact/unit/observe_until.test.ts`
|
|
194
|
+
- Device runner: `test/interact/device/observe_until_device.ts` (requires devices/emulators and adb/xcrun in PATH)
|
|
195
|
+
|
|
196
|
+
Example:
|
|
197
|
+
```
|
|
198
|
+
// Wait up to 5s for a button labeled "Generate Session" on Android
|
|
199
|
+
ToolsInteract.observeUntilHandler({ type: 'ui', query: 'Generate Session', timeoutMs: 5000, platform: 'android' })
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Troubleshooting:
|
|
203
|
+
- If observe_until(log) never matches, ensure log streaming is started for the target package and baseline logs captured correctly.
|
|
204
|
+
- If observe_until(screen) times out despite visible UI change, try type='ui' to validate content-level changes.
|
|
205
|
+
|
package/package.json
CHANGED
package/src/interact/index.ts
CHANGED
|
@@ -29,6 +29,8 @@ interface UiElement {
|
|
|
29
29
|
_interactable?: boolean
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
const STABLE_IDLE_MS = 1000
|
|
33
|
+
|
|
32
34
|
export class ToolsInteract {
|
|
33
35
|
|
|
34
36
|
private static async getInteractionService(platform?: 'android' | 'ios', deviceId?: string) {
|
|
@@ -260,4 +262,147 @@ export class ToolsInteract {
|
|
|
260
262
|
return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start }
|
|
261
263
|
}
|
|
262
264
|
|
|
265
|
+
static async observeUntilHandler({ type, query, timeoutMs = 5000, pollIntervalMs = 200, platform, deviceId }: { type: 'ui' | 'log' | 'screen' | 'idle', query?: string, timeoutMs?: number, pollIntervalMs?: number, platform?: 'android' | 'ios', deviceId?: string }) {
|
|
266
|
+
const start = Date.now()
|
|
267
|
+
const deadline = start + (timeoutMs || 0)
|
|
268
|
+
const q = (query === null || query === undefined) ? '' : String(query)
|
|
269
|
+
|
|
270
|
+
// Baseline state
|
|
271
|
+
let initialFingerprint: string | null = null
|
|
272
|
+
try {
|
|
273
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
274
|
+
initialFingerprint = fpRes?.fingerprint ?? null
|
|
275
|
+
} catch (err) { console.error('observeUntil: error getting initial fingerprint', err); initialFingerprint = null }
|
|
276
|
+
|
|
277
|
+
// For logs, capture a baseline snapshot (count or last line) to avoid matching historical lines
|
|
278
|
+
let baselineLastLine: string | null = null
|
|
279
|
+
try {
|
|
280
|
+
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 })
|
|
281
|
+
const logsArr = Array.isArray((gl as any).logs) ? (gl as any).logs : []
|
|
282
|
+
baselineLastLine = logsArr.length ? logsArr[logsArr.length - 1] : null
|
|
283
|
+
} catch (err) {
|
|
284
|
+
// non-fatal but surface warning to aid debugging
|
|
285
|
+
try { console.warn('observeUntil: failed to get baseline logs (non-fatal):', err instanceof Error ? err.message : String(err)) } catch { }
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
let lastChangeAt = Date.now()
|
|
290
|
+
let prevFingerprint = initialFingerprint
|
|
291
|
+
|
|
292
|
+
const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))
|
|
293
|
+
|
|
294
|
+
// Telemetry
|
|
295
|
+
let pollCount = 0
|
|
296
|
+
let timeToMatch: number | null = null
|
|
297
|
+
let matchSource: string | null = null
|
|
298
|
+
|
|
299
|
+
while (Date.now() <= deadline) {
|
|
300
|
+
pollCount++
|
|
301
|
+
try {
|
|
302
|
+
if (type === 'ui') {
|
|
303
|
+
// fast findElement with short timeout to avoid blocking
|
|
304
|
+
try {
|
|
305
|
+
const found = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, timeoutMs || 500), platform, deviceId })
|
|
306
|
+
if (found && (found as any).found) {
|
|
307
|
+
timeToMatch = Date.now() - start
|
|
308
|
+
// determine matchSource heuristics
|
|
309
|
+
const el = (found as any).element || {}
|
|
310
|
+
if (el && el.resourceId && String(el.resourceId).toLowerCase().includes(q.toLowerCase())) matchSource = 'ui-resourceId'
|
|
311
|
+
else if (el && el.text && String(el.text).toLowerCase() === q.toLowerCase()) matchSource = 'ui-exact'
|
|
312
|
+
else matchSource = 'ui-partial'
|
|
313
|
+
|
|
314
|
+
return { success: true, type: 'ui', matched: true, details: `UI element matched '${q}'`, timestamp: Date.now(), element: (found as any).element, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } }
|
|
315
|
+
}
|
|
316
|
+
} catch (err) { console.error('observeUntil(ui) find error:', err) }
|
|
317
|
+
} else if (type === 'log') {
|
|
318
|
+
try {
|
|
319
|
+
// Try reading from active stream first
|
|
320
|
+
const stream = await ToolsObserve.readLogStreamHandler({ platform, sessionId: 'default', limit: 200 }) as any
|
|
321
|
+
const entries = (stream && Array.isArray(stream.entries)) ? stream.entries : []
|
|
322
|
+
for (const ent of entries) {
|
|
323
|
+
const msg = ent && (ent.message || ent.msg || ent) ? (ent.message || ent.msg || ent) : ''
|
|
324
|
+
if (q && String(msg).includes(q)) {
|
|
325
|
+
timeToMatch = Date.now() - start
|
|
326
|
+
matchSource = 'log-stream'
|
|
327
|
+
return { success: true, type: 'log', matched: true, details: `Log matched '${q}'`, timestamp: Date.now(), log: { message: msg, raw: ent }, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } }
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// Fallback to snapshot logs
|
|
332
|
+
const gl = await ToolsObserve.getLogsHandler({ platform, deviceId, lines: 200 }) as any
|
|
333
|
+
const logsArr = Array.isArray(gl && gl.logs) ? gl.logs : []
|
|
334
|
+
// Only consider new lines after baselineLastLine when possible
|
|
335
|
+
let startIndex = 0
|
|
336
|
+
if (baselineLastLine) {
|
|
337
|
+
const idx = logsArr.lastIndexOf(baselineLastLine)
|
|
338
|
+
startIndex = idx >= 0 ? idx + 1 : 0
|
|
339
|
+
}
|
|
340
|
+
for (let i = startIndex; i < logsArr.length; i++) {
|
|
341
|
+
const line = logsArr[i]
|
|
342
|
+
if (q && String(line).includes(q)) {
|
|
343
|
+
timeToMatch = Date.now() - start
|
|
344
|
+
matchSource = 'log-snapshot'
|
|
345
|
+
return { success: true, type: 'log', matched: true, details: `Log matched '${q}'`, timestamp: Date.now(), log: { message: line }, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } }
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
} catch (err) { console.error('observeUntil(log) error:', err) }
|
|
349
|
+
} else if (type === 'screen') {
|
|
350
|
+
try {
|
|
351
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
352
|
+
const fp = fpRes?.fingerprint ?? null
|
|
353
|
+
if (fp !== null && fp !== undefined && fp !== initialFingerprint) {
|
|
354
|
+
if (q) {
|
|
355
|
+
// optionally validate query against new screen context
|
|
356
|
+
try {
|
|
357
|
+
const found = await ToolsInteract.findElementHandler({ query: q, exact: false, timeoutMs: Math.min(500, timeoutMs || 500), platform, deviceId })
|
|
358
|
+
if (found && (found as any).found) {
|
|
359
|
+
timeToMatch = Date.now() - start
|
|
360
|
+
matchSource = 'screen-validated-ui'
|
|
361
|
+
return { success: true, type: 'screen', matched: true, details: `Screen changed and query matched on new screen`, timestamp: Date.now(), newFingerprint: fp, element: (found as any).element, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } }
|
|
362
|
+
}
|
|
363
|
+
} catch (err) { console.error('observeUntil(screen) find error:', err) }
|
|
364
|
+
// If query provided but not matched yet, continue polling until timeout
|
|
365
|
+
} else {
|
|
366
|
+
timeToMatch = Date.now() - start
|
|
367
|
+
matchSource = 'screen-fingerprint'
|
|
368
|
+
return { success: true, type: 'screen', matched: true, details: 'Screen fingerprint changed', timestamp: Date.now(), newFingerprint: fp, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } }
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
} catch (err) { console.error('observeUntil(screen) error:', err) }
|
|
372
|
+
} else if (type === 'idle') {
|
|
373
|
+
try {
|
|
374
|
+
const fpRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
375
|
+
const fp = fpRes?.fingerprint ?? null
|
|
376
|
+
if (fp !== prevFingerprint) {
|
|
377
|
+
prevFingerprint = fp
|
|
378
|
+
lastChangeAt = Date.now()
|
|
379
|
+
} else {
|
|
380
|
+
if (Date.now() - lastChangeAt >= STABLE_IDLE_MS) {
|
|
381
|
+
timeToMatch = Date.now() - start
|
|
382
|
+
matchSource = 'idle-stable'
|
|
383
|
+
return { success: true, type: 'idle', matched: true, details: `UI stable for ${STABLE_IDLE_MS}ms`, timestamp: Date.now(), fingerprint: fp, telemetry: { pollCount, timeToMatch, elapsedMs: Date.now() - start, matchSource } }
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
} catch (err) { console.error('observeUntil(idle) error:', err) }
|
|
387
|
+
}
|
|
388
|
+
} catch (err) {
|
|
389
|
+
console.error('observeUntil: unexpected error', err)
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// Respect poll interval and avoid tight loop
|
|
393
|
+
await sleep(pollIntervalMs || 200)
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// On timeout, capture a failure snapshot to aid debugging (best-effort)
|
|
397
|
+
let snapshot: any = null
|
|
398
|
+
try {
|
|
399
|
+
snapshot = await ToolsObserve.captureDebugSnapshotHandler({ reason: `observe_until timeout for ${type}`, includeLogs: true, platform, deviceId })
|
|
400
|
+
} catch (err) {
|
|
401
|
+
snapshot = { error: err instanceof Error ? err.message : String(err) }
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const elapsed = Date.now() - start
|
|
405
|
+
return { success: false, error: 'Timeout waiting for condition', type, timeoutMs, telemetry: { pollCount, elapsedMs: elapsed, matchSource: null }, snapshot }
|
|
406
|
+
}
|
|
407
|
+
|
|
263
408
|
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
(async function main(){
|
|
2
|
+
try{
|
|
3
|
+
const inter = await import('../../src/interact/index.ts')
|
|
4
|
+
const manage = await import('../../src/manage/index.ts')
|
|
5
|
+
const ToolsInteract = (inter as any).ToolsInteract
|
|
6
|
+
const ToolsManage = (manage as any).ToolsManage
|
|
7
|
+
|
|
8
|
+
const ANDROID_ID = process.env.ANDROID_DEVICE || 'emulator-5554'
|
|
9
|
+
const IOS_UDID = process.env.IOS_DEVICE || '2EFFD8FD-5D09-47CC-95F8-28BBE30AF7ED'
|
|
10
|
+
console.log('Device test starting. Android:', ANDROID_ID, 'iOS:', IOS_UDID)
|
|
11
|
+
|
|
12
|
+
// Start modul8 on both platforms if present
|
|
13
|
+
try { await ToolsManage.startAppHandler({ platform: 'android', appId: 'com.ideamechanics.modul8', deviceId: ANDROID_ID }); console.log('Started android app (if installed)') } catch(e){ console.error('Android start skipped:', e.message || e) }
|
|
14
|
+
try { await ToolsManage.startAppHandler({ platform: 'ios', appId: 'com.ideamechanics.modul8.Modul8', deviceId: IOS_UDID }); console.log('Started ios app (if installed)') } catch(e){ console.error('iOS start skipped:', e.message || e) }
|
|
15
|
+
|
|
16
|
+
// Observe UI for Generate Session on both devices (will timeout if not present)
|
|
17
|
+
const aRes = await ToolsInteract.observeUntilHandler({ type: 'ui', query: 'Generate Session', timeoutMs: 20000, pollIntervalMs: 500, platform: 'android', deviceId: ANDROID_ID })
|
|
18
|
+
console.log('Android observe result:', JSON.stringify(aRes, null, 2))
|
|
19
|
+
|
|
20
|
+
const iRes = await ToolsInteract.observeUntilHandler({ type: 'ui', query: 'Generate Session', timeoutMs: 20000, pollIntervalMs: 500, platform: 'ios', deviceId: IOS_UDID })
|
|
21
|
+
console.log('iOS observe result:', JSON.stringify(iRes, null, 2))
|
|
22
|
+
|
|
23
|
+
} catch (e) { console.error('ERR', e); process.exit(1) }
|
|
24
|
+
})()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { ToolsInteract } from '../../../src/interact/index.js'
|
|
2
|
+
import * as Observe from '../../../src/observe/index.js'
|
|
3
|
+
|
|
4
|
+
async function runTests() {
|
|
5
|
+
console.log('Starting observe_until unit tests...')
|
|
6
|
+
|
|
7
|
+
const origFind = (ToolsInteract as any).findElementHandler
|
|
8
|
+
const origReadLog = (Observe as any).ToolsObserve.readLogStreamHandler
|
|
9
|
+
const origGetLogs = (Observe as any).ToolsObserve.getLogsHandler
|
|
10
|
+
const origGetFp = (Observe as any).ToolsObserve.getScreenFingerprintHandler
|
|
11
|
+
const origResolveObserve = (Observe as any).ToolsObserve.resolveObserve
|
|
12
|
+
const origGetScreenFp = (Observe as any).ToolsObserve.getScreenFingerprintHandler
|
|
13
|
+
|
|
14
|
+
try {
|
|
15
|
+
// Timeout / snapshot case: ensure snapshot captured when condition not met
|
|
16
|
+
const origCapture = (Observe as any).ToolsObserve.captureDebugSnapshotHandler
|
|
17
|
+
;(Observe as any).ToolsObserve.captureDebugSnapshotHandler = async ({ reason }: any) => ({ reason, fingerprint: 'snap-123', ui_tree: null, logs: [] })
|
|
18
|
+
// make findElement always fail
|
|
19
|
+
(ToolsInteract as any).findElementHandler = async () => ({ found: false })
|
|
20
|
+
const resTimeout = await ToolsInteract.observeUntilHandler({ type: 'ui', query: 'WillNeverExist', timeoutMs: 500, pollIntervalMs: 100, platform: 'android' })
|
|
21
|
+
const okTimeout = resTimeout && !(resTimeout as any).success && (resTimeout as any).snapshot && (resTimeout as any).snapshot.fingerprint === 'snap-123' && (resTimeout as any).telemetry && (resTimeout as any).telemetry.pollCount > 0
|
|
22
|
+
console.log('Timeout Snapshot Test:', okTimeout ? 'PASS' : 'FAIL', JSON.stringify((resTimeout as any).telemetry || {}, null, 2))
|
|
23
|
+
;(Observe as any).ToolsObserve.captureDebugSnapshotHandler = origCapture
|
|
24
|
+
|
|
25
|
+
// UI condition: findElement returns found on 2nd call
|
|
26
|
+
let calls = 0
|
|
27
|
+
;(ToolsInteract as any).findElementHandler = async (args) => {
|
|
28
|
+
calls++
|
|
29
|
+
const query = (args && (args.query || args)) || ''
|
|
30
|
+
if (calls >= 2) return { found: true, element: { text: query } }
|
|
31
|
+
return { found: false }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const resUi = await ToolsInteract.observeUntilHandler({ type: 'ui', query: 'Generate Session', timeoutMs: 3000, pollIntervalMs: 100, platform: 'android' })
|
|
35
|
+
const okUi = resUi && (resUi as any).success && (resUi as any).telemetry && (resUi as any).telemetry.pollCount > 0 && (resUi as any).telemetry.timeToMatch >= 0
|
|
36
|
+
console.log('UI Test:', okUi ? 'PASS' : 'FAIL', JSON.stringify((resUi as any).telemetry || {}, null, 2))
|
|
37
|
+
|
|
38
|
+
// Log condition: stream empty, snapshot contains matching line
|
|
39
|
+
;(Observe as any).ToolsObserve.readLogStreamHandler = async () => ({ entries: [ { message: 'nothing' } ] })
|
|
40
|
+
let glCalls = 0
|
|
41
|
+
;(Observe as any).ToolsObserve.getLogsHandler = async () => {
|
|
42
|
+
glCalls++
|
|
43
|
+
if (glCalls === 1) return { device: {}, logs: ['INFO start'] }
|
|
44
|
+
return { device: {}, logs: ['INFO start', 'ERROR Exception occurred', 'Server: Boom'] }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const resLog = await ToolsInteract.observeUntilHandler({ type: 'log', query: 'Server', timeoutMs: 3000, pollIntervalMs: 100, platform: 'android' })
|
|
48
|
+
const okLog = resLog && (resLog as any).success && (resLog as any).telemetry && (resLog as any).telemetry.pollCount > 0 && (resLog as any).telemetry.matchSource === 'log-snapshot'
|
|
49
|
+
console.log('Log Test:', okLog ? 'PASS' : 'FAIL', JSON.stringify((resLog as any).telemetry || {}, null, 2))
|
|
50
|
+
|
|
51
|
+
// Screen condition: fingerprint changes after a few polls
|
|
52
|
+
let seq = ['A', 'A', 'B']
|
|
53
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => ({ fingerprint: seq.length ? seq.shift() : null })
|
|
54
|
+
const resScreen = await ToolsInteract.observeUntilHandler({ type: 'screen', timeoutMs: 3000, pollIntervalMs: 100, platform: 'android' })
|
|
55
|
+
const okScreen = resScreen && (resScreen as any).success && (resScreen as any).telemetry && (resScreen as any).telemetry.matchSource === 'screen-fingerprint'
|
|
56
|
+
console.log('Screen Test:', okScreen ? 'PASS' : 'FAIL', JSON.stringify((resScreen as any).telemetry || {}, null, 2))
|
|
57
|
+
|
|
58
|
+
// Idle condition: stable fingerprints observed
|
|
59
|
+
let idleSeq = ['X', 'X', 'X']
|
|
60
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => ({ fingerprint: idleSeq.length ? idleSeq.shift() : 'X' })
|
|
61
|
+
const resIdle = await ToolsInteract.observeUntilHandler({ type: 'idle', timeoutMs: 3000, pollIntervalMs: 100, platform: 'android' })
|
|
62
|
+
const okIdle = resIdle && (resIdle as any).success && (resIdle as any).telemetry && (resIdle as any).telemetry.matchSource === 'idle-stable'
|
|
63
|
+
console.log('Idle Test:', okIdle ? 'PASS' : 'FAIL', JSON.stringify((resIdle as any).telemetry || {}, null, 2))
|
|
64
|
+
|
|
65
|
+
} finally {
|
|
66
|
+
;(ToolsInteract as any).findElementHandler = origFind
|
|
67
|
+
;(Observe as any).ToolsObserve.readLogStreamHandler = origReadLog
|
|
68
|
+
;(Observe as any).ToolsObserve.getLogsHandler = origGetLogs
|
|
69
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = origGetFp
|
|
70
|
+
;(Observe as any).ToolsObserve.resolveObserve = origResolveObserve
|
|
71
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = origGetScreenFp
|
|
72
|
+
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = origGetScreenFp
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
runTests().catch(console.error)
|
package/test/unit/index.ts
CHANGED
|
@@ -13,5 +13,6 @@ import '../manage/unit/mcp_disable_autodetect.test.ts'
|
|
|
13
13
|
import '../interact/unit/wait_for_screen_change.test.ts'
|
|
14
14
|
import '../observe/unit/capture_debug_snapshot.test.ts'
|
|
15
15
|
import '../observe/unit/find_element.test.ts'
|
|
16
|
+
import '../interact/unit/observe_until.test.ts'
|
|
16
17
|
|
|
17
18
|
console.log('Unit tests loaded.')
|